Cache fetched submodules across jobs

Saves storage and bandwidth by cloning the submodules only once per workflow run. I also do some small steps to reduce the size of checked out repos so that the size of the cache is reduced. Signed-off-by: Nick Kossifidis <[email protected]>
riscv-collab · Nov 2, 2024 · c45ac89 · c45ac89
1 parent bd33836
commit c45ac89
Show file tree

Hide file tree

Showing 2 changed files with 86 additions and 0 deletions.
diff --git a/.github/dedup-dir.sh b/.github/dedup-dir.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+function deduplicate_files() {
+	local DUPLICATES=()
+	local DIR=${1}
+	local OLDIFS=${IFS}
+	local LINK_CHECK=""
+
+	readarray -t DUPLICATES < <(for i in `find ${DIR} -type f ! -empty`; do sha1sum ${i}; done | sort | uniq -w 40 --all-repeated=separate)
+
+	for ((i=1; i < ${#DUPLICATES[@]}; i++ )); do
+		if [[ ${DUPLICATES[$i]} == "" ]]; then
+			continue
+		elif [[ ${DUPLICATES[$i-1]} = "" ]]; then
+			continue
+		else
+			LINK_CHECK=$(ls -li "${DUPLICATES[$i]:42}" "${DUPLICATES[$i-1]:42}" |awk '{print $1}' | uniq | wc -l)
+			if [[ ${LINK_CHECK} != "1" ]]; then
+				ln -f "${DUPLICATES[$i-1]:42}" "${DUPLICATES[$i]:42}"
+			fi
+		fi
+	done
+}
+
+deduplicate_files ${1}
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -10,8 +10,38 @@ on:
       - master
 
 jobs:
+  submodule_cache:
+    name: Initialize submodule cache
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Checkout required submodules
+        run: git submodule update --init --progress -j $(nproc) --depth 1 binutils gcc gdb glibc llvm musl newlib uclibc-ng
+
+      - name: Storage size optimization
+        run: |
+          git submodule foreach 'git maintenance run'
+          ./.github/dedup-dir.sh ./.git/modules
+
+      - name: Setup submodule cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            binutils
+            gcc
+            gdb
+            glibc
+            llvm
+            musl
+            newlib
+            uclibc-ng
+            .git/modules
+          key: submodule-cache-${{ github.run_id }}-${{ github.run_attempt }}
+
   build:
     runs-on: ${{ matrix.os }}
+    needs: [submodule_cache]
     strategy:
       matrix:
         os:     [ubuntu-latest]
@@ -38,6 +68,21 @@ jobs:
       - name: install dependencies
         run: sudo ./.github/setup-apt.sh
 
+      - name: Load submodule cache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            binutils
+            gcc
+            gdb
+            glibc
+            llvm
+            musl
+            newlib
+            uclibc-ng
+            .git/modules
+          key: submodule-cache-${{ github.run_id }}-${{ github.run_attempt }}
+
       - name: build toolchain
         run: |
           TARGET_TUPLE=($(echo ${{ matrix.target }} | tr "-" "\n"))
@@ -88,6 +133,7 @@ jobs:
 
   test-sim:
     runs-on: ${{ matrix.os }}
+    needs: [submodule_cache]
     strategy:
       matrix:
         os:     [ubuntu-latest]
@@ -109,6 +155,21 @@ jobs:
       - name: install dependencies
         run: sudo ./.github/setup-apt.sh
 
+      - name: Load submodule cache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            binutils
+            gcc
+            gdb
+            glibc
+            llvm
+            musl
+            newlib
+            uclibc-ng
+            .git/modules
+          key: submodule-cache-${{ github.run_id }}-${{ github.run_attempt }}
+
       - name: build toolchain
         run: |
           TARGET_TUPLE=($(echo ${{ matrix.target }} | tr "-" "\n"))