diff --git a/.github/workflows/build-targets.yml b/.github/workflows/build-targets.yml
index a57ffddd..f0d3e9df 100644
--- a/.github/workflows/build-targets.yml
+++ b/.github/workflows/build-targets.yml
@@ -187,12 +187,12 @@ jobs:
           targets_to_update=()
           sha_of_targets_to_update=()
           build_directories_of_targets_to_update=()
-          skipped_targets=()
+          up_to_date_targets=()
 
           for target in "${!target_shas[@]}"; do
               if [[ -v release_shas["$target"] && "${target_shas[$target]}" == "${release_shas[$target]}" ]]; then
                   echo "Skipping $target due to same SHA from release"
-                  skipped_targets+=("${target}")
+                  up_to_date_targets+=("${target}")
               else
                   echo "Adding $target to the update list"
                   targets_to_update+=("${target}")
@@ -204,9 +204,28 @@ jobs:
           echo "TARGETS_TO_UPDATE=${targets_to_update[@]}" >>$GITHUB_ENV
           echo "SHA_OF_TARGETS_TO_UPDATE=${sha_of_targets_to_update[@]}" >>$GITHUB_ENV
           echo "BUILD_DIRECTORIES_OF_TARGETS_TO_UPDATE=${build_directories_of_targets_to_update[@]}" >>$GITHUB_ENV
-          echo "SKIPPED_TARGETS=${skipped_targets}" >>$GITHUB_ENV
+          echo "UP_TO_DATE_TARGETS=${up_to_date_targets}" >>$GITHUB_ENV
         shell: bash
 
+      - name: Download up to date targets
+        working-directory: ${{env.BUILD_DIRECTORY}}
+        run: |
+          up_to_date_targets=($UP_TO_DATE_TARGETS)
+
+          for target in "${up_to_date_targets[@]}"; do
+              compressed_target="${target}-lib.tar.gz"
+              echo Downloading ${compressed_target}
+              gh release download ${RELEASE_TAG} --pattern "${COMPRESSED_TARGET}" --clobber
+          done
+        shell: bash
+
+      - name: Restore `src` directories
+        uses: actions/cache@v4
+        with:
+            path: ${{env.BUILD_DIRECTORY}}/*/src
+            key: ${{env.RELEASE_TAG}}-src
+            restore-keys: ${{env.RELEASE_TAG}}-src
+
       - name: Build targets in POSIX hosts
         working-directory: ${{env.BUILD_DIRECTORY}}
         if: ${{matrix.host != 'windows-latest'}}
@@ -265,3 +284,9 @@ jobs:
           echo "$json_data" >release.json
           gh release upload --clobber ${RELEASE_TAG} release.json
         shell: bash
+
+      - name: Save `src` directories
+        uses: actions/cache@v4
+        with:
+            path: ${{env.BUILD_DIRECTORY}}/*/src
+            key: ${{env.RELEASE_TAG}}-src