[WIP] Test workflow to support caching & less configuration.

.github/workflows/build.yaml

@@ -12,121 +12,76 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build_attest_all:
-    strategy:
-      fail-fast: false
-      matrix:
-        buildconfig:
-          - buildconfigs/agg_core.sh
-          - buildconfigs/fed_sql.sh
-          - buildconfigs/ledger_enclave_app.sh
-          - buildconfigs/sql_server.sh
+  build:
+    outputs:
+      binaries: ${{ steps.list-binaries.outputs.binaries }}
 
     permissions:
-      actions: read
-      id-token: write
-      attestations: write
       contents: read
 
     runs-on: ubuntu-20.04
 
     steps:
-      # Needed for GCS upload.
-      - name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@v2
+      - uses: actions/checkout@v4
+      - uses: bazel-contrib/[email protected]
         with:
-          credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY_JSON }}
-
-      # Needed for GCS upload.
-      - name: Setup Google Cloud
-        uses: google-github-actions/setup-gcloud@v2
-
-      - name: Mount main branch
-        uses: actions/checkout@v4
-
-      # Keeps two versions of SUBJECT_PATHS, with space resp. comma as
-      # path separator. Both are needed in later steps.
-      - name: Parse buildconfig
-        id: parse
+          bazelisk-cache: true
+          disk-cache: ${{ github.workflow }}
+          repository-cache: true
+      - name: Bazel Build
+        run: bazelisk run //:install_release_binaries -- --destdir="${{ runner.temp }}/binaries"
+      - name: List Binaries
+        id: list-binaries
         run: |
-          set -o errexit
-          set -o nounset
-          set -o pipefail
-          source ${{ matrix.buildconfig }}
-          echo "package-name=${PACKAGE_NAME}" >> "${GITHUB_OUTPUT}"
-          paths="${SUBJECT_PATHS[@]}"
-          echo "subject-paths=${paths}" >> "${GITHUB_OUTPUT}"
-          echo "subject-paths-commas=${paths// /,}" >> "${GITHUB_OUTPUT}"
+          set -euxo pipefail
+          binaries="$(find "${{ runner.temp }}/binaries" -type f -printf '%P\n' | jq -Rsc 'split("\n")[:-1]')"
+          echo "binaries=${binaries}" >> "$GITHUB_OUTPUT"
+      - uses: actions/upload-artifact@v4
+        with:
+          name: binaries
+          path: ${{ runner.temp }}/binaries/
+          if-no-files-found: error
+          retention-days: 5
 
-      - name: Show values
-        run: |
-          set -o errexit
-          set -o nounset
-          set -o pipefail
-          gsutil --version
-          echo "package_name: ${{ steps.parse.outputs.package-name }}"
-          echo "subject_paths: ${{ steps.parse.outputs.subject-paths }}"
-          echo "subject_paths_commas: ${{ steps.parse.outputs.subject-paths-commas }}"
-          echo "GITHUB_SHA: ${GITHUB_SHA}"
+  attest:
+    needs: build
 
-      - name: Build
-        id: build
-        run: |
-          set -o errexit
-          set -o nounset
-          set -o pipefail
-          source ${{ matrix.buildconfig }}
-          eval "${BUILD_COMMAND[@]}"
+    permissions:
+      id-token: write
+      attestations: write
 
-      - name: Show build artifacts
-        run: |
-          echo "${{ steps.parse.outputs.subject-paths }}"
-          ls -la ${{ steps.parse.outputs.subject-paths }}
+    runs-on: ubuntu-20.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+        binary: ${{ fromJson(needs.build.outputs.binaries) }}
 
-      - name: Attest
+    steps:
+      - uses: google-github-actions/auth@v2
+        with:
+          credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY_JSON }}
+      - uses: google-github-actions/setup-gcloud@v2
+        with:
+          skip_install: true
+      - uses: actions/download-artifact@v4
+        with:
+          name: binaries
+      - uses: actions/attest-build-provenance@v1
         id: attest
-        uses: actions/[email protected]
         with:
-          subject-path: ${{ steps.parse.outputs.subject-paths-commas }}
-
-      - name: Show bundle
-        run: |
-          echo "${{ steps.attest.outputs.bundle-path }}"
-          ls -la "${{ steps.attest.outputs.bundle-path }}"
-          cat "${{ steps.attest.outputs.bundle-path }}"
-
-      # Upload binary and provenance to GCS and index via http://static.space
-      # so that, regardless of the GCS bucket and path, it can easily be
-      # located by its digest.
+          subject-path: ${{ matrix.binary }}
       - name: Upload
-        id: upload
         run: |
-          set -o errexit
-          set -o nounset
-          set -o pipefail
-          set -o xtrace
+          set -euxo pipefail
 
           bucket=oak-bins
-          package_name=${{ steps.parse.outputs.package-name }}
-          subject_paths=( ${{ steps.parse.outputs.subject-paths }} )
-          binary_path="${subject_paths[0]}"
-          provenance_path=${{ steps.attest.outputs.bundle-path }}
+          binary_path="${{ matrix.binary }}"
+          package_name="${binary_path%%/*}"
+          provenance_path="${{ steps.attest.outputs.bundle-path }}"
 
           gcs_binary_path="binary/${GITHUB_SHA}/${package_name}/binary"
           gcs_provenance_path="provenance/${GITHUB_SHA}/${package_name}/attestation.jsonl"
-          binary_url="https://storage.googleapis.com/${bucket}/${gcs_binary_path}"
-          provenance_url="https://storage.googleapis.com/${bucket}/${gcs_provenance_path}"
 
           gsutil cp "${binary_path}" "gs://${bucket}/${gcs_binary_path}"
           gsutil cp "${provenance_path}" "gs://${bucket}/${gcs_provenance_path}"
-
-          curl --fail \
-            --request POST \
-            --header 'Content-Type: application/json' \
-            --data "{ \"url\": \"${binary_url}\" }" \
-            https://api.static.space/v1/snapshot
-          curl --fail \
-            --request POST \
-            --header 'Content-Type: application/json' \
-            --data "{ \"url\": \"${provenance_url}\" }" \
-            https://api.static.space/v1/snapshot

BUILD

@@ -31,18 +31,34 @@ platform(
     parents = ["@local_config_platform//:host"],
 )
 
-# All artifacts that will be built for release, along with their names in the
+# All release (i.e. production) binaries, along with their names in the
 # destination directory.
-_ALL_BINARIES = {
+_RELEASE_BINARIES = {
     "//containers/agg_core:oci_runtime_bundle.tar": "agg_core/container.tar",
-    "//containers/confidential_transform_test_concat:oci_runtime_bundle.tar": "confidential_transform_test_concat/container.tar",
     "//containers/fed_sql:oci_runtime_bundle.tar": "fed_sql/container.tar",
     "//containers/sql_server:oci_runtime_bundle.tar": "sql_server/container.tar",
+    "//ledger_enclave_app": "ledger/binary",
+    "//replicated_ledger_enclave_app": "replicated_ledger/binary",
+}
+
+pkg_files(
+    name = "release_binaries",
+    srcs = _RELEASE_BINARIES.keys(),
+    renames = _RELEASE_BINARIES,
+)
+
+pkg_install(
+    name = "install_release_binaries",
+    srcs = [":release_binaries"],
+)
+
+# All release and testing binaries, along with their names in the destination
+# directory.
+_ALL_BINARIES = _RELEASE_BINARIES | {
+    "//containers/confidential_transform_test_concat:oci_runtime_bundle.tar": "confidential_transform_test_concat/container.tar",
     "//containers/test_concat:oci_runtime_bundle.tar": "test_concat/container.tar",
     "//examples/square_enclave_app": "square_example/binary",
     "//examples/sum_enclave_app": "sum_example/binary",
-    "//ledger_enclave_app": "ledger/binary",
-    "//replicated_ledger_enclave_app": "replicated_ledger/binary",
 }
 
 pkg_files(
@@ -52,6 +68,6 @@ pkg_files(
 )
 
 pkg_install(
-    name = "install_binaries",
+    name = "install_all_binaries",
     srcs = [":all_binaries"],
 )

scripts/build.sh

@@ -46,10 +46,10 @@ elif [ "$1" == "release" ]; then
   # BINARY_OUTPUTS_DIR may be unset if this script is run manually; it'll
   # always be set during CI builds.
   if [[ -n "${BINARY_OUTPUTS_DIR}" ]]; then
-    ${BAZELISK} run -c opt //:install_binaries -- --destdir "${BINARY_OUTPUTS_DIR}"
+    ${BAZELISK} run -c opt //:install_all_binaries -- --destdir "${BINARY_OUTPUTS_DIR}"
   else
     # If unset, verify the binaries can be built with -c opt.
-    ${BAZELISK} build -c opt //:install_binaries
+    ${BAZELISK} build -c opt //:install_all_binaries
   fi
 else
   ${BAZELISK} test //...