ci.yaml

name: CI

on:
  pull_request: {}
  push:
    branches:
      - main

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

permissions:
  contents: write

jobs:
  git-sha:
    name: Git SHA
    runs-on: ubuntu-latest
    outputs:
      git_sha: ${{ steps.git_sha.outputs.git_sha }}
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/git-sha
        id: git_sha

  sanitize:
    name: Sanitize
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Setup go
        uses: actions/setup-go@v5
        with:
          go-version-file: go.mod
          cache-dependency-path: "**/*.sum"
      - name: Go vet
        run: |
          make vet
      - name: Lint
        uses: golangci/golangci-lint-action@v6
        with:
          version: v1.60.3

  test:
    name: Unit tests
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Setup go
        uses: actions/setup-go@v5
        with:
          go-version-file: go.mod
          cache-dependency-path: "**/*.sum"
      - name: Unit tests
        run: |
          make unit-tests

  check-operator-crds:
    name: Check operator CRDs
    runs-on: ubuntu-latest
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Make manifests
      run: make -C operator manifests
    - name: Check CRDs
      run: |
        git diff --exit-code --name-only
        if [ $? -eq 0 ]; then
          echo "CRDs are up to date"
        else
          echo "CRDs are out of date"
          exit 1
        fi

  buildtools:
    name: Build buildtools
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Setup go
        uses: actions/setup-go@v5
        with:
          go-version-file: go.mod
          cache-dependency-path: "**/*.sum"
      - name: Compile buildtools
        run: |
          make buildtools
      - name: Upload buildtools artifact
        uses: actions/upload-artifact@v4
        with:
          name: buildtools
          path: output/bin/buildtools

  build-current:
    name: Build current
    runs-on: ubuntu-latest
    needs:
      - git-sha
    outputs:
      k0s_version: ${{ steps.export.outputs.k0s_version }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Cache embedded bins
        uses: actions/cache@v4
        with:
          path: |
            output/bins
          key: bins-cache

      - name: Cache melange
        uses: actions/cache@v4
        with:
          path: |
            local-artifact-mirror/cache/.melange-cache
            operator/cache/.melange-cache
          key: melange-cache

      - name: Setup melange
        uses: chainguard-dev/actions/setup-melange@main

      - name: Setup go
        uses: actions/setup-go@v5
        with:
          go-version-file: go.mod
          cache-dependency-path: "**/*.sum"

      - name: Build
        env:
          APP_CHANNEL_ID: 2cHXb1RCttzpR0xvnNWyaZCgDBP
          APP_CHANNEL_SLUG: ci
          RELEASE_YAML_DIR: e2e/kots-release-install
          S3_BUCKET: "tf-staging-embedded-cluster-bin"
          USES_DEV_BUCKET: "0"
          AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_SECRET }}
          AWS_REGION: "us-east-1"
          USE_CHAINGUARD: "1"
          UPLOAD_BINARIES: "1"
          SKIP_RELEASE: "1"
        run: |
          export K0S_VERSION=$(make print-K0S_VERSION)
          export EC_VERSION=$(git describe --tags --match='[0-9]*.[0-9]*.[0-9]*')
          export SHORT_SHA=dev-${{ needs.git-sha.outputs.git_sha }}
          export APP_VERSION=appver-dev-${{ needs.git-sha.outputs.git_sha }}

          ./scripts/build-and-release.sh

      - name: Upload release
        uses: actions/upload-artifact@v4
        with:
          name: current-release
          path: |
            output/bin/embedded-cluster
            output/bin/embedded-cluster-original
            output/bin/embedded-cluster-release-builder

      - name: Export k0s version
        id: export
        run: |
          K0S_VERSION="$(make print-K0S_VERSION)"
          echo "K0S_VERSION=\"$K0S_VERSION\""
          echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT"

  build-previous-k0s:
    name: Build previous k0s
    runs-on: ubuntu-latest
    needs:
      - git-sha
    outputs:
      k0s_version: ${{ steps.export.outputs.k0s_version }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Cache embedded bins
        uses: actions/cache@v4
        with:
          path: |
            output/bins
          key: bins-cache

      - name: Cache melange
        uses: actions/cache@v4
        with:
          path: |
            local-artifact-mirror/cache/.melange-cache
            operator/cache/.melange-cache
          key: melange-cache

      - name: Setup melange
        uses: chainguard-dev/actions/setup-melange@main

      - name: Setup go
        uses: actions/setup-go@v5
        with:
          go-version-file: go.mod
          cache-dependency-path: "**/*.sum"

      - name: Build
        env:
          APP_CHANNEL_ID: 2cHXb1RCttzpR0xvnNWyaZCgDBP
          APP_CHANNEL_SLUG: ci
          RELEASE_YAML_DIR: e2e/kots-release-install
          S3_BUCKET: "tf-staging-embedded-cluster-bin"
          USES_DEV_BUCKET: "0"
          AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_SECRET }}
          AWS_REGION: "us-east-1"
          USE_CHAINGUARD: "1"
          UPLOAD_BINARIES: "1"
          SKIP_RELEASE: "1"
          MANGLE_METADATA: "1"
        run: |
          export K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION)
          export EC_VERSION=$(git describe --tags --match='[0-9]*.[0-9]*.[0-9]*')-previous-k0s
          export APP_VERSION=appver-dev-${{ needs.git-sha.outputs.git_sha }}-previous-k0s

          ./scripts/build-and-release.sh

      - name: Upload release
        uses: actions/upload-artifact@v4
        with:
          name: previous-k0s-release
          path: |
            output/bin/embedded-cluster-previous-k0s

      - name: Export k0s version
        id: export
        run: |
          K0S_VERSION="$(make print-PREVIOUS_K0S_VERSION)"
          echo "K0S_VERSION=\"$K0S_VERSION\""
          echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT"

  build-upgrade:
    name: Build upgrade
    runs-on: ubuntu-latest
    needs:
      - git-sha
    outputs:
      k0s_version: ${{ steps.export.outputs.k0s_version }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Cache embedded bins
        uses: actions/cache@v4
        with:
          path: |
            output/bins
          key: bins-cache

      - name: Cache melange
        uses: actions/cache@v4
        with:
          path: |
            local-artifact-mirror/cache/.melange-cache
            operator/cache/.melange-cache
          key: melange-cache

      - name: Setup melange
        uses: chainguard-dev/actions/setup-melange@main

      - name: Setup go
        uses: actions/setup-go@v5
        with:
          go-version-file: go.mod
          cache-dependency-path: "**/*.sum"

      - name: Build
        env:
          APP_CHANNEL_ID: 2cHXb1RCttzpR0xvnNWyaZCgDBP
          APP_CHANNEL_SLUG: ci
          RELEASE_YAML_DIR: e2e/kots-release-upgrade
          S3_BUCKET: "tf-staging-embedded-cluster-bin"
          USES_DEV_BUCKET: "0"
          AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_SECRET }}
          AWS_REGION: "us-east-1"
          USE_CHAINGUARD: "1"
          UPLOAD_BINARIES: "1"
          SKIP_RELEASE: "1"
          MANGLE_METADATA: "1"
        run: |
          export K0S_VERSION=$(make print-K0S_VERSION)
          export EC_VERSION=$(git describe --tags --match='[0-9]*.[0-9]*.[0-9]*')-upgrade
          export APP_VERSION=appver-dev-${{ needs.git-sha.outputs.git_sha }}-upgrade

          ./scripts/build-and-release.sh

      - name: Upload release
        uses: actions/upload-artifact@v4
        with:
          name: upgrade-release
          path: |
            output/bin/embedded-cluster-upgrade

      - name: Export k0s version
        id: export
        run: |
          K0S_VERSION="$(make print-K0S_VERSION)"
          echo "K0S_VERSION=\"$K0S_VERSION\""
          echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT"

  check-images:
    name: Check images
    runs-on: ubuntu-latest
    needs:
      - buildtools
      - build-current
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Download buildtools artifact
        uses: actions/download-artifact@v4
        with:
          name: buildtools
          path: output/bin
      - name: Download embedded-cluster artifact
        uses: actions/download-artifact@v4
        with:
          name: current-release
          path: output/bin
      - name: Check for missing images
        run: |
          chmod +x ./output/bin/buildtools
          chmod +x ./output/bin/embedded-cluster-original
          ./output/bin/embedded-cluster-original version metadata > version-metadata.json
          ./output/bin/embedded-cluster-original version list-images > expected.txt
          printf "Expected images:\n$(cat expected.txt)\n"
          ./output/bin/buildtools metadata extract-helm-chart-images --metadata-path version-metadata.json > images.txt
          printf "Found images:\n$(cat images.txt)\n"
          missing_images=0
          while read img; do
            grep -q "$img" expected.txt || { echo "Missing image: $img" && missing_images=$((missing_images+1)) ; }
          done <images.txt
          if [ $missing_images -gt 0 ]; then
            echo "Found $missing_images missing images"
            exit 1
          fi

  release-app:
    name: Create app releases
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write
    needs:
      - git-sha
      - build-current
      - build-previous-k0s
      - build-upgrade
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Install replicated CLI
        run: |
          curl --retry 5 --retry-all-errors -fs https://api.github.com/repos/replicatedhq/replicated/releases/latest \
          | grep "browser_download_url.*linux_amd64.tar.gz" \
          | cut -d : -f 2,3 \
          | tr -d \" \
          | wget -O replicated.tar.gz -qi -
          tar xf replicated.tar.gz replicated && rm replicated.tar.gz
          mv replicated /usr/local/bin/replicated
      - name: Create CI releases
        env:
          REPLICATED_APP: "embedded-cluster-smoke-test-staging-app"
          REPLICATED_API_TOKEN: ${{ secrets.STAGING_REPLICATED_API_TOKEN }}
          REPLICATED_API_ORIGIN: "https://api.staging.replicated.com/vendor"
          APP_CHANNEL: CI
          USES_DEV_BUCKET: "0"
        run: |
          export SHORT_SHA=dev-${{ needs.git-sha.outputs.git_sha }}

          # re-promote a release containing an old version of embedded-cluster to test upgrades
          export APP_VERSION="appver-${SHORT_SHA}-pre-minio-removal"
          replicated release promote 807 2cHXb1RCttzpR0xvnNWyaZCgDBP --version "${APP_VERSION}"

          # install the previous k0s version to ensure an upgrade occurs
          export EC_VERSION="$(git describe --tags --match='[0-9]*.[0-9]*.[0-9]*')-previous-k0s"
          export APP_VERSION="appver-${SHORT_SHA}-previous-k0s"
          export RELEASE_YAML_DIR=e2e/kots-release-install
          ./scripts/ci-release-app.sh

          # then install the current k0s version
          export EC_VERSION="$(git describe --tags --match='[0-9]*.[0-9]*.[0-9]*')"
          export APP_VERSION="appver-${SHORT_SHA}"
          export RELEASE_YAML_DIR=e2e/kots-release-install
          ./scripts/ci-release-app.sh

          # then a noop upgrade
          export EC_VERSION="$(git describe --tags --match='[0-9]*.[0-9]*.[0-9]*')"
          export APP_VERSION="appver-${SHORT_SHA}-noop"
          export RELEASE_YAML_DIR=e2e/kots-release-install
          ./scripts/ci-release-app.sh

          # and finally an app upgrade
          export EC_VERSION="$(git describe --tags --match='[0-9]*.[0-9]*.[0-9]*')-upgrade"
          export APP_VERSION="appver-${SHORT_SHA}-upgrade"
          export RELEASE_YAML_DIR=e2e/kots-release-upgrade
          ./scripts/ci-release-app.sh

      - name: Create airgap releases
        env:
          REPLICATED_APP: "embedded-cluster-smoke-test-staging-app"
          REPLICATED_API_TOKEN: ${{ secrets.STAGING_REPLICATED_API_TOKEN }}
          REPLICATED_API_ORIGIN: "https://api.staging.replicated.com/vendor"
          APP_CHANNEL: CI-airgap
          USES_DEV_BUCKET: "0"
        run: |
          export SHORT_SHA=dev-${{ needs.git-sha.outputs.git_sha }}

          # install the previous k0s version to ensure an upgrade occurs
          export EC_VERSION="$(git describe --tags --match='[0-9]*.[0-9]*.[0-9]*')-previous-k0s"
          export APP_VERSION="appver-${SHORT_SHA}-previous-k0s"
          export RELEASE_YAML_DIR=e2e/kots-release-install
          ./scripts/ci-release-app.sh

          # then install the current k0s version
          export EC_VERSION="$(git describe --tags --match='[0-9]*.[0-9]*.[0-9]*')"
          export APP_VERSION="appver-${SHORT_SHA}"
          export RELEASE_YAML_DIR=e2e/kots-release-install
          ./scripts/ci-release-app.sh

          # and finally an app upgrade
          export EC_VERSION="$(git describe --tags --match='[0-9]*.[0-9]*.[0-9]*')-upgrade"
          export APP_VERSION="appver-${SHORT_SHA}-upgrade"
          export RELEASE_YAML_DIR=e2e/kots-release-upgrade
          ./scripts/ci-release-app.sh

      - name: Create download link message text
        if: github.event_name == 'pull_request'
        run: |
          export SHORT_SHA=dev-${{ needs.git-sha.outputs.git_sha }}
          export EC_VERSION="$(git describe --tags --match='[0-9]*.[0-9]*.[0-9]*')"
          export APP_VERSION="appver-${SHORT_SHA}"

          echo "This PR has been released (on staging) and is available for download with a embedded-cluster-smoke-test-staging-app [license ID](https://vendor.staging.replicated.com/apps/embedded-cluster-smoke-test-staging-app/customers?sort=name-asc)." > download-link.txt
          echo "" >> download-link.txt
          echo "Online Installer:" >> download-link.txt
          echo "\`\`\`" >> download-link.txt
          echo "curl \"https://staging.replicated.app/embedded/embedded-cluster-smoke-test-staging-app/ci/${APP_VERSION}\" -H \"Authorization: \$EC_SMOKE_TEST_LICENSE_ID\" -o embedded-cluster-smoke-test-staging-app-ci.tgz" >> download-link.txt
          echo "\`\`\`" >> download-link.txt
          echo "Airgap Installer (may take a few minutes before the airgap bundle is built):" >> download-link.txt
          echo "\`\`\`" >> download-link.txt
          echo "curl \"https://staging.replicated.app/embedded/embedded-cluster-smoke-test-staging-app/ci-airgap/${APP_VERSION}?airgap=true\" -H \"Authorization: \$EC_SMOKE_TEST_LICENSE_ID\" -o embedded-cluster-smoke-test-staging-app-ci.tgz" >> download-link.txt
          echo "\`\`\`" >> download-link.txt
          echo "Happy debugging!" >> download-link.txt
          cat download-link.txt

      - name: Comment download link
        if: github.event_name == 'pull_request'
        uses: mshick/add-pr-comment@v2
        with:
          message-path: download-link.txt

  # e2e-docker runs the e2e tests inside a docker container rather than a full VM
  e2e-docker:
    name: E2E docker
    runs-on: ubuntu-latest
    needs:
      - build-current
    strategy:
      fail-fast: false
      matrix:
        test:
          - TestPreflights
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Download binary
        uses: actions/download-artifact@v4
        with:
          name: current-release
          path: output/bin
      - name: Setup go
        uses: actions/setup-go@v5
        with:
          go-version-file: go.mod
          cache-dependency-path: "**/*.sum"
      - name: Write license file
        run: |
          echo "${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE }}" | base64 --decode > e2e/license.yaml
      - name: Run test
        env:
          LICENSE_PATH: license.yaml
        run: |
          make e2e-test TEST_NAME=${{ matrix.test }}

  e2e:
    name: E2E
    runs-on: ${{ matrix.runner || 'ubuntu-latest' }}
    needs:
      - build-current
      - build-previous-k0s
      - build-upgrade
      - release-app
    strategy:
      fail-fast: false
      matrix:
        test:
          - TestSingleNodeInstallation
          - TestSingleNodeInstallationAlmaLinux8
          - TestSingleNodeInstallationDebian11
          - TestSingleNodeInstallationDebian12
          - TestSingleNodeInstallationCentos9Stream
          - TestVersion
          - TestHostPreflightCustomSpec
          - TestHostPreflightInBuiltSpec
          - TestUnsupportedOverrides
          - TestMultiNodeInstallation
          - TestMultiNodeReset
          - TestCommandsRequireSudo
          - TestInstallWithoutEmbed
          - TestInstallFromReplicatedApp
          - TestUpgradeFromReplicatedApp
          - TestUpgradeEC18FromReplicatedApp
          - TestResetAndReinstall
          - TestResetAndReinstallAirgap
          - TestCollectSupportBundle
          - TestOldVersionUpgrade
          - TestMaterialize
          - TestLocalArtifactMirror
          - TestSingleNodeAirgapUpgrade
          - TestSingleNodeAirgapUpgradeFromEC18
          - TestSingleNodeAirgapUpgradeCustomCIDR
          - TestInstallSnapshotFromReplicatedApp
          - TestMultiNodeAirgapUpgrade
          - TestSingleNodeDisasterRecovery
          - TestSingleNodeDisasterRecoveryWithProxy
          - TestSingleNodeResumeDisasterRecovery
          - TestProxiedEnvironment
          - TestMultiNodeHAInstallation
          - TestMultiNodeHADisasterRecovery
          - TestCustomCIDR
          - TestProxiedCustomCIDR
          - TestSingleNodeInstallationNoopUpgrade
          - TestInstallWithPrivateCAs
        include:
          - test: TestMultiNodeAirgapUpgrade
            runner: embedded-cluster
          - test: TestMultiNodeAirgapUpgradeSameK0s
            runner: embedded-cluster
          - test: TestSingleNodeAirgapDisasterRecovery
            runner: embedded-cluster
          - test: TestMultiNodeAirgapHAInstallation
            runner: embedded-cluster
          - test: TestMultiNodeAirgapHADisasterRecovery
            runner: embedded-cluster
          - test: TestFiveNodesAirgapUpgrade
            runner: embedded-cluster
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Download current binary
        uses: actions/download-artifact@v4
        with:
          name: current-release
          path: output/bin

      - uses: ./.github/actions/e2e
        with:
          test-name: '${{ matrix.test }}'
          is-large-runner: ${{ matrix.runner == 'embedded-cluster' }}
          airgap-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_AIRGAP_LICENSE_ID }}
          snapshot-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE_ID }}
          snapshot-license: ${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE }}
          airgap-snapshot-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_AIRGAP_SNAPSHOT_LICENSE_ID }}
          license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE_ID }}
          license: ${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE }}
          dr-aws-access-key-id: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }}
          dr-aws-secret-access-key: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }}
          k0s-version: ${{ needs.build-current.outputs.k0s_version }}
          k0s-version-previous: ${{ needs.build-previous-k0s.outputs.k0s_version }}

  # this job will validate that all the tests passed
  # it is used for the github branch protection rule
  validate-success:
    name: Validate success
    runs-on: ubuntu-20.04
    needs:
      - e2e
      - e2e-docker
      - sanitize
      - test
      - check-images
      - check-operator-crds
    if: always()
    steps:
      # https://docs.github.com/en/actions/learn-github-actions/contexts#needs-context
      - name: fail if e2e job was not successful
        if: needs.e2e.result != 'success'
        run: exit 1
      - name: fail if sanitize job was not successful
        if: needs.sanitize.result != 'success'
        run: exit 1
      - name: fail if tests job was not successful
        if: needs.test.result != 'success'
        run: exit 1
      - name: succeed if everything else passed
        run: echo "Validation succeeded"