automated postrelease testing (#1460)

* wip postrelease testing

* fix replicated CLI install

* f

* fix bin download

* disable CI again

* needs get-tag

* test reset+reinstall with actual build, not previous k0s

* add previous-stable upgrade tests, build 'previous-k0s' app version

* that's silly

* f

* fix previous-stable test

* reenable CI

* disable almost all of CI again

* pass the post-upgrade target ec version to the check instead of expecting a static suffix

* add new param in CI as well

* reenable CI

* fix previous version upgrade onrelease test

* fix upgrade version specifier

* include upgrade version specifier in docker tests

.github/actions/e2e/action.yml

@@ -43,6 +43,9 @@ inputs:
   version-specifier:
     description: 'the git sha or tag used to generate application version strings'
     required: true
+  upgrade-target-ec-version:
+    description: 'the embedded cluster version to expect after upgrades complete'
+    required: false # this is only set by post-release testing
 
 runs:
   using: composite
@@ -110,6 +113,7 @@ runs:
       export EXPECT_K0S_VERSION=${{ inputs.k0s-version }}
       export EXPECT_K0S_VERSION_PREVIOUS=${{ inputs.k0s-version-previous }}
       export EXPECT_K0S_VERSION_PREVIOUS_STABLE=${{ inputs.k0s-version-previous-stable }}
+      export EXPECT_EMBEDDED_CLUSTER_UPGRADE_TARGET_VERSION=${{ inputs.upgrade-target-ec-version }}
       make e2e-test TEST_NAME=${{ inputs.test-name }}
   - name: Troubleshoot
     if: ${{ !cancelled() }}

.github/workflows/ci.yaml

@@ -552,6 +552,7 @@ jobs:
       - build-upgrade
       - build-previous-stable
       - release-app
+      - export-version-specifier
     strategy:
       fail-fast: false
       matrix:

.github/workflows/release-prod.yaml

@@ -211,3 +211,168 @@ jobs:
           files: |
             build/*.tgz
             build/metadata.json
+
+  build-previous-stable:
+    name: Build previous stable
+    runs-on: embedded-cluster
+    needs:
+      - get-tag
+    outputs:
+      ec_version: ${{ steps.export.outputs.ec_version }}
+      k0s_version: ${{ steps.export.outputs.k0s_version }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Export k0s version
+        id: export
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          k0s_majmin_version="$(make print-PREVIOUS_K0S_VERSION | sed 's/v\([0-9]*\.[0-9]*\).*/\1/')"
+          EC_VERSION="$(gh release list --repo replicatedhq/embedded-cluster \
+            --exclude-drafts --exclude-pre-releases --json name \
+            --jq '.[] | .name' \
+            | grep "k8s-${k0s_majmin_version}" \
+            | head -n1)"
+
+          gh release download "$EC_VERSION" --repo replicatedhq/embedded-cluster --pattern 'metadata.json'
+          K0S_VERSION="$(jq -r '.Versions.Kubernetes' metadata.json)"
+
+          echo "EC_VERSION=\"$EC_VERSION\""
+          echo "K0S_VERSION=\"$K0S_VERSION\""
+          echo "ec_version=$EC_VERSION" >> "$GITHUB_OUTPUT"
+          echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT"
+
+  release-app:
+    name: Create app releases
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    needs:
+      - release
+      - get-tag
+      - build-previous-stable
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install replicated CLI
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh release download --repo replicatedhq/replicated --pattern '*linux_amd64.tar.gz' --output replicated.tar.gz
+          tar xf replicated.tar.gz replicated && rm replicated.tar.gz
+          mv replicated /usr/local/bin/replicated
+      - name: Create CI releases
+        env:
+          REPLICATED_APP: "embedded-cluster-smoke-test-staging-app"
+          REPLICATED_API_TOKEN: ${{ secrets.STAGING_REPLICATED_API_TOKEN }}
+          REPLICATED_API_ORIGIN: "https://api.staging.replicated.com/vendor"
+          APP_CHANNEL: CI
+          USES_DEV_BUCKET: "0"
+        run: |
+          # install the current k0s version	
+          export EC_VERSION="${{ needs.get-tag.outputs.tag-name }}"	
+          export APP_VERSION="appver-${{ needs.get-tag.outputs.tag-name }}"	
+          export RELEASE_YAML_DIR=e2e/kots-release-install	
+          ./scripts/ci-release-app.sh	
+          # and finally an app upgrade	
+          export EC_VERSION="${{ needs.get-tag.outputs.tag-name }}-upgrade"	
+          export APP_VERSION="appver-${{ needs.get-tag.outputs.tag-name }}-upgrade"	
+          export RELEASE_YAML_DIR=e2e/kots-release-upgrade	
+          ./scripts/ci-release-app.sh
+      - name: Create airgap releases
+        env:
+          REPLICATED_APP: "embedded-cluster-smoke-test-staging-app"
+          REPLICATED_API_TOKEN: ${{ secrets.STAGING_REPLICATED_API_TOKEN }}
+          REPLICATED_API_ORIGIN: "https://api.staging.replicated.com/vendor"
+          APP_CHANNEL: CI-airgap
+          USES_DEV_BUCKET: "0"
+        run: |
+          # re-promote a release containing an old version of embedded-cluster to test upgrades
+          export APP_VERSION="appver-${{ needs.get-tag.outputs.tag-name }}-1.8.0-k8s-1.28"
+          replicated release promote 11615 2cHXb1RCttzpR0xvnNWyaZCgDBP --version "${APP_VERSION}"
+          replicated release promote 11615 2eAqMYG1IEtX8cwpaO1kgNV6EB3 --version "${APP_VERSION}"
+          
+          # promote a release containing the previous stable version of embedded-cluster to test upgrades
+          export EC_VERSION="${{ needs.build-previous-stable.outputs.ec_version }}"
+          export APP_VERSION="appver-${{ needs.get-tag.outputs.tag-name }}-previous-stable"
+          export RELEASE_YAML_DIR=e2e/kots-release-install-stable
+          ./scripts/ci-release-app.sh
+          # promote a release with the current k0s version, but call it the previous version to test noop upgrades
+          export EC_VERSION="${{ needs.get-tag.outputs.tag-name }}"	
+          export APP_VERSION="appver-${{ needs.get-tag.outputs.tag-name }}-previous-k0s"
+          export RELEASE_YAML_DIR=e2e/kots-release-install
+          ./scripts/ci-release-app.sh
+          # promote a release with the current k0s version
+          export EC_VERSION="${{ needs.get-tag.outputs.tag-name }}"	
+          export APP_VERSION="appver-${{ needs.get-tag.outputs.tag-name }}"	
+          export RELEASE_YAML_DIR=e2e/kots-release-install
+          ./scripts/ci-release-app.sh
+          # and finally an app upgrade	
+          export EC_VERSION="${{ needs.get-tag.outputs.tag-name }}"	
+          export APP_VERSION="appver-${{ needs.get-tag.outputs.tag-name }}-upgrade"	
+          export RELEASE_YAML_DIR=e2e/kots-release-upgrade
+          ./scripts/ci-release-app.sh
+  e2e:
+    name: E2E
+    runs-on: ${{ matrix.runner || 'ubuntu-22.04' }}
+    needs:
+      - release
+      - release-app
+      - get-tag
+      - build-previous-stable
+    strategy:
+      fail-fast: false
+      matrix:
+        test:
+          - TestResetAndReinstallAirgap
+          - TestSingleNodeAirgapUpgrade
+        include:
+          - test: TestMultiNodeAirgapUpgrade
+            runner: embedded-cluster
+          - test: TestAirgapUpgradeFromEC18
+            runner: embedded-cluster
+          - test: TestMultiNodeAirgapUpgradePreviousStable
+            runner: embedded-cluster
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - uses: ./.github/actions/e2e
+        with:
+          test-name: '${{ matrix.test }}'
+          is-large-runner: ${{ matrix.runner == 'embedded-cluster' }}
+          airgap-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_AIRGAP_LICENSE_ID }}
+          snapshot-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE_ID }}
+          snapshot-license: ${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE }}
+          airgap-snapshot-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_AIRGAP_SNAPSHOT_LICENSE_ID }}
+          license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE_ID }}
+          license: ${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE }}
+          dr-aws-access-key-id: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }}
+          dr-aws-secret-access-key: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }}
+          version-specifier: ${{ needs.get-tag.outputs.tag-name }}
+
+          k0s-version: ${{ needs.get-tag.outputs.k0s_version }}
+          k0s-version-previous: ${{ needs.get-tag.outputs.k0s_version }} # we do not run k8s upgrade tests on release
+          k0s-version-previous-stable: ${{ needs.build-previous-stable.outputs.k0s_version }}
+          upgrade-target-ec-version: ${{ needs.get-tag.outputs.tag-name }}
+
+  # this job will validate that all the tests passed
+  # it is used for the github branch protection rule
+  validate-release-success:
+    name: Validate success
+    runs-on: ubuntu-20.04
+    needs:
+      - e2e
+      - release
+      - release-app
+    if: always()
+    steps:
+      # https://docs.github.com/en/actions/learn-github-actions/contexts#needs-context
+      - name: fail if e2e job was not successful
+        if: needs.e2e.result != 'success'
+        run: exit 1
+      - name: succeed if everything else passed
+        run: echo "Validation succeeded"