Skip to content

Commit

Permalink
chore(ci): e2e test previous stable release (#1450)
Browse files Browse the repository at this point in the history 
* chore(ci): e2e test previous stable release

* f

* f

* f

* f

* f

* use correct stable ec version for previous k0s release

* use correct stable ec version for previous k0s release

* use correct stable ec version for previous k0s release

* use correct stable ec version for previous k0s release

* use correct stable ec version for previous k0s release

* use correct stable ec version for previous k0s release
  • Loading branch information
@emosbaugh
emosbaugh authored Nov 5, 2024
1 parent c4a63f8 commit 789bdf5
Show file tree
Hide file tree
Showing 13 changed files with 557 additions and 0 deletions.
4 changes: 4 additions & 0 deletions .github/actions/e2e/action.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ inputs:
k0s-version-previous:
description: 'k0s previous version to expect in e2e tests'
required: true
k0s-version-previous-stable:
description: 'k0s previous stable version to expect in e2e tests'
required: true
version-specifier:
description: 'the git sha or tag used to generate application version strings'
required: true
Expand Down Expand Up @@ -106,6 +109,7 @@ runs:
export DR_AWS_SECRET_ACCESS_KEY=${{ inputs.dr-aws-secret-access-key }}
export EXPECT_K0S_VERSION=${{ inputs.k0s-version }}
export EXPECT_K0S_VERSION_PREVIOUS=${{ inputs.k0s-version-previous }}
export EXPECT_K0S_VERSION_PREVIOUS_STABLE=${{ inputs.k0s-version-previous-stable }}
make e2e-test TEST_NAME=${{ inputs.test-name }}
- name: Troubleshoot
if: ${{ !cancelled() }}
Expand Down
52 changes: 52 additions & 0 deletions .github/workflows/ci.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,37 @@ jobs:
echo "K0S_VERSION=\"$K0S_VERSION\""
echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT"
build-previous-stable:
name: Build previous stable
runs-on: embedded-cluster
needs:
- git-sha
outputs:
ec_version: ${{ steps.export.outputs.ec_version }}
k0s_version: ${{ steps.export.outputs.k0s_version }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Export k0s version
id: export
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
k0s_majmin_version="$(make print-PREVIOUS_K0S_VERSION | sed 's/v\([0-9]*\.[0-9]*\).*/\1/')"
EC_VERSION="$(gh release list --repo replicatedhq/embedded-cluster \
--exclude-drafts --exclude-pre-releases --json name \
--jq '.[] | .name' \
| grep "k8s-${k0s_majmin_version}" \
| head -n1)"
gh release download "$EC_VERSION" --repo replicatedhq/embedded-cluster --pattern 'metadata.json'
K0S_VERSION="$(jq -r '.Versions.Kubernetes' metadata.json)"
echo "EC_VERSION=\"$EC_VERSION\""
echo "K0S_VERSION=\"$K0S_VERSION\""
echo "ec_version=$EC_VERSION" >> "$GITHUB_OUTPUT"
echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT"
build-upgrade:
name: Build upgrade
runs-on: embedded-cluster
Expand Down Expand Up @@ -387,6 +418,7 @@ jobs:
- build-current
- build-previous-k0s
- build-upgrade
- build-previous-stable
steps:
- name: Checkout
uses: actions/checkout@v4
Expand All @@ -406,6 +438,7 @@ jobs:
REPLICATED_API_ORIGIN: "https://api.staging.replicated.com/vendor"
APP_CHANNEL: CI
USES_DEV_BUCKET: "0"
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
export SHORT_SHA=dev-${{ needs.git-sha.outputs.git_sha }}
Expand All @@ -418,6 +451,12 @@ jobs:
replicated release promote 11615 2cHXb1RCttzpR0xvnNWyaZCgDBP --version "${APP_VERSION}"
replicated release promote 11615 2eAqMYG1IEtX8cwpaO1kgNV6EB3 --version "${APP_VERSION}"
# promote a release containing the previous stable version of embedded-cluster to test upgrades
export EC_VERSION="${{ needs.build-previous-stable.outputs.ec_version }}"
export APP_VERSION="appver-${SHORT_SHA}-previous-stable"
export RELEASE_YAML_DIR=e2e/kots-release-install-stable
./scripts/ci-release-app.sh
# install the previous k0s version to ensure an upgrade occurs
export EC_VERSION="$(git describe --tags --match='[0-9]*.[0-9]*.[0-9]*')-previous-k0s"
export APP_VERSION="appver-${SHORT_SHA}-previous-k0s"
Expand Down Expand Up @@ -452,6 +491,12 @@ jobs:
run: |
export SHORT_SHA=dev-${{ needs.git-sha.outputs.git_sha }}
# promote a release containing the previous stable version of embedded-cluster to test upgrades
export EC_VERSION="${{ needs.build-previous-stable.outputs.ec_version }}"
export APP_VERSION="appver-${SHORT_SHA}-previous-stable"
export RELEASE_YAML_DIR=e2e/kots-release-install-stable
./scripts/ci-release-app.sh
# install the previous k0s version to ensure an upgrade occurs
export EC_VERSION="$(git describe --tags --match='[0-9]*.[0-9]*.[0-9]*')-previous-k0s"
export APP_VERSION="appver-${SHORT_SHA}-previous-k0s"
Expand Down Expand Up @@ -505,6 +550,7 @@ jobs:
- build-current
- build-previous-k0s
- build-upgrade
- build-previous-stable
- release-app
strategy:
fail-fast: false
Expand All @@ -519,6 +565,7 @@ jobs:
- TestSingleNodeInstallationDebian11
- TestSingleNodeInstallationDebian12
- TestSingleNodeInstallationCentos9Stream
- TestSingleNodeUpgradePreviousStable
- TestInstallFromReplicatedApp
- TestUpgradeFromReplicatedApp
- TestInstallWithoutEmbed
Expand Down Expand Up @@ -578,6 +625,7 @@ jobs:
DR_AWS_SECRET_ACCESS_KEY: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }}
EXPECT_K0S_VERSION: ${{ needs.build-current.outputs.k0s_version }}
EXPECT_K0S_VERSION_PREVIOUS: ${{ needs.build-previous-k0s.outputs.k0s_version }}
EXPECT_K0S_VERSION_PREVIOUS_STABLE: ${{ needs.build-previous-stable.outputs.k0s_version }}
run: |
make e2e-test TEST_NAME=${{ matrix.test }}
- name: Troubleshoot
Expand All @@ -593,6 +641,7 @@ jobs:
- build-current
- build-previous-k0s
- build-upgrade
- build-previous-stable
- release-app
- export-version-specifier
strategy:
Expand All @@ -614,6 +663,8 @@ jobs:
runner: embedded-cluster
- test: TestMultiNodeAirgapUpgradeSameK0s
runner: embedded-cluster
- test: TestMultiNodeAirgapUpgradePreviousStable
runner: embedded-cluster
- test: TestAirgapUpgradeFromEC18
runner: embedded-cluster
- test: TestSingleNodeAirgapDisasterRecovery
Expand Down Expand Up @@ -647,6 +698,7 @@ jobs:
dr-aws-secret-access-key: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }}
k0s-version: ${{ needs.build-current.outputs.k0s_version }}
k0s-version-previous: ${{ needs.build-previous-k0s.outputs.k0s_version }}
k0s-version-previous-stable: ${{ needs.build-previous-stable.outputs.k0s_version }}
version-specifier: ${{ needs.export-version-specifier.outputs.version_specifier }}

# this job will validate that all the tests passed
Expand Down
197 changes: 197 additions & 0 deletions e2e/install_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -485,6 +485,57 @@ func TestInstallFromReplicatedApp(t *testing.T) {
t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
}

func TestSingleNodeUpgradePreviousStable(t *testing.T) {
t.Parallel()

RequireEnvVars(t, []string{"SHORT_SHA"})

tc := docker.NewCluster(&docker.ClusterInput{
T: t,
Nodes: 1,
Distro: "debian-bookworm",
})
defer tc.Cleanup()

t.Logf("%s: downloading embedded-cluster on node 0", time.Now().Format(time.RFC3339))
line := []string{"vandoor-prepare.sh", fmt.Sprintf("appver-%s-previous-stable", os.Getenv("SHORT_SHA")), os.Getenv("LICENSE_ID"), "false"}
if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
t.Fatalf("fail to download embedded-cluster on node 0: %v: %s: %s", err, stdout, stderr)
}

t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
line = []string{"single-node-install.sh", "ui"}
if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
t.Fatalf("fail to install embedded-cluster on node 0: %v: %s: %s", err, stdout, stderr)
}

if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil {
t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr)
}

t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339))
line = []string{"check-installation-state.sh", fmt.Sprintf("appver-%s-previous-stable", os.Getenv("SHORT_SHA")), k8sVersionPreviousStable()}
if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
t.Fatalf("fail to check installation state: %v: %s: %s", err, stdout, stderr)
}

appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
testArgs := []string{appUpgradeVersion}

t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
if stdout, stderr, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil {
t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr)
}

t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
line = []string{"check-postupgrade-state.sh", k8sVersion()}
if stdout, stderr, err := tc.RunCommandOnNode(0, line); err != nil {
t.Fatalf("fail to check postupgrade state: %v: %s: %s", err, stdout, stderr)
}

t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
}

func TestUpgradeFromReplicatedApp(t *testing.T) {
t.Parallel()

Expand Down Expand Up @@ -1513,6 +1564,152 @@ func TestMultiNodeAirgapUpgrade(t *testing.T) {
t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
}

func TestMultiNodeAirgapUpgradePreviousStable(t *testing.T) {
t.Parallel()

RequireEnvVars(t, []string{"SHORT_SHA", "AIRGAP_LICENSE_ID"})

t.Logf("%s: downloading airgap files", time.Now().Format(time.RFC3339))
airgapInstallBundlePath := "/tmp/airgap-install-bundle.tar.gz"
airgapUpgradeBundlePath := "/tmp/airgap-upgrade-bundle.tar.gz"
runInParallel(t,
func(t *testing.T) error {
return downloadAirgapBundle(t, fmt.Sprintf("appver-%s-previous-stable", os.Getenv("SHORT_SHA")), airgapInstallBundlePath, os.Getenv("AIRGAP_LICENSE_ID"))
}, func(t *testing.T) error {
return downloadAirgapBundle(t, fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA")), airgapUpgradeBundlePath, os.Getenv("AIRGAP_LICENSE_ID"))
},
)

tc := lxd.NewCluster(&lxd.ClusterInput{
T: t,
Nodes: 2,
Image: "debian/12",
WithProxy: true,
AirgapInstallBundlePath: airgapInstallBundlePath,
AirgapUpgradeBundlePath: airgapUpgradeBundlePath,
})
defer tc.Cleanup()

// install "curl" dependency on node 0 for app version checks.
tc.InstallTestDependenciesDebian(t, 0, true)

// delete airgap bundles once they've been copied to the nodes
if err := os.Remove(airgapInstallBundlePath); err != nil {
t.Logf("failed to remove airgap install bundle: %v", err)
}
if err := os.Remove(airgapUpgradeBundlePath); err != nil {
t.Logf("failed to remove airgap upgrade bundle: %v", err)
}

// upgrade airgap bundle is only needed on the first node
line := []string{"rm", "/assets/ec-release-upgrade.tgz"}
if _, _, err := tc.RunCommandOnNode(1, line); err != nil {
t.Fatalf("fail to remove upgrade airgap bundle on node %s: %v", tc.Nodes[1], err)
}

t.Logf("%s: preparing embedded cluster airgap files on node 0", time.Now().Format(time.RFC3339))
line = []string{"airgap-prepare.sh"}
if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
t.Fatalf("fail to prepare airgap files on node %s: %v", tc.Nodes[0], err)
}

t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
line = []string{"single-node-airgap-install.sh", "--local-artifact-mirror-port", "50001"} // choose an alternate lam port
if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
}
// remove the airgap bundle and binary after installation
line = []string{"rm", "/assets/release.airgap"}
if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
t.Fatalf("fail to remove airgap bundle on node %s: %v", tc.Nodes[0], err)
}
line = []string{"rm", "/usr/local/bin/embedded-cluster"}
if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
t.Fatalf("fail to remove embedded-cluster binary on node %s: %v", tc.Nodes[0], err)
}

if _, _, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil {
t.Fatalf("fail to run playwright test deploy-app: %v", err)
}

// generate worker node join command.
t.Logf("%s: generating a new worker token command", time.Now().Format(time.RFC3339))
stdout, stderr, err := tc.RunPlaywrightTest("get-join-worker-command")
if err != nil {
t.Fatalf("fail to generate worker join token:\nstdout: %s\nstderr: %s", stdout, stderr)
}
workerCommand, err := findJoinCommandInOutput(stdout)
if err != nil {
t.Fatalf("fail to find the join command in the output: %v", err)
}
t.Log("worker join token command:", workerCommand)

// join the worker node
t.Logf("%s: preparing embedded cluster airgap files on worker node", time.Now().Format(time.RFC3339))
line = []string{"airgap-prepare.sh"}
if _, _, err := tc.RunCommandOnNode(1, line); err != nil {
t.Fatalf("fail to prepare airgap files on worker node: %v", err)
}
t.Logf("%s: joining worker node to the cluster", time.Now().Format(time.RFC3339))
if _, _, err := tc.RunCommandOnNode(1, strings.Split(workerCommand, " ")); err != nil {
t.Fatalf("fail to join worker node to the cluster: %v", err)
}
// remove the airgap bundle and binary after joining
line = []string{"rm", "/assets/release.airgap"}
if _, _, err := tc.RunCommandOnNode(1, line); err != nil {
t.Fatalf("fail to remove airgap bundle on worker node: %v", err)
}
line = []string{"rm", "/usr/local/bin/embedded-cluster"}
if _, _, err := tc.RunCommandOnNode(1, line); err != nil {
t.Fatalf("fail to remove embedded-cluster binary on worker node: %v", err)
}

// wait for the nodes to report as ready.
t.Logf("%s: all nodes joined, waiting for them to be ready", time.Now().Format(time.RFC3339))
stdout, _, err = tc.RunCommandOnNode(0, []string{"wait-for-ready-nodes.sh", "2"})
if err != nil {
t.Log(stdout)
t.Fatalf("fail to wait for ready nodes: %v", err)
}

t.Logf("%s: checking installation state after app deployment", time.Now().Format(time.RFC3339))
line = []string{"check-airgap-installation-state.sh", fmt.Sprintf("appver-%s-previous-stable", os.Getenv("SHORT_SHA")), k8sVersionPreviousStable()}
if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
t.Fatalf("fail to check installation state: %v", err)
}

t.Logf("%s: running airgap update", time.Now().Format(time.RFC3339))
line = []string{"airgap-update.sh"}
if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
t.Fatalf("fail to run airgap update: %v", err)
}
// remove the airgap bundle and binary after upgrade
line = []string{"rm", "/assets/upgrade/release.airgap"}
if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
t.Fatalf("fail to remove airgap bundle on node %s: %v", tc.Nodes[0], err)
}
line = []string{"rm", "/usr/local/bin/embedded-cluster-upgrade"}
if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
t.Fatalf("fail to remove embedded-cluster-upgrade binary on node %s: %v", tc.Nodes[0], err)
}

appUpgradeVersion := fmt.Sprintf("appver-%s-upgrade", os.Getenv("SHORT_SHA"))
testArgs := []string{appUpgradeVersion}

t.Logf("%s: upgrading cluster", time.Now().Format(time.RFC3339))
if _, _, err := tc.RunPlaywrightTest("deploy-upgrade", testArgs...); err != nil {
t.Fatalf("fail to run playwright test deploy-app: %v", err)
}

t.Logf("%s: checking installation state after upgrade", time.Now().Format(time.RFC3339))
line = []string{"check-postupgrade-state.sh", k8sVersion()}
if _, _, err := tc.RunCommandOnNode(0, line); err != nil {
t.Fatalf("fail to check postupgrade state: %v", err)
}

t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
}

// This test creates 4 nodes, installs on the first one and then generate 2 join tokens
// for controllers and one join token for worker nodes. Joins the nodes as HA and then waits
// for them to report ready. Runs additional high availability validations afterwards.
Expand Down
Loading

0 comments on commit 789bdf5

Please sign in to comment.