From a1c8b0fb1a4d6e1fb720b9408b4d9638ddea5a92 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Fri, 18 Oct 2024 22:49:42 -0500 Subject: [PATCH] fix: upgrades from versions prior to 1.15.0 have incorrect data directories (#1341) --- .github/actions/e2e/action.yml | 2 +- .github/workflows/ci.yaml | 5 +- e2e/cluster/docker/cluster.go | 8 +- e2e/cluster/lxd/cluster.go | 8 +- e2e/install_test.go | 118 +++++- .../get-ec18-join-worker-command/test.spec.ts | 13 + e2e/scripts/common.sh | 1 + e2e/scripts/reset-installation.sh | 2 +- operator/pkg/charts/charts.go | 72 +++- operator/pkg/cli/upgrade.go | 5 +- operator/pkg/upgrade/installation.go | 50 +-- operator/pkg/upgrade/job.go | 30 +- operator/pkg/upgrade/job_test.go | 129 ------ operator/pkg/upgrade/upgrade.go | 33 +- .../embeddedclusteroperator.go | 10 - pkg/defaults/provider.go | 6 +- pkg/kubeutils/kubeutils.go | 128 +++++- pkg/kubeutils/kubeutils_test.go | 394 ++++++++++++++++++ 18 files changed, 744 insertions(+), 270 deletions(-) create mode 100644 e2e/playwright/tests/get-ec18-join-worker-command/test.spec.ts delete mode 100644 operator/pkg/upgrade/job_test.go create mode 100644 pkg/kubeutils/kubeutils_test.go diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml index 766596665..b7c4461bb 100644 --- a/.github/actions/e2e/action.yml +++ b/.github/actions/e2e/action.yml @@ -108,7 +108,7 @@ runs: export EXPECT_K0S_VERSION_PREVIOUS=${{ inputs.k0s-version-previous }} make e2e-test TEST_NAME=${{ inputs.test-name }} - name: Troubleshoot - if: ${{ failure() }} + if: ${{ !cancelled() }} uses: ./.github/actions/e2e-troubleshoot with: test-name: ${{ inputs.test-name }} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ff8b63707..aa383dcaf 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -549,7 +549,7 @@ jobs: run: | make e2e-test TEST_NAME=${{ matrix.test }} - name: Troubleshoot - if: ${{ failure() }} + if: ${{ !cancelled() }} uses: ./.github/actions/e2e-troubleshoot with: test-name: '${{ matrix.test }}' @@ -571,7 +571,6 @@ jobs: - TestCommandsRequireSudo - TestResetAndReinstallAirgap - TestSingleNodeAirgapUpgrade - - TestSingleNodeAirgapUpgradeFromEC18 - TestSingleNodeAirgapUpgradeCustomCIDR - TestSingleNodeDisasterRecoveryWithProxy - TestProxiedEnvironment @@ -583,6 +582,8 @@ jobs: runner: embedded-cluster - test: TestMultiNodeAirgapUpgradeSameK0s runner: embedded-cluster + - test: TestAirgapUpgradeFromEC18 + runner: embedded-cluster - test: TestSingleNodeAirgapDisasterRecovery runner: embedded-cluster - test: TestMultiNodeAirgapHAInstallation diff --git a/e2e/cluster/docker/cluster.go b/e2e/cluster/docker/cluster.go index c83c0be61..a0b486814 100644 --- a/e2e/cluster/docker/cluster.go +++ b/e2e/cluster/docker/cluster.go @@ -82,10 +82,10 @@ func (c *Cluster) WaitForReady() { } func (c *Cluster) Cleanup(envs ...map[string]string) { - if c.t.Failed() { - c.generateSupportBundle(envs...) - c.copyPlaywrightReport() - } + // if c.t.Failed() { + c.generateSupportBundle(envs...) + c.copyPlaywrightReport() + //} for _, node := range c.Nodes { node.Destroy() } diff --git a/e2e/cluster/lxd/cluster.go b/e2e/cluster/lxd/cluster.go index ce627cba8..8e27aad00 100644 --- a/e2e/cluster/lxd/cluster.go +++ b/e2e/cluster/lxd/cluster.go @@ -989,10 +989,10 @@ func (c *Cluster) InstallTestDependenciesDebian(t *testing.T, node int, withProx } func (c *Cluster) Cleanup(envs ...map[string]string) { - if c.T.Failed() { - c.generateSupportBundle(envs...) - c.copyPlaywrightReport() - } + // if c.T.Failed() { + c.generateSupportBundle(envs...) + c.copyPlaywrightReport() + // } } func (c *Cluster) SetupPlaywrightAndRunTest(testName string, args ...string) (string, string, error) { diff --git a/e2e/install_test.go b/e2e/install_test.go index de0792f7c..43916d302 100644 --- a/e2e/install_test.go +++ b/e2e/install_test.go @@ -535,7 +535,7 @@ func TestUpgradeEC18FromReplicatedApp(t *testing.T) { tc := docker.NewCluster(&docker.ClusterInput{ T: t, - Nodes: 1, + Nodes: 2, Distro: "debian-bookworm", K0sDir: "/var/lib/k0s", }) @@ -547,6 +547,12 @@ func TestUpgradeEC18FromReplicatedApp(t *testing.T) { t.Fatalf("fail to download embedded-cluster on node 0: %v: %s: %s", err, stdout, stderr) } + t.Logf("%s: downloading embedded-cluster 1.8.0+k8s-1.28 on worker node", time.Now().Format(time.RFC3339)) + line = []string{"vandoor-prepare.sh", "1.8.0+k8s-1.28", os.Getenv("LICENSE_ID"), "false"} + if stdout, stderr, err := tc.RunCommandOnNode(1, line); err != nil { + t.Fatalf("fail to download embedded-cluster on node 0: %v: %s: %s", err, stdout, stderr) + } + t.Logf("%s: installing embedded-cluster 1.8.0+k8s-1.28 on node 0", time.Now().Format(time.RFC3339)) line = []string{"single-node-install.sh", "ui"} if stdout, stderr, err := tc.RunCommandOnNode(0, line, withEnv); err != nil { @@ -560,6 +566,29 @@ func TestUpgradeEC18FromReplicatedApp(t *testing.T) { t.Fatalf("fail to run playwright test deploy-ec18-app-version: %v: %s: %s", err, stdout, stderr) } + t.Logf("%s: generating a new worker token command", time.Now().Format(time.RFC3339)) + stdout, stderr, err := tc.RunPlaywrightTest("get-ec18-join-worker-command") + if err != nil { + t.Fatalf("fail to generate worker join token:\nstdout: %s\nstderr: %s", stdout, stderr) + } + command, err := findJoinCommandInOutput(stdout) + if err != nil { + t.Fatalf("fail to find the join command in the output: %v: %s: %s", err, stdout, stderr) + } + t.Log("worker join token command:", command) + + t.Logf("%s: joining worker node to the cluster as a worker", time.Now().Format(time.RFC3339)) + if stdout, stderr, err := tc.RunCommandOnNode(1, strings.Split(command, " ")); err != nil { + t.Fatalf("fail to join worker node to the cluster as a worker: %v: %s: %s", err, stdout, stderr) + } + + // wait for the nodes to report as ready. + t.Logf("%s: all nodes joined, waiting for them to be ready", time.Now().Format(time.RFC3339)) + stdout, stderr, err = tc.RunCommandOnNode(0, []string{"wait-for-ready-nodes.sh", "2"}, withEnv) + if err != nil { + t.Fatalf("fail to wait for ready nodes: %v: %s: %s", err, stdout, stderr) + } + t.Logf("%s: checking installation state", time.Now().Format(time.RFC3339)) line = []string{"check-installation-state.sh", "1.8.0+k8s-1.28", "v1.28.11"} if stdout, stderr, err := tc.RunCommandOnNode(0, line, withEnv); err != nil { @@ -580,6 +609,18 @@ func TestUpgradeEC18FromReplicatedApp(t *testing.T) { t.Fatalf("fail to check postupgrade state: %v: %s: %s", err, stdout, stderr) } + t.Logf("%s: resetting worker node", time.Now().Format(time.RFC3339)) + line = []string{"reset-installation.sh"} + if stdout, stderr, err := tc.RunCommandOnNode(1, line, withEnv); err != nil { + t.Fatalf("fail to reset worker node: %v: %s: %s", err, stdout, stderr) + } + + t.Logf("%s: resetting node 0", time.Now().Format(time.RFC3339)) + line = []string{"reset-installation.sh"} + if stdout, stderr, err := tc.RunCommandOnNode(0, line, withEnv); err != nil { + t.Fatalf("fail to reset node 0: %v: %s: %s", err, stdout, stderr) + } + t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } @@ -967,7 +1008,7 @@ func TestSingleNodeAirgapUpgradeCustomCIDR(t *testing.T) { t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } -func TestSingleNodeAirgapUpgradeFromEC18(t *testing.T) { +func TestAirgapUpgradeFromEC18(t *testing.T) { t.Parallel() RequireEnvVars(t, []string{"SHORT_SHA", "AIRGAP_LICENSE_ID"}) @@ -987,7 +1028,7 @@ func TestSingleNodeAirgapUpgradeFromEC18(t *testing.T) { tc := lxd.NewCluster(&lxd.ClusterInput{ T: t, - Nodes: 1, + Nodes: 2, Image: "debian/12", WithProxy: true, AirgapInstallBundlePath: airgapInstallBundlePath, @@ -1003,11 +1044,17 @@ func TestSingleNodeAirgapUpgradeFromEC18(t *testing.T) { t.Logf("failed to remove airgap upgrade bundle: %v", err) } + // upgrade airgap bundle is only needed on the first node + line := []string{"rm", "/assets/ec-release-upgrade.tgz"} + if _, _, err := tc.RunCommandOnNode(1, line); err != nil { + t.Fatalf("fail to remove upgrade airgap bundle on node %s: %v", tc.Nodes[1], err) + } + // install "curl" dependency on node 0 for app version checks. tc.InstallTestDependenciesDebian(t, 0, true) t.Logf("%s: preparing embedded cluster airgap files", time.Now().Format(time.RFC3339)) - line := []string{"airgap-prepare.sh"} + line = []string{"airgap-prepare.sh"} if _, _, err := tc.RunCommandOnNode(0, line); err != nil { t.Fatalf("fail to prepare airgap files on node %s: %v", tc.Nodes[0], err) } @@ -1030,6 +1077,50 @@ func TestSingleNodeAirgapUpgradeFromEC18(t *testing.T) { t.Fatalf("fail to run playwright test deploy-ec18-app-version: %v", err) } + // generate worker node join command. + t.Logf("%s: generating a new worker token command", time.Now().Format(time.RFC3339)) + stdout, stderr, err := tc.RunPlaywrightTest("get-ec18-join-worker-command") + if err != nil { + t.Fatalf("fail to generate worker join token:\nstdout: %s\nstderr: %s", stdout, stderr) + } + workerCommand, err := findJoinCommandInOutput(stdout) + if err != nil { + t.Fatalf("fail to find the join command in the output: %v", err) + } + t.Log("worker join token command:", workerCommand) + + // join the worker node + t.Logf("%s: preparing embedded cluster airgap files on worker node", time.Now().Format(time.RFC3339)) + line = []string{"airgap-prepare.sh"} + if _, _, err := tc.RunCommandOnNode(1, line); err != nil { + t.Fatalf("fail to prepare airgap files on worker node: %v", err) + } + t.Logf("%s: joining worker node to the cluster", time.Now().Format(time.RFC3339)) + if _, _, err := tc.RunCommandOnNode(1, strings.Split(workerCommand, " ")); err != nil { + t.Fatalf("fail to join worker node to the cluster: %v", err) + } + // remove artifacts after joining to save space + line = []string{"rm", "/assets/release.airgap"} + if _, _, err := tc.RunCommandOnNode(1, line); err != nil { + t.Fatalf("fail to remove airgap bundle on worker node: %v", err) + } + line = []string{"rm", "/usr/local/bin/embedded-cluster"} + if _, _, err := tc.RunCommandOnNode(1, line); err != nil { + t.Fatalf("fail to remove embedded-cluster binary on worker node: %v", err) + } + line = []string{"rm", "/var/lib/embedded-cluster/bin/embedded-cluster"} + if _, _, err := tc.RunCommandOnNode(1, line); err != nil { + t.Fatalf("fail to remove embedded-cluster binary on node %s: %v", tc.Nodes[0], err) + } + + // wait for the nodes to report as ready. + t.Logf("%s: all nodes joined, waiting for them to be ready", time.Now().Format(time.RFC3339)) + stdout, _, err = tc.RunCommandOnNode(0, []string{"wait-for-ready-nodes.sh", "2"}, withEnv) + if err != nil { + t.Log(stdout) + t.Fatalf("fail to wait for ready nodes: %v", err) + } + t.Logf("%s: checking installation state after app deployment", time.Now().Format(time.RFC3339)) line = []string{ "check-airgap-installation-state.sh", @@ -1066,6 +1157,25 @@ func TestSingleNodeAirgapUpgradeFromEC18(t *testing.T) { t.Fatalf("fail to check postupgrade state: %v", err) } + // TODO: reset fails with the following error: + // error: could not reset k0s: exit status 1, time="2024-10-17 22:44:52" level=warning msg="To ensure a full reset, a node reboot is recommended." + // Error: errors received during clean-up: [failed to delete /run/k0s. err: unlinkat /run/k0s/containerd/io.containerd.grpc.v1.cri/sandboxes/.../shm: device or resource busy] + + // t.Logf("%s: resetting worker node", time.Now().Format(time.RFC3339)) + // line = []string{"reset-installation.sh"} + // if stdout, stderr, err := tc.RunCommandOnNode(1, line, withEnv); err != nil { + // t.Fatalf("fail to reset worker node: %v: %s: %s", err, stdout, stderr) + // } + + // // use upgrade binary for reset + // withUpgradeBin := map[string]string{"EMBEDDED_CLUSTER_BIN": "embedded-cluster-upgrade"} + + // t.Logf("%s: resetting node 0", time.Now().Format(time.RFC3339)) + // line = []string{"reset-installation.sh"} + // if stdout, stderr, err := tc.RunCommandOnNode(0, line, withEnv, withUpgradeBin); err != nil { + // t.Fatalf("fail to reset node 0: %v: %s: %s", err, stdout, stderr) + // } + t.Logf("%s: test complete", time.Now().Format(time.RFC3339)) } diff --git a/e2e/playwright/tests/get-ec18-join-worker-command/test.spec.ts b/e2e/playwright/tests/get-ec18-join-worker-command/test.spec.ts new file mode 100644 index 000000000..14a56caaa --- /dev/null +++ b/e2e/playwright/tests/get-ec18-join-worker-command/test.spec.ts @@ -0,0 +1,13 @@ +import { test, expect } from '@playwright/test'; +import { login } from '../shared'; + +test('get join worker command', async ({ page }) => { + await login(page); + await page.locator('.NavItem').getByText('Cluster Management', { exact: true }).click(); + await page.getByRole('button', { name: 'Add node', exact: true }).click(); + await expect(page.locator('.Modal-body')).toBeVisible(); + await expect(page.getByRole('heading')).toContainText('Add a Node'); + await page.locator('.BoxedCheckbox').getByText('abc', { exact: true }).click(); + const joinCommand = await page.locator('.react-prism.language-bash').first().textContent(); + console.log(`{"command":"${joinCommand}"}`); +}); diff --git a/e2e/scripts/common.sh b/e2e/scripts/common.sh index 712bbf7dd..a57f8f03a 100644 --- a/e2e/scripts/common.sh +++ b/e2e/scripts/common.sh @@ -1,5 +1,6 @@ #!/bin/bash +export EMBEDDED_CLUSTER_BIN="${EMBEDDED_CLUSTER_BIN:-embedded-cluster}" export EMBEDDED_CLUSTER_BASE_DIR="${EMBEDDED_CLUSTER_BASE_DIR:-/var/lib/embedded-cluster}" export EMBEDDED_CLUSTER_METRICS_BASEURL="https://staging.replicated.app" export PATH="$PATH:${EMBEDDED_CLUSTER_BASE_DIR}/bin" diff --git a/e2e/scripts/reset-installation.sh b/e2e/scripts/reset-installation.sh index 16e1ddcfb..1ef88119d 100755 --- a/e2e/scripts/reset-installation.sh +++ b/e2e/scripts/reset-installation.sh @@ -7,7 +7,7 @@ DIR=/usr/local/bin main() { local additional_flags=("$@") - if ! embedded-cluster reset --no-prompt "${additional_flags[@]}" | tee /tmp/log ; then + if ! "${EMBEDDED_CLUSTER_BIN}" reset --no-prompt "${additional_flags[@]}" | tee /tmp/log ; then echo "Failed to uninstall embedded-cluster" exit 1 fi diff --git a/operator/pkg/charts/charts.go b/operator/pkg/charts/charts.go index 9d3021946..6c544b4d9 100644 --- a/operator/pkg/charts/charts.go +++ b/operator/pkg/charts/charts.go @@ -195,7 +195,7 @@ func updateInfraChartsFromInstall(in *v1beta1.Installation, clusterConfig *k0sv1 if chart.Name == "embedded-cluster-operator" { newVals, err := helm.UnmarshalValues(chart.Values) if err != nil { - return nil, fmt.Errorf("unmarshal admin-console.values: %w", err) + return nil, fmt.Errorf("unmarshal embedded-cluster-operator.values: %w", err) } // embedded-cluster-operator has "embeddedBinaryName" and "embeddedClusterID" as dynamic values @@ -219,17 +219,29 @@ func updateInfraChartsFromInstall(in *v1beta1.Installation, clusterConfig *k0sv1 charts[i].Values, err = helm.MarshalValues(newVals) if err != nil { - return nil, fmt.Errorf("marshal admin-console.values: %w", err) + return nil, fmt.Errorf("marshal embedded-cluster-operator.values: %w", err) } } - if chart.Name == "docker-registry" { - if !in.Spec.AirGap { - continue + if chart.Name == "openebs" { + newVals, err := helm.UnmarshalValues(chart.Values) + if err != nil { + return nil, fmt.Errorf("unmarshal openebs.values: %w", err) } + newVals, err = helm.SetValue(newVals, `["localpv-provisioner"].localpv.basePath`, provider.EmbeddedClusterOpenEBSLocalSubDir()) + if err != nil { + return nil, fmt.Errorf("set helm values openebs.localpv-provisioner.localpv.basePath: %w", err) + } + + charts[i].Values, err = helm.MarshalValues(newVals) + if err != nil { + return nil, fmt.Errorf("marshal openebs.values: %w", err) + } + } + if chart.Name == "docker-registry" { newVals, err := helm.UnmarshalValues(chart.Values) if err != nil { - return nil, fmt.Errorf("unmarshal admin-console.values: %w", err) + return nil, fmt.Errorf("unmarshal docker-registry.values: %w", err) } // handle the registry IP, which will always be present in airgap @@ -259,16 +271,38 @@ func updateInfraChartsFromInstall(in *v1beta1.Installation, clusterConfig *k0sv1 charts[i].Values, err = helm.MarshalValues(newVals) if err != nil { - return nil, fmt.Errorf("marshal admin-console.values: %w", err) + return nil, fmt.Errorf("marshal docker-registry.values: %w", err) + } + } + if chart.Name == "seaweedfs" { + newVals, err := helm.UnmarshalValues(chart.Values) + if err != nil { + return nil, fmt.Errorf("unmarshal seaweedfs.values: %w", err) + } + + dataPath := filepath.Join(provider.EmbeddedClusterSeaweedfsSubDir(), "ssd") + newVals, err = helm.SetValue(newVals, "global.data.hostPathPrefix", dataPath) + if err != nil { + return nil, fmt.Errorf("set helm values seaweedfs.global.data.hostPathPrefix: %w", err) + } + logsPath := filepath.Join(provider.EmbeddedClusterSeaweedfsSubDir(), "storage") + newVals, err = helm.SetValue(newVals, "global.logs.hostPathPrefix", logsPath) + if err != nil { + return nil, fmt.Errorf("set helm values seaweedfs.global.logs.hostPathPrefix: %w", err) + } + + charts[i].Values, err = helm.MarshalValues(newVals) + if err != nil { + return nil, fmt.Errorf("marshal seaweedfs.values: %w", err) } } if chart.Name == "velero" { - if in.Spec.Proxy != nil { - newVals, err := helm.UnmarshalValues(chart.Values) - if err != nil { - return nil, fmt.Errorf("unmarshal admin-console.values: %w", err) - } + newVals, err := helm.UnmarshalValues(chart.Values) + if err != nil { + return nil, fmt.Errorf("unmarshal velero.values: %w", err) + } + if in.Spec.Proxy != nil { extraEnvVars := map[string]interface{}{ "extraEnvVars": map[string]string{ "HTTP_PROXY": in.Spec.Proxy.HTTPProxy, @@ -281,11 +315,17 @@ func updateInfraChartsFromInstall(in *v1beta1.Installation, clusterConfig *k0sv1 if err != nil { return nil, fmt.Errorf("set helm values velero.configuration: %w", err) } + } - charts[i].Values, err = helm.MarshalValues(newVals) - if err != nil { - return nil, fmt.Errorf("marshal admin-console.values: %w", err) - } + podVolumePath := filepath.Join(provider.EmbeddedClusterK0sSubDir(), "kubelet/pods") + newVals, err = helm.SetValue(newVals, "nodeAgent.podVolumePath", podVolumePath) + if err != nil { + return nil, fmt.Errorf("set helm values velero.nodeAgent.podVolumePath: %w", err) + } + + charts[i].Values, err = helm.MarshalValues(newVals) + if err != nil { + return nil, fmt.Errorf("marshal velero.values: %w", err) } } } diff --git a/operator/pkg/cli/upgrade.go b/operator/pkg/cli/upgrade.go index c0bf69b79..c79257d19 100644 --- a/operator/pkg/cli/upgrade.go +++ b/operator/pkg/cli/upgrade.go @@ -3,13 +3,14 @@ package cli import ( "context" "fmt" - "github.com/replicatedhq/embedded-cluster/operator/pkg/metrics" "io" "os" clusterv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" "github.com/replicatedhq/embedded-cluster/operator/pkg/k8sutil" + "github.com/replicatedhq/embedded-cluster/operator/pkg/metrics" "github.com/replicatedhq/embedded-cluster/operator/pkg/upgrade" + "github.com/replicatedhq/embedded-cluster/pkg/kubeutils" "github.com/spf13/cobra" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -49,7 +50,7 @@ func UpgradeCmd() *cobra.Command { if err != nil { return fmt.Errorf("apply installation: %w", err) } - previousInstallation, err := upgrade.GetPreviousInstallation(cmd.Context(), cli, in) + previousInstallation, err := kubeutils.GetPreviousInstallation(cmd.Context(), cli, in) if err != nil { return fmt.Errorf("get previous installation: %w", err) } diff --git a/operator/pkg/upgrade/installation.go b/operator/pkg/upgrade/installation.go index 24fa8c228..149cf365d 100644 --- a/operator/pkg/upgrade/installation.go +++ b/operator/pkg/upgrade/installation.go @@ -4,13 +4,10 @@ import ( "context" "fmt" - "k8s.io/apimachinery/pkg/types" - controllerruntime "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - clusterv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" - "github.com/replicatedhq/embedded-cluster/pkg/addons/embeddedclusteroperator" "github.com/replicatedhq/embedded-cluster/pkg/kubeutils" + controllerruntime "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" ) func CreateInstallation(ctx context.Context, cli client.Client, original *clusterv1beta1.Installation) error { @@ -19,20 +16,13 @@ func CreateInstallation(ctx context.Context, cli client.Client, original *cluste // check if the installation already exists - this function can be called multiple times // if the installation is already created, we can just return - nsn := types.NamespacedName{Name: in.Name} - var existingInstallation clusterv1beta1.Installation - if err := cli.Get(ctx, nsn, &existingInstallation); err == nil { + if in, err := kubeutils.GetInstallation(ctx, cli, in.Name); err == nil { log.Info(fmt.Sprintf("Installation %s already exists", in.Name)) return nil } log.Info(fmt.Sprintf("Creating installation %s", in.Name)) - in, err := maybeOverrideInstallationDataDirs(ctx, cli, in) - if err != nil { - return fmt.Errorf("override installation data dirs: %w", err) - } - - err = cli.Create(ctx, in) + err := cli.Create(ctx, in) if err != nil { return fmt.Errorf("create installation: %w", err) } @@ -65,8 +55,7 @@ func setInstallationState(ctx context.Context, cli client.Client, name string, s // reApplyInstallation updates the installation spec to match what's in the configmap used by the upgrade job. // This is required because the installation CRD may have been updated as part of this upgrade, and additional fields may be present now. func reApplyInstallation(ctx context.Context, cli client.Client, in *clusterv1beta1.Installation) error { - existingInstallation := &clusterv1beta1.Installation{} - err := cli.Get(ctx, client.ObjectKey{Name: in.Name}, existingInstallation) + existingInstallation, err := kubeutils.GetInstallation(ctx, cli, in.Name) if err != nil { return fmt.Errorf("get installation: %w", err) } @@ -79,32 +68,3 @@ func reApplyInstallation(ctx context.Context, cli client.Client, in *clusterv1be return nil } - -// maybeOverrideInstallationDataDirs checks if the installation has an annotation indicating that -// it was created or updated by a version that stored the location of the data directories in the -// installation object. If it is not set, it will set the annotation and update the installation -// object with the old location of the data directories. -func maybeOverrideInstallationDataDirs(ctx context.Context, cli client.Client, in *clusterv1beta1.Installation) (*clusterv1beta1.Installation, error) { - previous, err := kubeutils.GetLatestInstallation(ctx, cli) - if err != nil { - return in, fmt.Errorf("get latest installation: %w", err) - } - - if ok := previous.Annotations[embeddedclusteroperator.AnnotationHasDataDirectories]; ok == "true" { - return in, nil - } - if in.ObjectMeta.Annotations == nil { - in.ObjectMeta.Annotations = map[string]string{} - } - in.ObjectMeta.Annotations[embeddedclusteroperator.AnnotationHasDataDirectories] = "true" - - if in.Spec.RuntimeConfig == nil { - in.Spec.RuntimeConfig = &clusterv1beta1.RuntimeConfigSpec{} - } - - // In prior versions, the data directories are not a subdirectory of /var/lib/embedded-cluster. - in.Spec.RuntimeConfig.K0sDataDirOverride = "/var/lib/k0s" - in.Spec.RuntimeConfig.OpenEBSDataDirOverride = "/var/openebs" - - return in, nil -} diff --git a/operator/pkg/upgrade/job.go b/operator/pkg/upgrade/job.go index ecee3581d..dec4df74e 100644 --- a/operator/pkg/upgrade/job.go +++ b/operator/pkg/upgrade/job.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "sort" "strings" "time" @@ -23,7 +22,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime" + controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -388,30 +387,3 @@ func getAutopilotAirgapArtifactsPlan(ctx context.Context, cli client.Client, in return plan, nil } - -// GetPreviousInstallation returns the latest installation object in the cluster OTHER than the one passed as an argument. -func GetPreviousInstallation(ctx context.Context, cli client.Client, in *clusterv1beta1.Installation) (*clusterv1beta1.Installation, error) { - installations := &clusterv1beta1.InstallationList{} - if err := cli.List(ctx, installations); err != nil { - return nil, fmt.Errorf("failed to list installations: %w", err) - } - - if len(installations.Items) == 0 { - return nil, fmt.Errorf("no installations found") - } - - // sort the installations by name in descending order - sort.Slice(installations.Items, func(i, j int) bool { - return installations.Items[i].Name > installations.Items[j].Name - }) - - // find the first installation with a different name than the one we're upgrading to - for _, installation := range installations.Items { - if installation.Name != in.Name { - return &installation, nil - } - } - - // if we get here, we didn't find a previous installation - return nil, fmt.Errorf("previous installation not found") -} diff --git a/operator/pkg/upgrade/job_test.go b/operator/pkg/upgrade/job_test.go deleted file mode 100644 index d0fca8eec..000000000 --- a/operator/pkg/upgrade/job_test.go +++ /dev/null @@ -1,129 +0,0 @@ -package upgrade - -import ( - "context" - clusterv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" - "github.com/stretchr/testify/require" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes/scheme" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "testing" -) - -func TestGetPreviousInstallation(t *testing.T) { - scheme := scheme.Scheme - clusterv1beta1.AddToScheme(scheme) - - tests := []struct { - name string - in *clusterv1beta1.Installation - want *clusterv1beta1.Installation - wantErr bool - objects []client.Object - }{ - { - name: "no installations at all", - in: &clusterv1beta1.Installation{ - ObjectMeta: metav1.ObjectMeta{ - Name: "20241002205018", - }, - }, - want: nil, - wantErr: true, - objects: []client.Object{}, - }, - { - name: "no previous installation", - in: &clusterv1beta1.Installation{ - ObjectMeta: metav1.ObjectMeta{ - Name: "20241002205018", - }, - }, - want: nil, - wantErr: true, - objects: []client.Object{ - &clusterv1beta1.Installation{ - TypeMeta: metav1.TypeMeta{ - Kind: "Installation", - APIVersion: "v1beta1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "20241002205018", - }, - }, - }, - }, - { - name: "multiple previous installations", - in: &clusterv1beta1.Installation{ - ObjectMeta: metav1.ObjectMeta{ - Name: "20241002205018", - }, - }, - want: &clusterv1beta1.Installation{ - TypeMeta: metav1.TypeMeta{ - Kind: "Installation", - APIVersion: "v1beta1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "20230000000000", - ResourceVersion: "999", - }, - }, - wantErr: false, - objects: []client.Object{ - &clusterv1beta1.Installation{ - TypeMeta: metav1.TypeMeta{ - Kind: "Installation", - APIVersion: "v1beta1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "20220000000000", - }, - }, - &clusterv1beta1.Installation{ - TypeMeta: metav1.TypeMeta{ - Kind: "Installation", - APIVersion: "v1beta1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "20241002205018", - }, - }, - &clusterv1beta1.Installation{ - TypeMeta: metav1.TypeMeta{ - Kind: "Installation", - APIVersion: "v1beta1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "20230000000000", - }, - }, - &clusterv1beta1.Installation{ - TypeMeta: metav1.TypeMeta{ - Kind: "Installation", - APIVersion: "v1beta1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "20210000000000", - }, - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - req := require.New(t) - cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(tt.objects...).Build() - - got, err := GetPreviousInstallation(context.Background(), cli, tt.in) - if tt.wantErr { - req.Error(err) - return - } - req.NoError(err) - req.Equal(tt.want, got) - }) - } -} diff --git a/operator/pkg/upgrade/upgrade.go b/operator/pkg/upgrade/upgrade.go index 9274b0e2f..6e3a91ac4 100644 --- a/operator/pkg/upgrade/upgrade.go +++ b/operator/pkg/upgrade/upgrade.go @@ -16,6 +16,7 @@ import ( "github.com/replicatedhq/embedded-cluster/operator/pkg/registry" "github.com/replicatedhq/embedded-cluster/operator/pkg/release" "github.com/replicatedhq/embedded-cluster/pkg/config" + "github.com/replicatedhq/embedded-cluster/pkg/kubeutils" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" @@ -30,14 +31,25 @@ const ( // Upgrade upgrades the embedded cluster to the version specified in the installation. // First the k0s cluster is upgraded, then addon charts are upgraded, and finally the installation is unlocked. func Upgrade(ctx context.Context, cli client.Client, in *clusterv1beta1.Installation) error { - err := clusterConfigUpdate(ctx, cli, in) + err := k0sUpgrade(ctx, cli, in) if err != nil { - return fmt.Errorf("cluster config update: %w", err) + return fmt.Errorf("k0s upgrade: %w", err) } - err = k0sUpgrade(ctx, cli, in) + // Augment the installation with data dirs that may not be present in the previous version. + // This is important to do ahead of updating the cluster config. + // We still cannot update the installation object as the CRDs are not updated yet. + in, err = maybeOverrideInstallationDataDirs(ctx, cli, in) if err != nil { - return fmt.Errorf("k0s upgrade: %w", err) + return fmt.Errorf("override installation data dirs: %w", err) + } + + // We must update the cluster config after we upgrade k0s as it is possible that the schema + // between versions has changed. One drawback of this is that the sandbox (pause) image does + // not get updated, and possibly others but I cannot confirm this. + err = clusterConfigUpdate(ctx, cli, in) + if err != nil { + return fmt.Errorf("cluster config update: %w", err) } err = registryMigrationStatus(ctx, cli, in) @@ -56,6 +68,7 @@ func Upgrade(ctx context.Context, cli client.Client, in *clusterv1beta1.Installa return fmt.Errorf("wait for operator chart: %w", err) } + // Finally, re-apply the installation as the CRDs are up-to-date. err = reApplyInstallation(ctx, cli, in) if err != nil { return fmt.Errorf("unlock installation: %w", err) @@ -64,6 +77,18 @@ func Upgrade(ctx context.Context, cli client.Client, in *clusterv1beta1.Installa return nil } +func maybeOverrideInstallationDataDirs(ctx context.Context, cli client.Client, in *clusterv1beta1.Installation) (*clusterv1beta1.Installation, error) { + previous, err := kubeutils.GetPreviousInstallation(ctx, cli, in) + if err != nil { + return in, fmt.Errorf("get latest installation: %w", err) + } + next, _, err := kubeutils.MaybeOverrideInstallationDataDirs(*in, previous) + if err != nil { + return in, fmt.Errorf("override installation data dirs: %w", err) + } + return &next, nil +} + func k0sUpgrade(ctx context.Context, cli client.Client, in *clusterv1beta1.Installation) error { meta, err := release.MetadataFor(ctx, in, cli) if err != nil { diff --git a/pkg/addons/embeddedclusteroperator/embeddedclusteroperator.go b/pkg/addons/embeddedclusteroperator/embeddedclusteroperator.go index 034a52369..f6a506076 100644 --- a/pkg/addons/embeddedclusteroperator/embeddedclusteroperator.go +++ b/pkg/addons/embeddedclusteroperator/embeddedclusteroperator.go @@ -31,13 +31,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -const ( - // AnnotationHasDataDirectories is an annotation on the installation object that indicates that - // it was created by an operator that stored information about the location of the data - // directories. If this is not set, the operator will update the installation object. - AnnotationHasDataDirectories = "embedded-cluster.replicated.com/has-data-directories" -) - const releaseName = "embedded-cluster-operator" var ( @@ -267,9 +260,6 @@ func (e *EmbeddedClusterOperator) Outro(ctx context.Context, provider *defaults. Labels: map[string]string{ "replicated.com/disaster-recovery": "ec-install", }, - Annotations: map[string]string{ - AnnotationHasDataDirectories: "true", - }, }, Spec: ecv1beta1.InstallationSpec{ ClusterID: metrics.ClusterID().String(), diff --git a/pkg/defaults/provider.go b/pkg/defaults/provider.go index cd37978d0..504a0204f 100644 --- a/pkg/defaults/provider.go +++ b/pkg/defaults/provider.go @@ -50,7 +50,8 @@ func NewProviderFromCluster(ctx context.Context, cli client.Client) (*Provider, // of EC that used a different directory for k0s and openebs. func NewProviderFromFilesystem() (*Provider, error) { provider := NewProvider(ecv1beta1.DefaultDataDir) - _, err := os.Stat(provider.PathToKubeConfig()) + // ca.crt is available on both control plane and worker nodes + _, err := os.Stat(filepath.Join(provider.EmbeddedClusterK0sSubDir(), "pki/ca.crt")) if err == nil { return provider, nil } @@ -60,7 +61,8 @@ func NewProviderFromFilesystem() (*Provider, error) { K0sDataDirOverride: "/var/lib/k0s", OpenEBSDataDirOverride: "/var/openebs", }) - _, err = os.Stat(provider.PathToKubeConfig()) + // ca.crt is available on both control plane and worker nodes + _, err = os.Stat(filepath.Join(provider.EmbeddedClusterK0sSubDir(), "pki/ca.crt")) if err == nil { return provider, nil } diff --git a/pkg/kubeutils/kubeutils.go b/pkg/kubeutils/kubeutils.go index a58b85cb7..0811e37ec 100644 --- a/pkg/kubeutils/kubeutils.go +++ b/pkg/kubeutils/kubeutils.go @@ -6,18 +6,18 @@ import ( "sort" "time" + "github.com/Masterminds/semver/v3" embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" + "github.com/replicatedhq/embedded-cluster/pkg/spinner" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/replicatedhq/embedded-cluster/pkg/spinner" ) type ErrNoInstallations struct{} @@ -26,6 +26,12 @@ func (e ErrNoInstallations) Error() string { return "no installations found" } +type ErrInstallationNotFound struct{} + +func (e ErrInstallationNotFound) Error() string { + return "installation not found" +} + // BackOffToDuration returns the maximum duration of the provided backoff. func BackOffToDuration(backoff wait.Backoff) time.Duration { var total time.Duration @@ -175,40 +181,128 @@ func WaitForInstallation(ctx context.Context, cli client.Client, writer *spinner func ListInstallations(ctx context.Context, cli client.Client) ([]embeddedclusterv1beta1.Installation, error) { var list embeddedclusterv1beta1.InstallationList - if err := cli.List(ctx, &list); err != nil { + err := cli.List(ctx, &list) + if err != nil { return nil, err } installs := list.Items sort.SliceStable(installs, func(i, j int) bool { return installs[j].Name < installs[i].Name }) + var previous *embeddedclusterv1beta1.Installation + for i := len(installs) - 1; i >= 0; i-- { + install, didUpdate, err := MaybeOverrideInstallationDataDirs(installs[i], previous) + if err != nil { + return nil, fmt.Errorf("override installation data dirs: %w", err) + } + if didUpdate { + err := cli.Update(ctx, &install) + if err != nil { + return nil, fmt.Errorf("update installation with legacy data dirs: %w", err) + } + log := ctrl.LoggerFrom(ctx) + log.Info("Updated installation with legacy data dirs", "installation", install.Name) + } + installs[i] = install + previous = &install + } return installs, nil } func GetInstallation(ctx context.Context, cli client.Client, name string) (*embeddedclusterv1beta1.Installation, error) { - nsn := types.NamespacedName{Name: name} - var install embeddedclusterv1beta1.Installation - if err := cli.Get(ctx, nsn, &install); err != nil { - return nil, fmt.Errorf("unable to get installation: %w", err) + installations, err := ListInstallations(ctx, cli) + if err != nil { + return nil, err } - return &install, nil + if len(installations) == 0 { + return nil, ErrNoInstallations{} + } + + for _, installation := range installations { + if installation.Name == name { + return &installation, nil + } + } + + // if we get here, we didn't find the installation + return nil, ErrInstallationNotFound{} } func GetLatestInstallation(ctx context.Context, cli client.Client) (*embeddedclusterv1beta1.Installation, error) { - installs, err := ListInstallations(ctx, cli) - if meta.IsNoMatchError(err) { - // this will happen if the CRD is not yet installed + installations, err := ListInstallations(ctx, cli) + if err != nil { + return nil, err + } + if len(installations) == 0 { return nil, ErrNoInstallations{} - } else if err != nil { - return nil, fmt.Errorf("unable to list installations: %v", err) } - if len(installs) == 0 { + // get the latest installation + return &installations[0], nil +} + +// GetPreviousInstallation returns the latest installation object in the cluster OTHER than the one passed as an argument. +func GetPreviousInstallation(ctx context.Context, cli client.Client, in *embeddedclusterv1beta1.Installation) (*embeddedclusterv1beta1.Installation, error) { + installations, err := ListInstallations(ctx, cli) + if err != nil { + return nil, err + } + if len(installations) == 0 { return nil, ErrNoInstallations{} } - // get the latest installation - return &installs[0], nil + // find the first installation with a different name than the one we're upgrading to + for _, installation := range installations { + if installation.Name != in.Name { + return &installation, nil + } + } + + // if we get here, we didn't find a previous installation + return nil, ErrInstallationNotFound{} +} + +var ( + Version115 = semver.MustParse("1.15.0") +) + +func lessThanK0s115(ver *semver.Version) bool { + return ver.LessThan(Version115) +} + +// MaybeOverrideInstallationDataDirs checks if the previous installation is less than 1.15.0 that +// didn't store the location of the data directories in the installation object. If it is not set, +// it will set the annotation and update the installation object with the old location of the data +// directories. +func MaybeOverrideInstallationDataDirs(in embeddedclusterv1beta1.Installation, previous *embeddedclusterv1beta1.Installation) (embeddedclusterv1beta1.Installation, bool, error) { + if previous != nil { + ver, err := semver.NewVersion(previous.Spec.Config.Version) + if err != nil { + return in, false, fmt.Errorf("parse version: %w", err) + } + + if lessThanK0s115(ver) { + didUpdate := false + + if in.Spec.RuntimeConfig == nil { + in.Spec.RuntimeConfig = &embeddedclusterv1beta1.RuntimeConfigSpec{} + } + + // In prior versions, the data directories are not a subdirectory of /var/lib/embedded-cluster. + if in.Spec.RuntimeConfig.K0sDataDirOverride != "/var/lib/k0s" { + in.Spec.RuntimeConfig.K0sDataDirOverride = "/var/lib/k0s" + didUpdate = true + } + if in.Spec.RuntimeConfig.OpenEBSDataDirOverride != "/var/openebs" { + in.Spec.RuntimeConfig.OpenEBSDataDirOverride = "/var/openebs" + didUpdate = true + } + + return in, didUpdate, nil + } + } + + return in, false, nil } func writeStatusMessage(writer *spinner.MessageWriter, install *embeddedclusterv1beta1.Installation) { diff --git a/pkg/kubeutils/kubeutils_test.go b/pkg/kubeutils/kubeutils_test.go new file mode 100644 index 000000000..f9692c527 --- /dev/null +++ b/pkg/kubeutils/kubeutils_test.go @@ -0,0 +1,394 @@ +package kubeutils + +import ( + "context" + "testing" + + "github.com/Masterminds/semver/v3" + embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestGetPreviousInstallation(t *testing.T) { + scheme := scheme.Scheme + embeddedclusterv1beta1.AddToScheme(scheme) + + tests := []struct { + name string + in *embeddedclusterv1beta1.Installation + want *embeddedclusterv1beta1.Installation + wantErr bool + objects []client.Object + }{ + { + name: "no installations at all", + in: &embeddedclusterv1beta1.Installation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "20241002205018", + }, + Spec: embeddedclusterv1beta1.InstallationSpec{ + Config: &embeddedclusterv1beta1.ConfigSpec{ + Version: "1.13.0+k8s-1.29-49-gf92daca6", + }, + }, + }, + want: nil, + wantErr: true, + objects: []client.Object{}, + }, + { + name: "no previous installation", + in: &embeddedclusterv1beta1.Installation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "20241002205018", + }, + Spec: embeddedclusterv1beta1.InstallationSpec{ + Config: &embeddedclusterv1beta1.ConfigSpec{ + Version: "1.13.0+k8s-1.29-49-gf92daca6", + }, + }, + }, + want: nil, + wantErr: true, + objects: []client.Object{ + &embeddedclusterv1beta1.Installation{ + TypeMeta: metav1.TypeMeta{ + Kind: "Installation", + APIVersion: "v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "20241002205018", + }, + Spec: embeddedclusterv1beta1.InstallationSpec{ + Config: &embeddedclusterv1beta1.ConfigSpec{ + Version: "1.13.0+k8s-1.29-49-gf92daca6", + }, + }, + }, + }, + }, + { + name: "multiple previous installations", + in: &embeddedclusterv1beta1.Installation{ + TypeMeta: metav1.TypeMeta{ + Kind: "Installation", + APIVersion: "v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "20241002205018", + }, + Spec: embeddedclusterv1beta1.InstallationSpec{ + Config: &embeddedclusterv1beta1.ConfigSpec{ + Version: "1.13.0+k8s-1.29-49-gf92daca6", + }, + }, + }, + want: &embeddedclusterv1beta1.Installation{ + TypeMeta: metav1.TypeMeta{ + Kind: "Installation", + APIVersion: "v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "20230000000000", + ResourceVersion: "1000", + }, + Spec: embeddedclusterv1beta1.InstallationSpec{ + Config: &embeddedclusterv1beta1.ConfigSpec{ + Version: "1.12.0+k8s-1.29-49-gf92daca6", + }, + RuntimeConfig: &embeddedclusterv1beta1.RuntimeConfigSpec{ + K0sDataDirOverride: "/var/lib/k0s", + OpenEBSDataDirOverride: "/var/openebs", + }, + }, + }, + wantErr: false, + objects: []client.Object{ + &embeddedclusterv1beta1.Installation{ + TypeMeta: metav1.TypeMeta{ + Kind: "Installation", + APIVersion: "v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "20220000000000", + }, + Spec: embeddedclusterv1beta1.InstallationSpec{ + Config: &embeddedclusterv1beta1.ConfigSpec{ + Version: "1.11.0+k8s-1.29-49-gf92daca6", + }, + }, + }, + &embeddedclusterv1beta1.Installation{ + TypeMeta: metav1.TypeMeta{ + Kind: "Installation", + APIVersion: "v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "20241002205018", + }, + Spec: embeddedclusterv1beta1.InstallationSpec{ + Config: &embeddedclusterv1beta1.ConfigSpec{ + Version: "1.13.0+k8s-1.29-49-gf92daca6", + }, + }, + }, + &embeddedclusterv1beta1.Installation{ + TypeMeta: metav1.TypeMeta{ + Kind: "Installation", + APIVersion: "v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "20230000000000", + }, + Spec: embeddedclusterv1beta1.InstallationSpec{ + Config: &embeddedclusterv1beta1.ConfigSpec{ + Version: "1.12.0+k8s-1.29-49-gf92daca6", + }, + }, + }, + &embeddedclusterv1beta1.Installation{ + TypeMeta: metav1.TypeMeta{ + Kind: "Installation", + APIVersion: "v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "20210000000000", + }, + Spec: embeddedclusterv1beta1.InstallationSpec{ + Config: &embeddedclusterv1beta1.ConfigSpec{ + Version: "1.10.0+k8s-1.29-49-gf92daca6", + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := require.New(t) + cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(tt.objects...).Build() + + got, err := GetPreviousInstallation(context.Background(), cli, tt.in) + if tt.wantErr { + req.Error(err) + return + } + req.NoError(err) + req.Equal(tt.want, got) + }) + } +} + +func Test_lessThanK0s115(t *testing.T) { + type args struct { + ver *semver.Version + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "less than 1.15", + args: args{ + ver: semver.MustParse("1.14.0+k8s-1.29-49-gf92daca6"), + }, + want: true, + }, + { + name: "greater than or equal to 1.15", + args: args{ + ver: semver.MustParse("1.15.0+k8s-1.29-49-gf92daca6"), + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := lessThanK0s115(tt.args.ver); got != tt.want { + t.Errorf("lessThanK0s115() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGetInstallation(t *testing.T) { + scheme := scheme.Scheme + embeddedclusterv1beta1.AddToScheme(scheme) + + type args struct { + name string + } + tests := []struct { + name string + args args + want *embeddedclusterv1beta1.Installation + wantErr bool + objects []client.Object + }{ + { + name: "migrates data dirs for previous versions prior to 1.15", + args: args{ + name: "20241002205018", + }, + want: &embeddedclusterv1beta1.Installation{ + TypeMeta: metav1.TypeMeta{ + Kind: "Installation", + APIVersion: "v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "20241002205018", + ResourceVersion: "1000", + }, + Spec: embeddedclusterv1beta1.InstallationSpec{ + Config: &embeddedclusterv1beta1.ConfigSpec{ + Version: "1.15.0+k8s-1.29-49-gf92daca6", + }, + RuntimeConfig: &embeddedclusterv1beta1.RuntimeConfigSpec{ + K0sDataDirOverride: "/var/lib/k0s", + OpenEBSDataDirOverride: "/var/openebs", + }, + }, + }, + wantErr: false, + objects: []client.Object{ + &embeddedclusterv1beta1.Installation{ + TypeMeta: metav1.TypeMeta{ + Kind: "Installation", + APIVersion: "v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "20241002205018", + }, + Spec: embeddedclusterv1beta1.InstallationSpec{ + Config: &embeddedclusterv1beta1.ConfigSpec{ + Version: "1.15.0+k8s-1.29-49-gf92daca6", + }, + }, + }, + &embeddedclusterv1beta1.Installation{ + TypeMeta: metav1.TypeMeta{ + Kind: "Installation", + APIVersion: "v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "20231002205018", + }, + Spec: embeddedclusterv1beta1.InstallationSpec{ + Config: &embeddedclusterv1beta1.ConfigSpec{ + Version: "1.14.0+k8s-1.29-49-gf92daca6", + }, + }, + }, + }, + }, + { + name: "does not migrate data dirs for previous version 1.15 or greater", + args: args{ + name: "20241002205018", + }, + want: &embeddedclusterv1beta1.Installation{ + TypeMeta: metav1.TypeMeta{ + Kind: "Installation", + APIVersion: "v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "20241002205018", + ResourceVersion: "999", + }, + Spec: embeddedclusterv1beta1.InstallationSpec{ + Config: &embeddedclusterv1beta1.ConfigSpec{ + Version: "1.15.1+k8s-1.29-49-gf92daca6", + }, + }, + }, + wantErr: false, + objects: []client.Object{ + &embeddedclusterv1beta1.Installation{ + TypeMeta: metav1.TypeMeta{ + Kind: "Installation", + APIVersion: "v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "20241002205018", + }, + Spec: embeddedclusterv1beta1.InstallationSpec{ + Config: &embeddedclusterv1beta1.ConfigSpec{ + Version: "1.15.1+k8s-1.29-49-gf92daca6", + }, + }, + }, + &embeddedclusterv1beta1.Installation{ + TypeMeta: metav1.TypeMeta{ + Kind: "Installation", + APIVersion: "v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "20231002205018", + }, + Spec: embeddedclusterv1beta1.InstallationSpec{ + Config: &embeddedclusterv1beta1.ConfigSpec{ + Version: "1.15.0+k8s-1.29-49-gf92daca6", + }, + }, + }, + }, + }, + { + name: "does not migrate data dirs if no previous installation", + args: args{ + name: "20241002205018", + }, + want: &embeddedclusterv1beta1.Installation{ + TypeMeta: metav1.TypeMeta{ + Kind: "Installation", + APIVersion: "v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "20241002205018", + ResourceVersion: "999", + }, + Spec: embeddedclusterv1beta1.InstallationSpec{ + Config: &embeddedclusterv1beta1.ConfigSpec{ + Version: "1.15.0+k8s-1.29-49-gf92daca6", + }, + }, + }, + wantErr: false, + objects: []client.Object{ + &embeddedclusterv1beta1.Installation{ + TypeMeta: metav1.TypeMeta{ + Kind: "Installation", + APIVersion: "v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "20241002205018", + }, + Spec: embeddedclusterv1beta1.InstallationSpec{ + Config: &embeddedclusterv1beta1.ConfigSpec{ + Version: "1.15.0+k8s-1.29-49-gf92daca6", + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := require.New(t) + cli := fake.NewClientBuilder().WithScheme(scheme).WithObjects(tt.objects...).Build() + + got, err := GetInstallation(context.Background(), cli, tt.args.name) + if tt.wantErr { + req.Error(err) + return + } + req.NoError(err) + req.Equal(tt.want, got) + }) + } +}