From 528b282d81da6627eb116bfae84892f169b361e6 Mon Sep 17 00:00:00 2001 From: Fabrizio Pandini Date: Fri, 10 Jan 2025 15:36:31 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=8C=B1=20Prepare=20main=20branch=20for=20?= =?UTF-8?q?v1.13=20development=20(#3321)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Prepare main branch for v1.13 development * Add clusterclass kustomization for v1.11 * Bump runtime extension providers * Fix CAPV n-1 version --- Makefile | 19 +- clusterctl-settings.json | 2 +- metadata.yaml | 3 + test/e2e/clusterctl_upgrade_test.go | 46 +- test/e2e/config/vsphere.yaml | 36 +- .../clusterclass-quick-start.yaml | 52 +- .../clusterclass/kustomization.yaml | 3 + .../patch-k8s-install-script.yaml | 221 ++++++++ .../clusterclass/patch-namingstrategy.yaml | 0 .../clusterclass/patch-prekubeadmscript.yaml | 0 .../clusterclass/patch-vsphere-template.yaml | 53 ++ .../commons/cluster-network-CIDR.yaml | 0 .../commons/cluster-resource-set-label.yaml | 0 .../commons/cluster-resource-set.yaml | 0 .../topology/cluster-template-topology.yaml | 507 +++++++++--------- .../topology/kustomization.yaml | 1 - .../workload/kustomization.yaml | 0 .../workload-control-plane-endpoint-ip.yaml | 0 .../clusterclass/patch-vsphere-template.yaml | 37 -- .../cluster-resource-set-csi-insecure.yaml | 27 - .../clusterclass-quick-start-supervisor.yaml | 2 +- .../clusterclass/kustomization.yaml | 7 +- .../patch-k8s-install-script.yaml | 221 ++++++++ .../clusterclass/patch-namingstrategy.yaml | 0 .../clusterclass/patch-prekubeadmscript.yaml | 14 +- .../clusterclass/patch-vm-namingstrategy.yaml | 4 + .../clusterclass/patch-vsphere-template.yaml | 20 +- .../commons/cluster-network-CIDR.yaml | 0 .../commons/cluster-resource-set-label.yaml | 0 .../commons/cluster-resource-set.yaml | 0 .../cluster-template-topology-supervisor.yaml | 110 ++-- .../topology/kustomization.yaml | 1 - .../workload/kustomization.yaml | 0 .../workload-control-plane-endpoint-ip.yaml | 2 +- .../patch-controlplaneendpoint.yaml | 5 - .../cluster-resource-set-csi-insecure.yaml | 27 - test/e2e/data/shared/capv/main/metadata.yaml | 3 + .../shared/capv/{v1.9 => v1.12}/metadata.yaml | 10 +- test/extension/tilt-provider.yaml | 2 +- .../net-operator/tilt-provider.yaml | 2 +- test/infrastructure/vcsim/tilt-provider.yaml | 2 +- tilt-provider.yaml | 4 +- 42 files changed, 927 insertions(+), 516 deletions(-) rename test/e2e/data/infrastructure-vsphere-govmomi/{v1.9 => v1.12}/clusterclass/clusterclass-quick-start.yaml (94%) rename test/e2e/data/infrastructure-vsphere-govmomi/{v1.9 => v1.12}/clusterclass/kustomization.yaml (81%) create mode 100644 test/e2e/data/infrastructure-vsphere-govmomi/v1.12/clusterclass/patch-k8s-install-script.yaml rename test/e2e/data/infrastructure-vsphere-govmomi/{v1.9 => v1.12}/clusterclass/patch-namingstrategy.yaml (100%) rename test/e2e/data/{infrastructure-vsphere-supervisor/v1.9 => infrastructure-vsphere-govmomi/v1.12}/clusterclass/patch-prekubeadmscript.yaml (100%) create mode 100644 test/e2e/data/infrastructure-vsphere-govmomi/v1.12/clusterclass/patch-vsphere-template.yaml rename test/e2e/data/infrastructure-vsphere-govmomi/{v1.9 => v1.12}/commons/cluster-network-CIDR.yaml (100%) rename test/e2e/data/infrastructure-vsphere-govmomi/{v1.9 => v1.12}/commons/cluster-resource-set-label.yaml (100%) rename test/e2e/data/infrastructure-vsphere-govmomi/{v1.9 => v1.12}/commons/cluster-resource-set.yaml (100%) rename test/e2e/data/infrastructure-vsphere-govmomi/{v1.9 => v1.12}/topology/cluster-template-topology.yaml (86%) rename test/e2e/data/infrastructure-vsphere-govmomi/{v1.9 => v1.12}/topology/kustomization.yaml (82%) rename test/e2e/data/infrastructure-vsphere-govmomi/{v1.9 => v1.12}/workload/kustomization.yaml (100%) rename test/e2e/data/{infrastructure-vsphere-supervisor/v1.9 => infrastructure-vsphere-govmomi/v1.12}/workload/workload-control-plane-endpoint-ip.yaml (100%) delete mode 100644 test/e2e/data/infrastructure-vsphere-govmomi/v1.9/clusterclass/patch-vsphere-template.yaml delete mode 100644 test/e2e/data/infrastructure-vsphere-govmomi/v1.9/commons/cluster-resource-set-csi-insecure.yaml rename test/e2e/data/infrastructure-vsphere-supervisor/{v1.9 => v1.12}/clusterclass/clusterclass-quick-start-supervisor.yaml (99%) rename test/e2e/data/infrastructure-vsphere-supervisor/{v1.9 => v1.12}/clusterclass/kustomization.yaml (70%) create mode 100644 test/e2e/data/infrastructure-vsphere-supervisor/v1.12/clusterclass/patch-k8s-install-script.yaml rename test/e2e/data/infrastructure-vsphere-supervisor/{v1.9 => v1.12}/clusterclass/patch-namingstrategy.yaml (100%) rename test/e2e/data/{infrastructure-vsphere-govmomi/v1.9 => infrastructure-vsphere-supervisor/v1.12}/clusterclass/patch-prekubeadmscript.yaml (70%) create mode 100644 test/e2e/data/infrastructure-vsphere-supervisor/v1.12/clusterclass/patch-vm-namingstrategy.yaml rename test/e2e/data/infrastructure-vsphere-supervisor/{v1.9 => v1.12}/clusterclass/patch-vsphere-template.yaml (70%) rename test/e2e/data/infrastructure-vsphere-supervisor/{v1.9 => v1.12}/commons/cluster-network-CIDR.yaml (100%) rename test/e2e/data/infrastructure-vsphere-supervisor/{v1.9 => v1.12}/commons/cluster-resource-set-label.yaml (100%) rename test/e2e/data/infrastructure-vsphere-supervisor/{v1.9 => v1.12}/commons/cluster-resource-set.yaml (100%) rename test/e2e/data/infrastructure-vsphere-supervisor/{v1.9 => v1.12}/topology/cluster-template-topology-supervisor.yaml (94%) rename test/e2e/data/infrastructure-vsphere-supervisor/{v1.9 => v1.12}/topology/kustomization.yaml (83%) rename test/e2e/data/infrastructure-vsphere-supervisor/{v1.9 => v1.12}/workload/kustomization.yaml (100%) rename test/e2e/data/{infrastructure-vsphere-govmomi/v1.9 => infrastructure-vsphere-supervisor/v1.12}/workload/workload-control-plane-endpoint-ip.yaml (74%) delete mode 100644 test/e2e/data/infrastructure-vsphere-supervisor/v1.9/clusterclass/patch-controlplaneendpoint.yaml delete mode 100644 test/e2e/data/infrastructure-vsphere-supervisor/v1.9/commons/cluster-resource-set-csi-insecure.yaml rename test/e2e/data/shared/capv/{v1.9 => v1.12}/metadata.yaml (84%) diff --git a/Makefile b/Makefile index 5f40866ff2..338e3575a1 100644 --- a/Makefile +++ b/Makefile @@ -367,7 +367,7 @@ generate-doctoc: TRACE=$(TRACE) ./hack/generate-doctoc.sh .PHONY: generate-e2e-templates -generate-e2e-templates: $(KUSTOMIZE) $(addprefix generate-e2e-templates-, v1.9 v1.10 v1.11 main) ## Generate test templates for all branches +generate-e2e-templates: $(KUSTOMIZE) $(addprefix generate-e2e-templates-, v1.10 v1.11 v1.12 main) ## Generate test templates for all branches .PHONY: generate-e2e-templates-main generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the main branch @@ -413,6 +413,14 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/fast-rollout" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-fast-rollout-supervisor.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/ownerrefs-finalizers" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-ownerrefs-finalizers-supervisor.yaml" +.PHONY: generate-e2e-templates-v1.12 +generate-e2e-templates-v1.12: $(KUSTOMIZE) + "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/v1.12/clusterclass" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/v1.12/clusterclass-quick-start.yaml" + "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/v1.12/workload" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/v1.12/cluster-template-workload.yaml" + + "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/v1.12/clusterclass" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/v1.12/clusterclass-quick-start-supervisor.yaml" + "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/v1.12/workload" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/v1.12/cluster-template-workload-supervisor.yaml" + .PHONY: generate-e2e-templates-v1.11 generate-e2e-templates-v1.11: $(KUSTOMIZE) "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/v1.11/clusterclass" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/v1.11/clusterclass-quick-start.yaml" @@ -421,7 +429,6 @@ generate-e2e-templates-v1.11: $(KUSTOMIZE) "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/v1.11/clusterclass" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/v1.11/clusterclass-quick-start-supervisor.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/v1.11/workload" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/v1.11/cluster-template-workload-supervisor.yaml" - .PHONY: generate-e2e-templates-v1.10 generate-e2e-templates-v1.10: $(KUSTOMIZE) "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/v1.10/clusterclass" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/v1.10/clusterclass-quick-start.yaml" @@ -430,14 +437,6 @@ generate-e2e-templates-v1.10: $(KUSTOMIZE) "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/v1.10/clusterclass" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/v1.10/clusterclass-quick-start-supervisor.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/v1.10/workload" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/v1.10/cluster-template-workload-supervisor.yaml" -.PHONY: generate-e2e-templates-v1.9 -generate-e2e-templates-v1.9: $(KUSTOMIZE) - "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/v1.9/clusterclass" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/v1.9/clusterclass-quick-start.yaml" - "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/v1.9/workload" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/v1.9/cluster-template-workload.yaml" - - "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/v1.9/clusterclass" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/v1.9/clusterclass-quick-start-supervisor.yaml" - "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/v1.9/workload" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/v1.9/cluster-template-workload-supervisor.yaml" - .PHONY: generate-test-infra-prowjobs generate-test-infra-prowjobs: $(PROWJOB_GEN) ## Generates the prowjob configurations in test-infra @if [ -z "${TEST_INFRA_DIR}" ]; then echo "TEST_INFRA_DIR is not set"; exit 1; fi diff --git a/clusterctl-settings.json b/clusterctl-settings.json index f91d971760..46bb68576b 100644 --- a/clusterctl-settings.json +++ b/clusterctl-settings.json @@ -2,6 +2,6 @@ "name": "infrastructure-vsphere", "config": { "componentsFile": "infrastructure-components.yaml", - "nextVersion": "v1.12.99" + "nextVersion": "v1.13.99" } } diff --git a/metadata.yaml b/metadata.yaml index b4c095f206..49225ef1f4 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -45,3 +45,6 @@ releaseSeries: - major: 1 minor: 12 contract: v1beta1 + - major: 1 + minor: 13 + contract: v1beta1 diff --git a/test/e2e/clusterctl_upgrade_test.go b/test/e2e/clusterctl_upgrade_test.go index d60bddfeec..2defb4f292 100644 --- a/test/e2e/clusterctl_upgrade_test.go +++ b/test/e2e/clusterctl_upgrade_test.go @@ -40,14 +40,14 @@ var ( ) // Note: This test should be changed during "prepare main branch", it should test CAPV n-1 => current (and then corresponding CAPI versions if already available). -var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.11=>current, CAPI 1.8=>1.9) on K8S latest ci mgmt cluster [vcsim] [supervisor] [ClusterClass]", func() { - const specName = "clusterctl-upgrade-1.11-current-latest-ci" // prefix (clusterctl-upgrade) copied from CAPI +var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.12=>current, CAPI 1.9=>1.10) on K8S latest ci mgmt cluster [vcsim] [supervisor] [ClusterClass]", func() { + const specName = "clusterctl-upgrade-1.12-current-latest-ci" // prefix (clusterctl-upgrade) copied from CAPI Setup(specName, func(testSpecificSettingsGetter func() testSettings) { capi_e2e.ClusterctlUpgradeSpec(ctx, func() capi_e2e.ClusterctlUpgradeSpecInput { - capiVersion := "1.8" + capiVersion := "1.9" capiStableRelease, err := getStableReleaseOfMinor(ctx, capiReleaseMarkerPrefix, capiVersion) Expect(err).ToNot(HaveOccurred(), "Failed to get stable version for minor release : %s", capiVersion) - capvVersion := "1.11" + capvVersion := "1.12" capvStableRelease, err := getStableReleaseOfMinor(ctx, capvReleaseMarkerPrefix, capvVersion) Expect(err).ToNot(HaveOccurred(), "Failed to get stable version for minor release : %s", capvVersion) initKubernetesVersion, err := kubernetesversions.ResolveVersion(ctx, e2eConfig.GetVariable("KUBERNETES_VERSION_LATEST_CI")) @@ -71,7 +71,7 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.11 // This is to guarantee that both, the old and new CAPI version, support the defined version. // Ensure all Kubernetes versions used here are covered in patch-vsphere-template.yaml InitWithKubernetesVersion: initKubernetesVersion, - WorkloadKubernetesVersion: "v1.31.0", + WorkloadKubernetesVersion: "v1.32.0", WorkloadFlavor: testSpecificSettingsGetter().FlavorForMode("workload"), // We are using a separate management cluster. For running in VCSim we also have to pass WithAdditionalVCSimServer // below otherwise there will be no VCSim instance created in the management cluster. @@ -87,14 +87,14 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.11 }) // Note: This test should be changed during "prepare main branch", it should test CAPV n-1 => current (and then corresponding CAPI versions if already available). -var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.11=>current, CAPI 1.8=>1.9) [vcsim] [supervisor] [ClusterClass]", func() { - const specName = "clusterctl-upgrade-1.11-current" // prefix (clusterctl-upgrade) copied from CAPI +var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.12=>current, CAPI 1.9=>1.10) [vcsim] [supervisor] [ClusterClass]", func() { + const specName = "clusterctl-upgrade-1.12-current" // prefix (clusterctl-upgrade) copied from CAPI Setup(specName, func(testSpecificSettingsGetter func() testSettings) { capi_e2e.ClusterctlUpgradeSpec(ctx, func() capi_e2e.ClusterctlUpgradeSpecInput { - capiVersion := "1.8" + capiVersion := "1.9" capiStableRelease, err := getStableReleaseOfMinor(ctx, capiReleaseMarkerPrefix, capiVersion) Expect(err).ToNot(HaveOccurred(), "Failed to get stable version for minor release : %s", capiVersion) - capvVersion := "1.11" + capvVersion := "1.12" capvStableRelease, err := getStableReleaseOfMinor(ctx, capvReleaseMarkerPrefix, capvVersion) Expect(err).ToNot(HaveOccurred(), "Failed to get stable version for minor release : %s", capvVersion) return capi_e2e.ClusterctlUpgradeSpecInput{ @@ -115,8 +115,8 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.11 // InitWithKubernetesVersion should be the highest kubernetes version supported by the init Cluster API version. // This is to guarantee that both, the old and new CAPI version, support the defined version. // Ensure all Kubernetes versions used here are covered in patch-vsphere-template.yaml - InitWithKubernetesVersion: "v1.31.0", - WorkloadKubernetesVersion: "v1.31.0", + InitWithKubernetesVersion: "v1.32.0", + WorkloadKubernetesVersion: "v1.32.0", WorkloadFlavor: testSpecificSettingsGetter().FlavorForMode("workload"), // We are using a separate management cluster. For running in VCSim we also have to pass WithAdditionalVCSimServer // below otherwise there will be no VCSim instance created in the management cluster. @@ -132,14 +132,14 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.11 }) // Note: This test should be changed during "prepare main branch", it should test CAPV n-2 => current (and then corresponding CAPI versions if already available). -var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.10=>current, CAPI 1.7=>1.9) [vcsim] [supervisor] [ClusterClass]", func() { - const specName = "clusterctl-upgrade-1.10-current" // prefix (clusterctl-upgrade) copied from CAPI +var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.11=>current, CAPI 1.8=>1.10) [vcsim] [supervisor] [ClusterClass]", func() { + const specName = "clusterctl-upgrade-1.11-current" // prefix (clusterctl-upgrade) copied from CAPI Setup(specName, func(testSpecificSettingsGetter func() testSettings) { capi_e2e.ClusterctlUpgradeSpec(ctx, func() capi_e2e.ClusterctlUpgradeSpecInput { - capiVersion := "1.7" + capiVersion := "1.8" capiStableRelease, err := getStableReleaseOfMinor(ctx, capiReleaseMarkerPrefix, capiVersion) Expect(err).ToNot(HaveOccurred(), "Failed to get stable version for minor release : %s", capiVersion) - capvVersion := "1.10" + capvVersion := "1.11" capvStableRelease, err := getStableReleaseOfMinor(ctx, capvReleaseMarkerPrefix, capvVersion) Expect(err).ToNot(HaveOccurred(), "Failed to get stable version for minor release : %s", capvVersion) return capi_e2e.ClusterctlUpgradeSpecInput{ @@ -160,8 +160,8 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.10 // InitWithKubernetesVersion should be the highest kubernetes version supported by the init Cluster API version. // This is to guarantee that both, the old and new CAPI version, support the defined version. // Ensure all Kubernetes versions used here are covered in patch-vsphere-template.yaml - InitWithKubernetesVersion: "v1.30.0", - WorkloadKubernetesVersion: "v1.30.0", + InitWithKubernetesVersion: "v1.31.0", + WorkloadKubernetesVersion: "v1.31.0", WorkloadFlavor: testSpecificSettingsGetter().FlavorForMode("workload"), // We are using a separate management cluster. For running in VCSim we also have to pass WithAdditionalVCSimServer // below otherwise there will be no VCSim instance created in the management cluster. @@ -177,14 +177,14 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.10 }) // Note: This test should be changed during "prepare main branch", it should test CAPV n-3 => current (and then corresponding CAPI versions if already available). -var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.9=>current, CAPI 1.6=>1.9) [vcsim] [supervisor] [ClusterClass]", func() { - const specName = "clusterctl-upgrade-1.9-current" // prefix (clusterctl-upgrade) copied from CAPI +var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.10=>current, CAPI 1.7=>1.10) [vcsim] [supervisor] [ClusterClass]", func() { + const specName = "clusterctl-upgrade-1.10-current" // prefix (clusterctl-upgrade) copied from CAPI Setup(specName, func(testSpecificSettingsGetter func() testSettings) { capi_e2e.ClusterctlUpgradeSpec(ctx, func() capi_e2e.ClusterctlUpgradeSpecInput { - capiVersion := "1.6" + capiVersion := "1.7" capiStableRelease, err := getStableReleaseOfMinor(ctx, capiReleaseMarkerPrefix, capiVersion) Expect(err).ToNot(HaveOccurred(), "Failed to get stable version for minor release : %s", capiVersion) - capvVersion := "1.9" + capvVersion := "1.10" capvStableRelease, err := getStableReleaseOfMinor(ctx, capvReleaseMarkerPrefix, capvVersion) Expect(err).ToNot(HaveOccurred(), "Failed to get stable version for minor release : %s", capvVersion) return capi_e2e.ClusterctlUpgradeSpecInput{ @@ -205,8 +205,8 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.9= // InitWithKubernetesVersion should be the highest kubernetes version supported by the init Cluster API version. // This is to guarantee that both, the old and new CAPI version, support the defined version. // Ensure all Kubernetes versions used here are covered in patch-vsphere-template.yaml - InitWithKubernetesVersion: "v1.29.0", - WorkloadKubernetesVersion: "v1.29.0", + InitWithKubernetesVersion: "v1.30.0", + WorkloadKubernetesVersion: "v1.30.0", WorkloadFlavor: testSpecificSettingsGetter().FlavorForMode("workload"), // We are using a separate management cluster. For running in VCSim we also have to pass WithAdditionalVCSimServer // below otherwise there will be no VCSim instance created in the management cluster. diff --git a/test/e2e/config/vsphere.yaml b/test/e2e/config/vsphere.yaml index de70e4cef3..44ef3023bf 100644 --- a/test/e2e/config/vsphere.yaml +++ b/test/e2e/config/vsphere.yaml @@ -159,12 +159,11 @@ providers: - name: vsphere type: InfrastructureProvider versions: - - name: v1.12.99 # next release + - name: v1.13.99 # next release # Use manifest from source files value: ../../../../cluster-api-provider-vsphere/config/default contract: v1beta1 files: - # Add a cluster template - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-conformance.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-dhcp-overrides.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-hw-upgrade.yaml" @@ -193,44 +192,41 @@ providers: - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-conformance-supervisor.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-ownerrefs-finalizers-supervisor.yaml" - sourcePath: "../data/shared/capv/main/metadata.yaml" - - name: "{go://sigs.k8s.io/cluster-api-provider-vsphere@v1.11}" # supported release in the v1beta1 series + - name: "{go://sigs.k8s.io/cluster-api-provider-vsphere@v1.12}" + value: "https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/{go://sigs.k8s.io/cluster-api-provider-vsphere@v1.12}/infrastructure-components.yaml" + type: "url" + contract: v1beta1 + files: + - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/v1.12/cluster-template-workload.yaml" + - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/v1.12/clusterclass-quick-start.yaml" + - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/v1.12/cluster-template-workload-supervisor.yaml" + - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/v1.12/clusterclass-quick-start-supervisor.yaml" + - sourcePath: "../data/shared/capv/v1.12/metadata.yaml" + - name: "{go://sigs.k8s.io/cluster-api-provider-vsphere@v1.11}" value: "https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/{go://sigs.k8s.io/cluster-api-provider-vsphere@v1.11}/infrastructure-components.yaml" type: "url" contract: v1beta1 files: - # Add a cluster template - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/v1.11/cluster-template-workload.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/v1.11/clusterclass-quick-start.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/v1.11/cluster-template-workload-supervisor.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/v1.11/clusterclass-quick-start-supervisor.yaml" - sourcePath: "../data/shared/capv/v1.11/metadata.yaml" - - name: "{go://sigs.k8s.io/cluster-api-provider-vsphere@v1.10}" # supported release in the v1beta1 series + - name: "{go://sigs.k8s.io/cluster-api-provider-vsphere@v1.10}" value: "https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/{go://sigs.k8s.io/cluster-api-provider-vsphere@v1.10}/infrastructure-components.yaml" type: "url" contract: v1beta1 files: - # Add a cluster template - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/v1.10/cluster-template-workload.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/v1.10/clusterclass-quick-start.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/v1.10/cluster-template-workload-supervisor.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/v1.10/clusterclass-quick-start-supervisor.yaml" - sourcePath: "../data/shared/capv/v1.10/metadata.yaml" - - name: "{go://sigs.k8s.io/cluster-api-provider-vsphere@v1.9}" # supported release in the v1beta1 series - value: "https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/{go://sigs.k8s.io/cluster-api-provider-vsphere@v1.9}/infrastructure-components.yaml" - type: "url" - contract: v1beta1 - files: - # Add a cluster template - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/v1.9/cluster-template-workload.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/v1.9/clusterclass-quick-start.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/v1.9/cluster-template-workload-supervisor.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/v1.9/clusterclass-quick-start-supervisor.yaml" - - sourcePath: "../data/shared/capv/v1.9/metadata.yaml" - name: vcsim type: RuntimeExtensionProvider # vcsim isn't a provider, but we fake it is so it can be handled by the clusterctl machinery. versions: - - name: v1.12.99 + - name: v1.13.99 # Use manifest from source files value: ../../../../cluster-api-provider-vsphere/test/infrastructure/vcsim/config/default contract: v1beta1 @@ -255,7 +251,7 @@ providers: - name: net-operator type: RuntimeExtensionProvider # net-operator isn't a provider, but we fake it is so it can be handled by the clusterctl machinery. versions: - - name: v1.12.99 + - name: v1.13.99 # Use manifest from source files value: ../../../../cluster-api-provider-vsphere/test/infrastructure/net-operator/config/default contract: v1beta1 @@ -268,7 +264,7 @@ providers: - name: capv-test-extension type: RuntimeExtensionProvider versions: - - name: v1.12.99 + - name: v1.13.99 # Use manifest from source files value: ../../../../cluster-api-provider-vsphere/test/extension/config/default contract: v1beta1 diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/v1.9/clusterclass/clusterclass-quick-start.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/v1.12/clusterclass/clusterclass-quick-start.yaml similarity index 94% rename from test/e2e/data/infrastructure-vsphere-govmomi/v1.9/clusterclass/clusterclass-quick-start.yaml rename to test/e2e/data/infrastructure-vsphere-govmomi/v1.12/clusterclass/clusterclass-quick-start.yaml index 125c7ed255..dad9f0c707 100644 --- a/test/e2e/data/infrastructure-vsphere-govmomi/v1.9/clusterclass/clusterclass-quick-start.yaml +++ b/test/e2e/data/infrastructure-vsphere-govmomi/v1.12/clusterclass/clusterclass-quick-start.yaml @@ -1,4 +1,3 @@ ---- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: VSphereClusterTemplate metadata: @@ -168,7 +167,7 @@ spec: # Nothing to do for kubernetes < v1.29 KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)" if [[ "$KUBEADM_MINOR" -lt "29" ]]; then - return + exit 0 fi IS_KUBEADM_INIT="false" @@ -188,12 +187,8 @@ spec: /etc/kubernetes/manifests/kube-vip.yaml fi owner: root:root - path: /etc/kube-vip-prepare.sh + path: /etc/pre-kubeadm-commands/50-kube-vip-prepare.sh permissions: "0700" - - op: add - path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/- - valueFrom: - template: /etc/kube-vip-prepare.sh selector: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlaneTemplate @@ -201,23 +196,15 @@ spec: controlPlane: true name: kubeVipPodManifest variables: - - name: sshKey + - metadata: {} + name: sshKey required: false schema: openAPIV3Schema: description: Public key to SSH onto the cluster nodes. type: string - - name: infraServer - required: true - schema: - openAPIV3Schema: - properties: - thumbprint: - type: string - url: - type: string - type: object - - name: controlPlaneIpAddr + - metadata: {} + name: controlPlaneIpAddr required: true schema: openAPIV3Schema: @@ -230,17 +217,30 @@ spec: openAPIV3Schema: description: Port for the control plane endpoint. type: integer - - name: credsSecretName + - metadata: {} + name: kubeVipPodManifest required: true schema: openAPIV3Schema: - description: Secret containing the credentials for the infra cluster. + description: kube-vip manifest for the control plane. type: string - - name: kubeVipPodManifest + - metadata: {} + name: infraServer required: true schema: openAPIV3Schema: - description: kube-vip manifest for the control plane. + properties: + thumbprint: + type: string + url: + type: string + type: object + - metadata: {} + name: credsSecretName + required: true + schema: + openAPIV3Schema: + description: Secret containing the credentials for the infra cluster. type: string workers: machineDeployments: @@ -348,6 +348,9 @@ spec: >/etc/hosts - echo "127.0.0.1 {{ ds.meta_data.hostname }} {{ local_hostname }} localhost localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts + - mkdir -p /etc/pre-kubeadm-commands + - for script in $(find /etc/pre-kubeadm-commands/ -name '*.sh' -type f | sort); + do echo "Running script $script"; "$script"; done users: - name: capv sshAuthorizedKeys: @@ -374,3 +377,6 @@ spec: >/etc/hosts - echo "127.0.0.1 {{ ds.meta_data.hostname }} {{ local_hostname }} localhost localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts + - mkdir -p /etc/pre-kubeadm-commands + - for script in $(find /etc/pre-kubeadm-commands/ -name '*.sh' -type f | sort); + do echo "Running script $script"; "$script"; done \ No newline at end of file diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/v1.9/clusterclass/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/v1.12/clusterclass/kustomization.yaml similarity index 81% rename from test/e2e/data/infrastructure-vsphere-govmomi/v1.9/clusterclass/kustomization.yaml rename to test/e2e/data/infrastructure-vsphere-govmomi/v1.12/clusterclass/kustomization.yaml index 4c0e41b050..4128896681 100644 --- a/test/e2e/data/infrastructure-vsphere-govmomi/v1.9/clusterclass/kustomization.yaml +++ b/test/e2e/data/infrastructure-vsphere-govmomi/v1.12/clusterclass/kustomization.yaml @@ -9,6 +9,9 @@ patches: - target: kind: ClusterClass path: ./patch-prekubeadmscript.yaml + - target: + kind: ClusterClass + path: ./patch-k8s-install-script.yaml - target: kind: ClusterClass path: ./patch-namingstrategy.yaml diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/v1.12/clusterclass/patch-k8s-install-script.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/v1.12/clusterclass/patch-k8s-install-script.yaml new file mode 100644 index 0000000000..51d92c895f --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-govmomi/v1.12/clusterclass/patch-k8s-install-script.yaml @@ -0,0 +1,221 @@ +- op: add + path: /spec/patches/- + value: + definitions: + - jsonPatches: + - op: add + path: /spec/template/spec/kubeadmConfigSpec/files/- + valueFrom: + template: | + owner: root:root + path: "/etc/pre-kubeadm-commands/20-k8s-install.sh" + permissions: "0755" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + DISTRO="ubuntu" + KUBE_BINARY_DIR="/usr/bin" + + source /etc/lsb-release + if [[ "$${DISTRIB_ID}" == *Flatcar* ]]; then + # Overrides for flatcar + DISTRO="flatcar" + KUBE_BINARY_DIR="/opt/bin" + fi + + function retry { + attempt=0 + max_attempts=$${1} + interval=$${2} + shift; shift + until [[ $${attempt} -ge "$${max_attempts}" ]] ; do + attempt=$((attempt+1)) + set +e + eval "$*" && return || echo "failed $${attempt} times: $*" + set -e + sleep "$${interval}" + done + echo "error: reached max attempts at retry($*)" + return 1 + } + + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$${LINE_SEPARATOR}" + + ## Variables (replaced by JSON patching) + KUBERNETES_VERSION={{ .builtin.controlPlane.version }} + ## + + # Note: We assume if kubectl has the right version, everything else has as well + if [[ $(kubectl version --client=true -o json | jq '.clientVersion.gitVersion' -r) = "$${KUBERNETES_VERSION}" ]]; then + echo "Detected Kubernetes $${KUBERNETES_VERSION} via kubectl version, nothing to do" + exit 0 + fi + + if [[ "$${KUBERNETES_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p "$${CI_DIR}" + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + # Let's just also download the control plane images for worker nodes. It's easier then optimizing it. + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing version $${KUBERNETES_VERSION}" + CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" + # Set CI_URL to the released binaries for actually released versions. + if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-(beta|rc).[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + fi + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev + # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl + echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" + wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" + chmod +x "$${CI_DIR}/$${CI_PACKAGE}" + mv "$${CI_DIR}/$${CI_PACKAGE}" "$${KUBE_BINARY_DIR}/$${CI_PACKAGE}" + done + systemctl restart kubelet + IMAGE_REGISTRY_PREFIX=registry.k8s.io + # Kubernetes builds from 1.20 through 1.24 are tagged with k8s.gcr.io + if [[ "$${KUBERNETES_VERSION}" =~ ^v1\.(1[0-9]|2[0-4])[\.[0-9]+ ]]; then + IMAGE_REGISTRY_PREFIX=k8s.gcr.io + fi + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/k8s-staging-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + done + fi + echo "* checking binary versions" + echo "ctr version: " "$(ctr version)" + echo "kubeadm version: " "$(kubeadm version -o=short)" + echo "kubectl version: " "$(kubectl version --client=true)" + echo "kubelet version: " "$(kubelet --version)" + echo "$${LINE_SEPARATOR}" + + selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + - jsonPatches: + - op: add + path: /spec/template/spec/files/- + valueFrom: + template: | + owner: root:root + path: "/etc/pre-kubeadm-commands/20-k8s-install.sh" + permissions: "0755" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + DISTRO="ubuntu" + KUBE_BINARY_DIR="/usr/bin" + + source /etc/lsb-release + if [[ "$${DISTRIB_ID}" == *Flatcar* ]]; then + # Overrides for flatcar + DISTRO="flatcar" + KUBE_BINARY_DIR="/opt/bin" + fi + + function retry { + attempt=0 + max_attempts=$${1} + interval=$${2} + shift; shift + until [[ $${attempt} -ge "$${max_attempts}" ]] ; do + attempt=$((attempt+1)) + set +e + eval "$*" && return || echo "failed $${attempt} times: $*" + set -e + sleep "$${interval}" + done + echo "error: reached max attempts at retry($*)" + return 1 + } + + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$${LINE_SEPARATOR}" + + ## Variables (replaced by JSON patching) + KUBERNETES_VERSION={{ .builtin.machineDeployment.version }} + ## + + # Note: We assume if kubectl has the right version, everything else has as well + if [[ $(kubectl version --client=true -o json | jq '.clientVersion.gitVersion' -r) = "$${KUBERNETES_VERSION}" ]]; then + echo "Detected Kubernetes $${KUBERNETES_VERSION} via kubectl version, nothing to do" + exit 0 + fi + + if [[ "$${KUBERNETES_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p "$${CI_DIR}" + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + # Let's just also download the control plane images for worker nodes. It's easier then optimizing it. + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing version $${KUBERNETES_VERSION}" + CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" + # Set CI_URL to the released binaries for actually released versions. + if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-(beta|rc).[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + fi + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev + # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl + echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" + wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" + chmod +x "$${CI_DIR}/$${CI_PACKAGE}" + mv "$${CI_DIR}/$${CI_PACKAGE}" "$${KUBE_BINARY_DIR}/$${CI_PACKAGE}" + done + systemctl restart kubelet + IMAGE_REGISTRY_PREFIX=registry.k8s.io + # Kubernetes builds from 1.20 through 1.24 are tagged with k8s.gcr.io + if [[ "$${KUBERNETES_VERSION}" =~ ^v1\.(1[0-9]|2[0-4])[\.[0-9]+ ]]; then + IMAGE_REGISTRY_PREFIX=k8s.gcr.io + fi + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/k8s-staging-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + done + fi + echo "* checking binary versions" + echo "ctr version: " "$(ctr version)" + echo "kubeadm version: " "$(kubeadm version -o=short)" + echo "kubectl version: " "$(kubectl version --client=true)" + echo "kubelet version: " "$(kubelet --version)" + echo "$${LINE_SEPARATOR}" + + selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + matchResources: + machineDeploymentClass: + names: + - ${CLUSTER_CLASS_NAME}-worker + name: k8sInstallScript diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/v1.9/clusterclass/patch-namingstrategy.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/v1.12/clusterclass/patch-namingstrategy.yaml similarity index 100% rename from test/e2e/data/infrastructure-vsphere-govmomi/v1.9/clusterclass/patch-namingstrategy.yaml rename to test/e2e/data/infrastructure-vsphere-govmomi/v1.12/clusterclass/patch-namingstrategy.yaml diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/v1.9/clusterclass/patch-prekubeadmscript.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/v1.12/clusterclass/patch-prekubeadmscript.yaml similarity index 100% rename from test/e2e/data/infrastructure-vsphere-supervisor/v1.9/clusterclass/patch-prekubeadmscript.yaml rename to test/e2e/data/infrastructure-vsphere-govmomi/v1.12/clusterclass/patch-prekubeadmscript.yaml diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/v1.12/clusterclass/patch-vsphere-template.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/v1.12/clusterclass/patch-vsphere-template.yaml new file mode 100644 index 0000000000..21a4cebc30 --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-govmomi/v1.12/clusterclass/patch-vsphere-template.yaml @@ -0,0 +1,53 @@ +- op: add + path: /spec/patches/- + value: + definitions: + - jsonPatches: + - op: replace + path: /spec/template/spec/template + valueFrom: + # We have to fall back to v1.30.0 for the conformance latest ci test which uses + # versions without corresponding templates like "v1.30.0-alpha.0.525+09a5049ca78502". + template: |- + {{- if eq .builtin.controlPlane.version "v1.28.0" -}} + ubuntu-2204-kube-v1.28.0 + {{- else if eq .builtin.controlPlane.version "v1.29.0" -}} + ubuntu-2204-kube-v1.29.0 + {{- else if eq .builtin.controlPlane.version "v1.30.0" -}} + ubuntu-2204-kube-v1.30.0 + {{- else if eq .builtin.controlPlane.version "v1.31.0" -}} + ubuntu-2404-kube-v1.31.0 + {{- else -}} + ubuntu-2404-kube-v1.32.0 + {{- end -}} + selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: VSphereMachineTemplate + matchResources: + controlPlane: true + - jsonPatches: + - op: replace + path: /spec/template/spec/template + valueFrom: + # We have to fall back to v1.30.0 for the conformance latest ci test which uses + # versions without corresponding templates like "v1.30.0-alpha.0.525+09a5049ca78502". + template: |- + {{- if eq .builtin.machineDeployment.version "v1.28.0" -}} + ubuntu-2204-kube-v1.28.0 + {{- else if eq .builtin.machineDeployment.version "v1.29.0" -}} + ubuntu-2204-kube-v1.29.0 + {{- else if eq .builtin.machineDeployment.version "v1.30.0" -}} + ubuntu-2204-kube-v1.30.0 + {{- else if eq .builtin.controlPlane.version "v1.31.0" -}} + ubuntu-2404-kube-v1.31.0 + {{- else -}} + ubuntu-2404-kube-v1.32.0 + {{- end -}} + selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: VSphereMachineTemplate + matchResources: + machineDeploymentClass: + names: + - ${CLUSTER_CLASS_NAME}-worker + name: vSphereTemplate diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/v1.9/commons/cluster-network-CIDR.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/v1.12/commons/cluster-network-CIDR.yaml similarity index 100% rename from test/e2e/data/infrastructure-vsphere-govmomi/v1.9/commons/cluster-network-CIDR.yaml rename to test/e2e/data/infrastructure-vsphere-govmomi/v1.12/commons/cluster-network-CIDR.yaml diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/v1.9/commons/cluster-resource-set-label.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/v1.12/commons/cluster-resource-set-label.yaml similarity index 100% rename from test/e2e/data/infrastructure-vsphere-govmomi/v1.9/commons/cluster-resource-set-label.yaml rename to test/e2e/data/infrastructure-vsphere-govmomi/v1.12/commons/cluster-resource-set-label.yaml diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/v1.9/commons/cluster-resource-set.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/v1.12/commons/cluster-resource-set.yaml similarity index 100% rename from test/e2e/data/infrastructure-vsphere-govmomi/v1.9/commons/cluster-resource-set.yaml rename to test/e2e/data/infrastructure-vsphere-govmomi/v1.12/commons/cluster-resource-set.yaml diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/v1.9/topology/cluster-template-topology.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/v1.12/topology/cluster-template-topology.yaml similarity index 86% rename from test/e2e/data/infrastructure-vsphere-govmomi/v1.9/topology/cluster-template-topology.yaml rename to test/e2e/data/infrastructure-vsphere-govmomi/v1.12/topology/cluster-template-topology.yaml index b6fb4ede6d..6b6669471e 100644 --- a/test/e2e/data/infrastructure-vsphere-govmomi/v1.9/topology/cluster-template-topology.yaml +++ b/test/e2e/data/infrastructure-vsphere-govmomi/v1.12/topology/cluster-template-topology.yaml @@ -1,4 +1,3 @@ ---- apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: @@ -14,15 +13,12 @@ spec: variables: - name: sshKey value: '${VSPHERE_SSH_AUTHORIZED_KEY}' - - name: infraServer - value: - thumbprint: '${VSPHERE_TLS_THUMBPRINT}' - url: '${VSPHERE_SERVER}' - name: kubeVipPodManifest value: | apiVersion: v1 kind: Pod metadata: + creationTimestamp: null name: kube-vip namespace: kube-system spec: @@ -30,29 +26,41 @@ spec: - args: - manager env: - - name: cp_enable + - name: vip_arp value: "true" - - name: vip_interface - value: ${VIP_NETWORK_INTERFACE:=""} - - name: address - value: ${CONTROL_PLANE_ENDPOINT_IP} - name: port value: "6443" - - name: vip_arp + - name: vip_interface + value: ${VIP_NETWORK_INTERFACE:=""} + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: svc_election value: "true" - name: vip_leaderelection value: "true" + - name: vip_leasename + value: plndr-cp-lock - name: vip_leaseduration value: "15" - name: vip_renewdeadline value: "10" - name: vip_retryperiod value: "2" - - name: svc_enable - value: "true" - - name: svc_election - value: "true" - image: ghcr.io/kube-vip/kube-vip:v0.6.3 + - name: address + value: ${CONTROL_PLANE_ENDPOINT_IP} + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.6.4 imagePullPolicy: IfNotPresent name: kube-vip resources: {} @@ -70,16 +78,20 @@ spec: volumes: - hostPath: path: /etc/kubernetes/admin.conf - type: File name: kubeconfig - hostPath: path: /etc/kube-vip.hosts type: File name: etchosts + status: {} - name: controlPlaneIpAddr value: ${CONTROL_PLANE_ENDPOINT_IP} - name: controlPlanePort value: ${CONTROL_PLANE_ENDPOINT_PORT:=6443} + - name: infraServer + value: + thumbprint: '${VSPHERE_TLS_THUMBPRINT}' + url: '${VSPHERE_SERVER}' - name: credsSecretName value: '${CLUSTER_NAME}' version: '${KUBERNETES_VERSION}' @@ -96,8 +108,8 @@ metadata: name: '${CLUSTER_NAME}' namespace: '${NAMESPACE}' stringData: - password: ${VSPHERE_PASSWORD} - username: ${VSPHERE_USERNAME} + password: "${VSPHERE_PASSWORD}" + username: "${VSPHERE_USERNAME}" --- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet @@ -115,8 +127,6 @@ spec: name: vsphere-config-secret - kind: ConfigMap name: csi-manifests - - kind: Secret - name: cloud-controller-manager - kind: Secret name: cloud-provider-vsphere-credentials - kind: ConfigMap @@ -128,7 +138,7 @@ metadata: name: vsphere-config-secret namespace: '${NAMESPACE}' stringData: - data: | + data: |- apiVersion: v1 kind: Secret metadata: @@ -152,8 +162,7 @@ type: addons.cluster.x-k8s.io/resource-set --- apiVersion: v1 data: - data: | - --- + data: |- apiVersion: v1 kind: Namespace metadata: @@ -445,20 +454,7 @@ data: --- apiVersion: v1 data: - async-query-volume: "true" - block-volume-snapshot: "true" - cnsmgr-suspend-create-volume: "true" - csi-auth-check: "true" - csi-internal-generated-cluster-id: "true" - csi-migration: "true" - csi-windows-support: "true" - list-volumes: "true" - listview-tasks: "true" - max-pvscsi-targets-per-vm: "true" - multi-vcenter-csi-topology: "true" - online-volume-extend: "true" pv-to-backingdiskobjectid-mapping: "false" - topology-preferential-datastores: "true" trigger-csi-fullsync: "false" kind: ConfigMap metadata: @@ -506,6 +502,18 @@ data: role: vsphere-csi spec: affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/controlplane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: @@ -529,7 +537,7 @@ data: env: - name: ADDRESS value: /csi/csi.sock - image: registry.k8s.io/sig-storage/csi-attacher:v4.3.0 + image: registry.k8s.io/sig-storage/csi-attacher:v4.5.1 name: csi-attacher volumeMounts: - mountPath: /csi @@ -548,7 +556,7 @@ data: env: - name: ADDRESS value: /csi/csi.sock - image: registry.k8s.io/sig-storage/csi-resizer:v1.8.0 + image: registry.k8s.io/sig-storage/csi-resizer:v1.10.1 name: csi-resizer volumeMounts: - mountPath: /csi @@ -577,7 +585,7 @@ data: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/csi-vsphere/driver:v3.1.2 + image: registry.k8s.io/csi-vsphere/driver:v3.3.1 imagePullPolicy: Always livenessProbe: failureThreshold: 3 @@ -608,7 +616,7 @@ data: - args: - --v=4 - --csi-address=/csi/csi.sock - image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 + image: registry.k8s.io/sig-storage/livenessprobe:v2.12.0 name: liveness-probe volumeMounts: - mountPath: /csi @@ -631,13 +639,11 @@ data: value: "100" - name: INCLUSTER_CLIENT_BURST value: "100" - - name: GODEBUG - value: x509sha1=1 - name: CSI_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/csi-vsphere/syncer:v3.1.2 + image: registry.k8s.io/csi-vsphere/syncer:v3.3.1 imagePullPolicy: Always name: vsphere-syncer ports: @@ -666,7 +672,7 @@ data: env: - name: ADDRESS value: /csi/csi.sock - image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v4.0.1 name: csi-provisioner volumeMounts: - mountPath: /csi @@ -684,7 +690,7 @@ data: env: - name: ADDRESS value: /csi/csi.sock - image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2 + image: registry.k8s.io/sig-storage/csi-snapshotter:v7.0.2 name: csi-snapshotter volumeMounts: - mountPath: /csi @@ -731,7 +737,7 @@ data: value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.10.1 livenessProbe: exec: command: @@ -765,15 +771,13 @@ data: value: "true" - name: LOGGER_LEVEL value: PRODUCTION - - name: GODEBUG - value: x509sha1=1 - name: CSI_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES value: "1" - image: registry.k8s.io/csi-vsphere/driver:v3.1.2 + image: registry.k8s.io/csi-vsphere/driver:v3.3.1 imagePullPolicy: Always livenessProbe: failureThreshold: 3 @@ -809,7 +813,7 @@ data: - args: - --v=4 - --csi-address=/csi/csi.sock - image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 + image: registry.k8s.io/sig-storage/livenessprobe:v2.12.0 name: liveness-probe volumeMounts: - mountPath: /csi @@ -879,7 +883,7 @@ data: value: unix://C:\\csi\\csi.sock - name: DRIVER_REG_SOCK_PATH value: C:\\var\\lib\\kubelet\\plugins\\csi.vsphere.vmware.com\\csi.sock - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.10.0 livenessProbe: exec: command: @@ -922,7 +926,7 @@ data: fieldPath: metadata.namespace - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES value: "1" - image: registry.k8s.io/csi-vsphere/driver:v3.1.2 + image: registry.k8s.io/csi-vsphere/driver:v3.3.1 imagePullPolicy: Always livenessProbe: failureThreshold: 3 @@ -953,7 +957,7 @@ data: - args: - --v=4 - --csi-address=/csi/csi.sock - image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 + image: registry.k8s.io/sig-storage/livenessprobe:v2.12.0 name: liveness-probe volumeMounts: - mountPath: /csi @@ -1007,28 +1011,11 @@ metadata: --- apiVersion: v1 kind: Secret -metadata: - name: cloud-controller-manager - namespace: '${NAMESPACE}' -stringData: - data: | - apiVersion: v1 - kind: ServiceAccount - metadata: - labels: - component: cloud-controller-manager - vsphere-cpi-infra: service-account - name: cloud-controller-manager - namespace: kube-system -type: addons.cluster.x-k8s.io/resource-set ---- -apiVersion: v1 -kind: Secret metadata: name: cloud-provider-vsphere-credentials namespace: '${NAMESPACE}' stringData: - data: | + data: |- apiVersion: v1 kind: Secret metadata: @@ -1038,117 +1025,224 @@ stringData: name: cloud-provider-vsphere-credentials namespace: kube-system stringData: - ${VSPHERE_SERVER}.password: ${VSPHERE_PASSWORD} - ${VSPHERE_SERVER}.username: ${VSPHERE_USERNAME} + ${VSPHERE_SERVER}.password: "${VSPHERE_PASSWORD}" + ${VSPHERE_SERVER}.username: "${VSPHERE_USERNAME}" type: Opaque type: addons.cluster.x-k8s.io/resource-set --- apiVersion: v1 data: - data: | + data: |- --- + # Source: vsphere-cpi/templates/service-account.yaml + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + labels: + app: vsphere-cpi + vsphere-cpi-infra: service-account + component: cloud-controller-manager + namespace: kube-system + --- + # Source: vsphere-cpi/templates/role.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: + name: cloud-controller-manager labels: - component: cloud-controller-manager + app: vsphere-cpi vsphere-cpi-infra: role - name: system:cloud-controller-manager + component: cloud-controller-manager rules: - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update - - apiGroups: - - "" - resources: - - nodes - verbs: - - '*' - - apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch - - apiGroups: - - "" - resources: - - services - verbs: - - list - - patch - - update - - watch - - apiGroups: - - "" - resources: - - services/status - verbs: - - patch - - apiGroups: - - "" - resources: - - serviceaccounts - verbs: - - create - - get - - list - - watch - - update - - apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - get - - list - - watch - - update - - apiGroups: - - "" - resources: - - endpoints - verbs: - - create - - get - - list - - watch - - update - - apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - watch - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - watch - - list - - update - - create + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - "*" + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - patch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch + - apiGroups: + - "coordination.k8s.io" + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + --- + # Source: vsphere-cpi/templates/daemonset.yaml + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: vsphere-cpi + labels: + app: vsphere-cpi + vsphere-cpi-infra: daemonset + component: cloud-controller-manager + tier: control-plane + namespace: kube-system + annotations: + spec: + selector: + matchLabels: + app: vsphere-cpi + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: vsphere-cpi + component: cloud-controller-manager + tier: control-plane + release: release-name + vsphere-cpi-infra: daemonset + spec: + tolerations: + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + - effect: NoExecute + key: CriticalAddonsOnly + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + securityContext: + fsGroup: 1001 + runAsUser: 1001 + serviceAccountName: cloud-controller-manager + hostNetwork: true + dnsPolicy: ClusterFirst + priorityClassName: system-node-critical + containers: + - name: vsphere-cpi + image: registry.k8s.io/cloud-pv-vsphere/cloud-provider-vsphere:${CPI_IMAGE_K8S_VERSION} + imagePullPolicy: IfNotPresent + args: + - --cloud-provider=vsphere + - --v=2 + - --cloud-config=/etc/cloud/vsphere.conf + volumeMounts: + - mountPath: /etc/cloud + name: vsphere-config-volume + readOnly: true + volumes: + - name: vsphere-config-volume + configMap: + name: cloud-config + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + labels: + app: vsphere-cpi + component: cloud-controller-manager + vsphere-cpi-infra: role-binding + name: servicecatalog.k8s.io:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + - apiGroup: "" + kind: User + name: cloud-controller-manager --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: + app: vsphere-cpi component: cloud-controller-manager vsphere-cpi-infra: cluster-role-binding - name: system:cloud-controller-manager + name: cloud-controller-manager roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: system:cloud-controller-manager + name: cloud-controller-manager subjects: - kind: ServiceAccount name: cloud-controller-manager @@ -1171,96 +1265,9 @@ data: server: '${VSPHERE_SERVER}' kind: ConfigMap metadata: - name: vsphere-cloud-config + name: cloud-config namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: RoleBinding - metadata: - labels: - component: cloud-controller-manager - vsphere-cpi-infra: role-binding - name: servicecatalog.k8s.io:apiserver-authentication-reader - namespace: kube-system - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader - subjects: - - kind: ServiceAccount - name: cloud-controller-manager - namespace: kube-system - - kind: User - name: cloud-controller-manager - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - component: cloud-controller-manager - tier: control-plane - name: vsphere-cloud-controller-manager - namespace: kube-system - spec: - selector: - matchLabels: - name: vsphere-cloud-controller-manager - template: - metadata: - labels: - component: cloud-controller-manager - name: vsphere-cloud-controller-manager - tier: control-plane - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - containers: - - args: - - --v=2 - - --cloud-provider=vsphere - - --cloud-config=/etc/cloud/vsphere.conf - image: registry.k8s.io/cloud-pv-vsphere/cloud-provider-vsphere:${CPI_IMAGE_K8S_VERSION} - name: vsphere-cloud-controller-manager - resources: - requests: - cpu: 200m - volumeMounts: - - mountPath: /etc/cloud - name: vsphere-config-volume - readOnly: true - hostNetwork: true - priorityClassName: system-node-critical - securityContext: - runAsUser: 1001 - serviceAccountName: cloud-controller-manager - tolerations: - - effect: NoSchedule - key: node.cloudprovider.kubernetes.io/uninitialized - value: "true" - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Exists - - effect: NoSchedule - key: node.kubernetes.io/not-ready - operator: Exists - volumes: - - configMap: - name: vsphere-cloud-config - name: vsphere-config-volume - updateStrategy: - type: RollingUpdate kind: ConfigMap metadata: name: cpi-manifests - namespace: '${NAMESPACE}' + namespace: '${NAMESPACE}' \ No newline at end of file diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/v1.9/topology/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/v1.12/topology/kustomization.yaml similarity index 82% rename from test/e2e/data/infrastructure-vsphere-govmomi/v1.9/topology/kustomization.yaml rename to test/e2e/data/infrastructure-vsphere-govmomi/v1.12/topology/kustomization.yaml index 0a431a5996..1e265e778b 100644 --- a/test/e2e/data/infrastructure-vsphere-govmomi/v1.9/topology/kustomization.yaml +++ b/test/e2e/data/infrastructure-vsphere-govmomi/v1.12/topology/kustomization.yaml @@ -6,4 +6,3 @@ resources: patchesStrategicMerge: - ../commons/cluster-resource-set-label.yaml - ../commons/cluster-network-CIDR.yaml - - ../commons/cluster-resource-set-csi-insecure.yaml diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/v1.9/workload/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/v1.12/workload/kustomization.yaml similarity index 100% rename from test/e2e/data/infrastructure-vsphere-govmomi/v1.9/workload/kustomization.yaml rename to test/e2e/data/infrastructure-vsphere-govmomi/v1.12/workload/kustomization.yaml diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/v1.9/workload/workload-control-plane-endpoint-ip.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/v1.12/workload/workload-control-plane-endpoint-ip.yaml similarity index 100% rename from test/e2e/data/infrastructure-vsphere-supervisor/v1.9/workload/workload-control-plane-endpoint-ip.yaml rename to test/e2e/data/infrastructure-vsphere-govmomi/v1.12/workload/workload-control-plane-endpoint-ip.yaml diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/v1.9/clusterclass/patch-vsphere-template.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/v1.9/clusterclass/patch-vsphere-template.yaml deleted file mode 100644 index 5f7f38db63..0000000000 --- a/test/e2e/data/infrastructure-vsphere-govmomi/v1.9/clusterclass/patch-vsphere-template.yaml +++ /dev/null @@ -1,37 +0,0 @@ -- op: add - path: /spec/patches/- - value: - definitions: - - jsonPatches: - - op: replace - path: /spec/template/spec/template - valueFrom: - template: |- - {{- if semverCompare ">= v1.28" .builtin.controlPlane.version -}} - ubuntu-2204-kube-{{ .builtin.controlPlane.version }} - {{- else -}} - ubuntu-2004-kube-{{ .builtin.controlPlane.version }} - {{- end -}} - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: VSphereMachineTemplate - matchResources: - controlPlane: true - - jsonPatches: - - op: replace - path: /spec/template/spec/template - valueFrom: - template: |- - {{- if semverCompare ">= v1.28" .builtin.machineDeployment.version -}} - ubuntu-2204-kube-{{ .builtin.machineDeployment.version }} - {{- else -}} - ubuntu-2004-kube-{{ .builtin.machineDeployment.version }} - {{- end -}} - selector: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: VSphereMachineTemplate - matchResources: - machineDeploymentClass: - names: - - ${CLUSTER_CLASS_NAME}-worker - name: vSphereTemplate diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/v1.9/commons/cluster-resource-set-csi-insecure.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/v1.9/commons/cluster-resource-set-csi-insecure.yaml deleted file mode 100644 index 1881a1cb52..0000000000 --- a/test/e2e/data/infrastructure-vsphere-govmomi/v1.9/commons/cluster-resource-set-csi-insecure.yaml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: vsphere-config-secret - namespace: '${NAMESPACE}' -stringData: - data: | - apiVersion: v1 - kind: Secret - metadata: - name: vsphere-config-secret - namespace: vmware-system-csi - stringData: - csi-vsphere.conf: |+ - [Global] - - [VirtualCenter "${VSPHERE_SERVER}"] - insecure-flag = "true" - user = "${VSPHERE_USERNAME}" - password = "${VSPHERE_PASSWORD}" - datacenters = "${VSPHERE_DATACENTER}" - - [Network] - public-network = "${VSPHERE_NETWORK}" - - type: Opaque -type: addons.cluster.x-k8s.io/resource-set diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/v1.9/clusterclass/clusterclass-quick-start-supervisor.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/v1.12/clusterclass/clusterclass-quick-start-supervisor.yaml similarity index 99% rename from test/e2e/data/infrastructure-vsphere-supervisor/v1.9/clusterclass/clusterclass-quick-start-supervisor.yaml rename to test/e2e/data/infrastructure-vsphere-supervisor/v1.12/clusterclass/clusterclass-quick-start-supervisor.yaml index f58e3d6dea..10561ef024 100644 --- a/test/e2e/data/infrastructure-vsphere-supervisor/v1.9/clusterclass/clusterclass-quick-start-supervisor.yaml +++ b/test/e2e/data/infrastructure-vsphere-supervisor/v1.12/clusterclass/clusterclass-quick-start-supervisor.yaml @@ -94,7 +94,7 @@ spec: name: enableSSHIntoNodes - definitions: - jsonPatches: - - op: replace + - op: add path: /spec/template/spec/controlPlaneEndpoint valueFrom: template: | diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/v1.9/clusterclass/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/v1.12/clusterclass/kustomization.yaml similarity index 70% rename from test/e2e/data/infrastructure-vsphere-supervisor/v1.9/clusterclass/kustomization.yaml rename to test/e2e/data/infrastructure-vsphere-supervisor/v1.12/clusterclass/kustomization.yaml index 83e3657c81..6858997f52 100644 --- a/test/e2e/data/infrastructure-vsphere-supervisor/v1.9/clusterclass/kustomization.yaml +++ b/test/e2e/data/infrastructure-vsphere-supervisor/v1.12/clusterclass/kustomization.yaml @@ -9,9 +9,12 @@ patches: - target: kind: ClusterClass path: ./patch-prekubeadmscript.yaml + - target: + kind: ClusterClass + path: ./patch-k8s-install-script.yaml - target: kind: ClusterClass path: ./patch-namingstrategy.yaml - target: - kind: VSphereClusterTemplate - path: ./patch-controlplaneendpoint.yaml + kind: VSphereMachineTemplate + path: ./patch-vm-namingstrategy.yaml diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/v1.12/clusterclass/patch-k8s-install-script.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/v1.12/clusterclass/patch-k8s-install-script.yaml new file mode 100644 index 0000000000..51d92c895f --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-supervisor/v1.12/clusterclass/patch-k8s-install-script.yaml @@ -0,0 +1,221 @@ +- op: add + path: /spec/patches/- + value: + definitions: + - jsonPatches: + - op: add + path: /spec/template/spec/kubeadmConfigSpec/files/- + valueFrom: + template: | + owner: root:root + path: "/etc/pre-kubeadm-commands/20-k8s-install.sh" + permissions: "0755" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + DISTRO="ubuntu" + KUBE_BINARY_DIR="/usr/bin" + + source /etc/lsb-release + if [[ "$${DISTRIB_ID}" == *Flatcar* ]]; then + # Overrides for flatcar + DISTRO="flatcar" + KUBE_BINARY_DIR="/opt/bin" + fi + + function retry { + attempt=0 + max_attempts=$${1} + interval=$${2} + shift; shift + until [[ $${attempt} -ge "$${max_attempts}" ]] ; do + attempt=$((attempt+1)) + set +e + eval "$*" && return || echo "failed $${attempt} times: $*" + set -e + sleep "$${interval}" + done + echo "error: reached max attempts at retry($*)" + return 1 + } + + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$${LINE_SEPARATOR}" + + ## Variables (replaced by JSON patching) + KUBERNETES_VERSION={{ .builtin.controlPlane.version }} + ## + + # Note: We assume if kubectl has the right version, everything else has as well + if [[ $(kubectl version --client=true -o json | jq '.clientVersion.gitVersion' -r) = "$${KUBERNETES_VERSION}" ]]; then + echo "Detected Kubernetes $${KUBERNETES_VERSION} via kubectl version, nothing to do" + exit 0 + fi + + if [[ "$${KUBERNETES_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p "$${CI_DIR}" + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + # Let's just also download the control plane images for worker nodes. It's easier then optimizing it. + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing version $${KUBERNETES_VERSION}" + CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" + # Set CI_URL to the released binaries for actually released versions. + if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-(beta|rc).[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + fi + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev + # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl + echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" + wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" + chmod +x "$${CI_DIR}/$${CI_PACKAGE}" + mv "$${CI_DIR}/$${CI_PACKAGE}" "$${KUBE_BINARY_DIR}/$${CI_PACKAGE}" + done + systemctl restart kubelet + IMAGE_REGISTRY_PREFIX=registry.k8s.io + # Kubernetes builds from 1.20 through 1.24 are tagged with k8s.gcr.io + if [[ "$${KUBERNETES_VERSION}" =~ ^v1\.(1[0-9]|2[0-4])[\.[0-9]+ ]]; then + IMAGE_REGISTRY_PREFIX=k8s.gcr.io + fi + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/k8s-staging-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + done + fi + echo "* checking binary versions" + echo "ctr version: " "$(ctr version)" + echo "kubeadm version: " "$(kubeadm version -o=short)" + echo "kubectl version: " "$(kubectl version --client=true)" + echo "kubelet version: " "$(kubelet --version)" + echo "$${LINE_SEPARATOR}" + + selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + - jsonPatches: + - op: add + path: /spec/template/spec/files/- + valueFrom: + template: | + owner: root:root + path: "/etc/pre-kubeadm-commands/20-k8s-install.sh" + permissions: "0755" + content: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + DISTRO="ubuntu" + KUBE_BINARY_DIR="/usr/bin" + + source /etc/lsb-release + if [[ "$${DISTRIB_ID}" == *Flatcar* ]]; then + # Overrides for flatcar + DISTRO="flatcar" + KUBE_BINARY_DIR="/opt/bin" + fi + + function retry { + attempt=0 + max_attempts=$${1} + interval=$${2} + shift; shift + until [[ $${attempt} -ge "$${max_attempts}" ]] ; do + attempt=$((attempt+1)) + set +e + eval "$*" && return || echo "failed $${attempt} times: $*" + set -e + sleep "$${interval}" + done + echo "error: reached max attempts at retry($*)" + return 1 + } + + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$${LINE_SEPARATOR}" + + ## Variables (replaced by JSON patching) + KUBERNETES_VERSION={{ .builtin.machineDeployment.version }} + ## + + # Note: We assume if kubectl has the right version, everything else has as well + if [[ $(kubectl version --client=true -o json | jq '.clientVersion.gitVersion' -r) = "$${KUBERNETES_VERSION}" ]]; then + echo "Detected Kubernetes $${KUBERNETES_VERSION} via kubectl version, nothing to do" + exit 0 + fi + + if [[ "$${KUBERNETES_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p "$${CI_DIR}" + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + # Let's just also download the control plane images for worker nodes. It's easier then optimizing it. + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing version $${KUBERNETES_VERSION}" + CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" + # Set CI_URL to the released binaries for actually released versions. + if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-(beta|rc).[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + fi + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev + # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl + echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" + wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" + chmod +x "$${CI_DIR}/$${CI_PACKAGE}" + mv "$${CI_DIR}/$${CI_PACKAGE}" "$${KUBE_BINARY_DIR}/$${CI_PACKAGE}" + done + systemctl restart kubelet + IMAGE_REGISTRY_PREFIX=registry.k8s.io + # Kubernetes builds from 1.20 through 1.24 are tagged with k8s.gcr.io + if [[ "$${KUBERNETES_VERSION}" =~ ^v1\.(1[0-9]|2[0-4])[\.[0-9]+ ]]; then + IMAGE_REGISTRY_PREFIX=k8s.gcr.io + fi + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/k8s-staging-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + done + fi + echo "* checking binary versions" + echo "ctr version: " "$(ctr version)" + echo "kubeadm version: " "$(kubeadm version -o=short)" + echo "kubectl version: " "$(kubectl version --client=true)" + echo "kubelet version: " "$(kubelet --version)" + echo "$${LINE_SEPARATOR}" + + selector: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + matchResources: + machineDeploymentClass: + names: + - ${CLUSTER_CLASS_NAME}-worker + name: k8sInstallScript diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/v1.9/clusterclass/patch-namingstrategy.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/v1.12/clusterclass/patch-namingstrategy.yaml similarity index 100% rename from test/e2e/data/infrastructure-vsphere-supervisor/v1.9/clusterclass/patch-namingstrategy.yaml rename to test/e2e/data/infrastructure-vsphere-supervisor/v1.12/clusterclass/patch-namingstrategy.yaml diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/v1.9/clusterclass/patch-prekubeadmscript.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/v1.12/clusterclass/patch-prekubeadmscript.yaml similarity index 70% rename from test/e2e/data/infrastructure-vsphere-govmomi/v1.9/clusterclass/patch-prekubeadmscript.yaml rename to test/e2e/data/infrastructure-vsphere-supervisor/v1.12/clusterclass/patch-prekubeadmscript.yaml index 3e6e63b28d..3345f5b6d2 100644 --- a/test/e2e/data/infrastructure-vsphere-govmomi/v1.9/clusterclass/patch-prekubeadmscript.yaml +++ b/test/e2e/data/infrastructure-vsphere-supervisor/v1.12/clusterclass/patch-prekubeadmscript.yaml @@ -3,34 +3,28 @@ value: definitions: - jsonPatches: - - op: add - path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/- - value: "/opt/prekubeadmscript.sh" - op: add path: /spec/template/spec/kubeadmConfigSpec/files/- valueFrom: template: | owner: root:root - path: "/opt/prekubeadmscript.sh" + path: "/etc/pre-kubeadm-commands/10-prekubeadmscript.sh" permissions: "0755" - content: {{ printf "%q" .preKubeadmScript }} + content: {{ printf "%q" (regexReplaceAll "(KUBERNETES_VERSION=.*)" .preKubeadmScript (printf "KUBERNETES_VERSION=%s" .builtin.controlPlane.version)) }} selector: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlaneTemplate matchResources: controlPlane: true - jsonPatches: - - op: add - path: /spec/template/spec/preKubeadmCommands/- - value: "/opt/prekubeadmscript.sh" - op: add path: /spec/template/spec/files/- valueFrom: template: | owner: root:root - path: "/opt/prekubeadmscript.sh" + path: "/etc/pre-kubeadm-commands/10-prekubeadmscript.sh" permissions: "0755" - content: {{ printf "%q" .preKubeadmScript }} + content: {{ printf "%q" (regexReplaceAll "(KUBERNETES_VERSION=.*)" .preKubeadmScript (printf "KUBERNETES_VERSION=%s" .builtin.machineDeployment.version)) }} selector: apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/v1.12/clusterclass/patch-vm-namingstrategy.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/v1.12/clusterclass/patch-vm-namingstrategy.yaml new file mode 100644 index 0000000000..0ec61e9df0 --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-supervisor/v1.12/clusterclass/patch-vm-namingstrategy.yaml @@ -0,0 +1,4 @@ +- op: add + path: /spec/template/spec/namingStrategy + value: + template: '{{ if le (len .machine.name) 20 }}{{ .machine.name }}{{else}}{{ trimSuffix "-" (trunc 14 .machine.name) }}-{{ trunc -5 .machine.name }}{{end}}' diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/v1.9/clusterclass/patch-vsphere-template.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/v1.12/clusterclass/patch-vsphere-template.yaml similarity index 70% rename from test/e2e/data/infrastructure-vsphere-supervisor/v1.9/clusterclass/patch-vsphere-template.yaml rename to test/e2e/data/infrastructure-vsphere-supervisor/v1.12/clusterclass/patch-vsphere-template.yaml index af98629345..cf501f348f 100644 --- a/test/e2e/data/infrastructure-vsphere-supervisor/v1.9/clusterclass/patch-vsphere-template.yaml +++ b/test/e2e/data/infrastructure-vsphere-supervisor/v1.12/clusterclass/patch-vsphere-template.yaml @@ -11,11 +11,15 @@ template: |- {{- if eq .builtin.controlPlane.version "v1.28.0" -}} ubuntu-2204-kube-v1.28.0 - {{- else -}}{{- if eq .builtin.controlPlane.version "v1.29.0" -}} + {{- else if eq .builtin.controlPlane.version "v1.29.0" -}} ubuntu-2204-kube-v1.29.0 - {{- else -}} + {{- else if eq .builtin.controlPlane.version "v1.30.0" -}} ubuntu-2204-kube-v1.30.0 - {{- end -}}{{- end -}} + {{- else if eq .builtin.controlPlane.version "v1.31.0" -}} + ubuntu-2404-kube-v1.31.0 + {{- else -}} + ubuntu-2404-kube-v1.32.0 + {{- end -}} selector: apiVersion: vmware.infrastructure.cluster.x-k8s.io/v1beta1 kind: VSphereMachineTemplate @@ -30,11 +34,15 @@ template: |- {{- if eq .builtin.machineDeployment.version "v1.28.0" -}} ubuntu-2204-kube-v1.28.0 - {{- else -}}{{- if eq .builtin.machineDeployment.version "v1.29.0" -}} + {{- else if eq .builtin.machineDeployment.version "v1.29.0" -}} ubuntu-2204-kube-v1.29.0 - {{- else -}} + {{- else if eq .builtin.machineDeployment.version "v1.30.0" -}} ubuntu-2204-kube-v1.30.0 - {{- end -}}{{- end -}} + {{- else if eq .builtin.controlPlane.version "v1.31.0" -}} + ubuntu-2404-kube-v1.31.0 + {{- else -}} + ubuntu-2404-kube-v1.32.0 + {{- end -}} selector: apiVersion: vmware.infrastructure.cluster.x-k8s.io/v1beta1 kind: VSphereMachineTemplate diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/v1.9/commons/cluster-network-CIDR.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/v1.12/commons/cluster-network-CIDR.yaml similarity index 100% rename from test/e2e/data/infrastructure-vsphere-supervisor/v1.9/commons/cluster-network-CIDR.yaml rename to test/e2e/data/infrastructure-vsphere-supervisor/v1.12/commons/cluster-network-CIDR.yaml diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/v1.9/commons/cluster-resource-set-label.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/v1.12/commons/cluster-resource-set-label.yaml similarity index 100% rename from test/e2e/data/infrastructure-vsphere-supervisor/v1.9/commons/cluster-resource-set-label.yaml rename to test/e2e/data/infrastructure-vsphere-supervisor/v1.12/commons/cluster-resource-set-label.yaml diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/v1.9/commons/cluster-resource-set.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/v1.12/commons/cluster-resource-set.yaml similarity index 100% rename from test/e2e/data/infrastructure-vsphere-supervisor/v1.9/commons/cluster-resource-set.yaml rename to test/e2e/data/infrastructure-vsphere-supervisor/v1.12/commons/cluster-resource-set.yaml diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/v1.9/topology/cluster-template-topology-supervisor.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/v1.12/topology/cluster-template-topology-supervisor.yaml similarity index 94% rename from test/e2e/data/infrastructure-vsphere-supervisor/v1.9/topology/cluster-template-topology-supervisor.yaml rename to test/e2e/data/infrastructure-vsphere-supervisor/v1.12/topology/cluster-template-topology-supervisor.yaml index 185164b2ee..0531a3095a 100644 --- a/test/e2e/data/infrastructure-vsphere-supervisor/v1.9/topology/cluster-template-topology-supervisor.yaml +++ b/test/e2e/data/infrastructure-vsphere-supervisor/v1.12/topology/cluster-template-topology-supervisor.yaml @@ -96,15 +96,6 @@ spec: name: md-0 replicas: ${WORKER_MACHINE_COUNT} --- -apiVersion: v1 -kind: Secret -metadata: - name: '${CLUSTER_NAME}' - namespace: '${NAMESPACE}' -stringData: - password: "${VSPHERE_PASSWORD}" - username: "${VSPHERE_USERNAME}" ---- apiVersion: addons.cluster.x-k8s.io/v1beta1 kind: ClusterResourceSet metadata: @@ -448,20 +439,7 @@ data: --- apiVersion: v1 data: - async-query-volume: "true" - block-volume-snapshot: "true" - cnsmgr-suspend-create-volume: "true" - csi-auth-check: "true" - csi-internal-generated-cluster-id: "true" - csi-migration: "true" - csi-windows-support: "true" - list-volumes: "true" - listview-tasks: "true" - max-pvscsi-targets-per-vm: "true" - multi-vcenter-csi-topology: "true" - online-volume-extend: "true" pv-to-backingdiskobjectid-mapping: "false" - topology-preferential-datastores: "true" trigger-csi-fullsync: "false" kind: ConfigMap metadata: @@ -509,6 +487,18 @@ data: role: vsphere-csi spec: affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/controlplane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: @@ -532,7 +522,7 @@ data: env: - name: ADDRESS value: /csi/csi.sock - image: registry.k8s.io/sig-storage/csi-attacher:v4.3.0 + image: registry.k8s.io/sig-storage/csi-attacher:v4.5.1 name: csi-attacher volumeMounts: - mountPath: /csi @@ -551,7 +541,7 @@ data: env: - name: ADDRESS value: /csi/csi.sock - image: registry.k8s.io/sig-storage/csi-resizer:v1.8.0 + image: registry.k8s.io/sig-storage/csi-resizer:v1.10.1 name: csi-resizer volumeMounts: - mountPath: /csi @@ -580,7 +570,7 @@ data: valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/csi-vsphere/driver:v3.1.2 + image: registry.k8s.io/csi-vsphere/driver:v3.3.1 imagePullPolicy: Always livenessProbe: failureThreshold: 3 @@ -611,7 +601,7 @@ data: - args: - --v=4 - --csi-address=/csi/csi.sock - image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 + image: registry.k8s.io/sig-storage/livenessprobe:v2.12.0 name: liveness-probe volumeMounts: - mountPath: /csi @@ -634,13 +624,11 @@ data: value: "100" - name: INCLUSTER_CLIENT_BURST value: "100" - - name: GODEBUG - value: x509sha1=1 - name: CSI_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - image: registry.k8s.io/csi-vsphere/syncer:v3.1.2 + image: registry.k8s.io/csi-vsphere/syncer:v3.3.1 imagePullPolicy: Always name: vsphere-syncer ports: @@ -669,7 +657,7 @@ data: env: - name: ADDRESS value: /csi/csi.sock - image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v4.0.1 name: csi-provisioner volumeMounts: - mountPath: /csi @@ -687,7 +675,7 @@ data: env: - name: ADDRESS value: /csi/csi.sock - image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2 + image: registry.k8s.io/sig-storage/csi-snapshotter:v7.0.2 name: csi-snapshotter volumeMounts: - mountPath: /csi @@ -734,7 +722,7 @@ data: value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH value: /var/lib/kubelet/plugins/csi.vsphere.vmware.com/csi.sock - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.10.1 livenessProbe: exec: command: @@ -768,15 +756,13 @@ data: value: "true" - name: LOGGER_LEVEL value: PRODUCTION - - name: GODEBUG - value: x509sha1=1 - name: CSI_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES value: "1" - image: registry.k8s.io/csi-vsphere/driver:v3.1.2 + image: registry.k8s.io/csi-vsphere/driver:v3.3.1 imagePullPolicy: Always livenessProbe: failureThreshold: 3 @@ -812,7 +798,7 @@ data: - args: - --v=4 - --csi-address=/csi/csi.sock - image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 + image: registry.k8s.io/sig-storage/livenessprobe:v2.12.0 name: liveness-probe volumeMounts: - mountPath: /csi @@ -882,7 +868,7 @@ data: value: unix://C:\\csi\\csi.sock - name: DRIVER_REG_SOCK_PATH value: C:\\var\\lib\\kubelet\\plugins\\csi.vsphere.vmware.com\\csi.sock - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0 + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.10.0 livenessProbe: exec: command: @@ -925,7 +911,7 @@ data: fieldPath: metadata.namespace - name: NODEGETINFO_WATCH_TIMEOUT_MINUTES value: "1" - image: registry.k8s.io/csi-vsphere/driver:v3.1.2 + image: registry.k8s.io/csi-vsphere/driver:v3.3.1 imagePullPolicy: Always livenessProbe: failureThreshold: 3 @@ -956,7 +942,7 @@ data: - args: - --v=4 - --csi-address=/csi/csi.sock - image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 + image: registry.k8s.io/sig-storage/livenessprobe:v2.12.0 name: liveness-probe volumeMounts: - mountPath: /csi @@ -1165,21 +1151,25 @@ data: vsphere-cpi-infra: daemonset spec: tolerations: - - key: node.cloudprovider.kubernetes.io/uninitialized - value: "true" - effect: NoSchedule - - key: node-role.kubernetes.io/master - effect: NoSchedule - operator: Exists - - key: node-role.kubernetes.io/control-plane - effect: NoSchedule - operator: Exists - - key: node.kubernetes.io/not-ready - effect: NoSchedule - operator: Exists - - key: CriticalAddonsOnly - effect: NoExecute - operator: Exists + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + - effect: NoExecute + key: CriticalAddonsOnly + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists securityContext: fsGroup: 1001 runAsUser: 1001 @@ -1203,16 +1193,6 @@ data: - name: vsphere-config-volume configMap: name: cloud-config - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -1275,4 +1255,4 @@ data: kind: ConfigMap metadata: name: cpi-manifests - namespace: '${NAMESPACE}' + namespace: '${NAMESPACE}' \ No newline at end of file diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/v1.9/topology/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/v1.12/topology/kustomization.yaml similarity index 83% rename from test/e2e/data/infrastructure-vsphere-supervisor/v1.9/topology/kustomization.yaml rename to test/e2e/data/infrastructure-vsphere-supervisor/v1.12/topology/kustomization.yaml index 2bcd486217..efdd2348c0 100644 --- a/test/e2e/data/infrastructure-vsphere-supervisor/v1.9/topology/kustomization.yaml +++ b/test/e2e/data/infrastructure-vsphere-supervisor/v1.12/topology/kustomization.yaml @@ -6,4 +6,3 @@ resources: patchesStrategicMerge: - ../commons/cluster-resource-set-label.yaml - ../commons/cluster-network-CIDR.yaml - - ../commons/cluster-resource-set-csi-insecure.yaml diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/v1.9/workload/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/v1.12/workload/kustomization.yaml similarity index 100% rename from test/e2e/data/infrastructure-vsphere-supervisor/v1.9/workload/kustomization.yaml rename to test/e2e/data/infrastructure-vsphere-supervisor/v1.12/workload/kustomization.yaml diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/v1.9/workload/workload-control-plane-endpoint-ip.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/v1.12/workload/workload-control-plane-endpoint-ip.yaml similarity index 74% rename from test/e2e/data/infrastructure-vsphere-govmomi/v1.9/workload/workload-control-plane-endpoint-ip.yaml rename to test/e2e/data/infrastructure-vsphere-supervisor/v1.12/workload/workload-control-plane-endpoint-ip.yaml index f3cfc35a4e..f36926df68 100644 --- a/test/e2e/data/infrastructure-vsphere-govmomi/v1.9/workload/workload-control-plane-endpoint-ip.yaml +++ b/test/e2e/data/infrastructure-vsphere-supervisor/v1.12/workload/workload-control-plane-endpoint-ip.yaml @@ -1,5 +1,5 @@ - op: replace - path: /spec/topology/variables/3 + path: /spec/topology/variables/2 value: name: controlPlaneIpAddr value: "${WORKLOAD_CONTROL_PLANE_ENDPOINT_IP}" diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/v1.9/clusterclass/patch-controlplaneendpoint.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/v1.9/clusterclass/patch-controlplaneendpoint.yaml deleted file mode 100644 index 470c5a85e4..0000000000 --- a/test/e2e/data/infrastructure-vsphere-supervisor/v1.9/clusterclass/patch-controlplaneendpoint.yaml +++ /dev/null @@ -1,5 +0,0 @@ -- op: add - path: /spec/template/spec/controlPlaneEndpoint - value: - host: "" - port: 0 diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/v1.9/commons/cluster-resource-set-csi-insecure.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/v1.9/commons/cluster-resource-set-csi-insecure.yaml deleted file mode 100644 index 1881a1cb52..0000000000 --- a/test/e2e/data/infrastructure-vsphere-supervisor/v1.9/commons/cluster-resource-set-csi-insecure.yaml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: vsphere-config-secret - namespace: '${NAMESPACE}' -stringData: - data: | - apiVersion: v1 - kind: Secret - metadata: - name: vsphere-config-secret - namespace: vmware-system-csi - stringData: - csi-vsphere.conf: |+ - [Global] - - [VirtualCenter "${VSPHERE_SERVER}"] - insecure-flag = "true" - user = "${VSPHERE_USERNAME}" - password = "${VSPHERE_PASSWORD}" - datacenters = "${VSPHERE_DATACENTER}" - - [Network] - public-network = "${VSPHERE_NETWORK}" - - type: Opaque -type: addons.cluster.x-k8s.io/resource-set diff --git a/test/e2e/data/shared/capv/main/metadata.yaml b/test/e2e/data/shared/capv/main/metadata.yaml index b4c095f206..49225ef1f4 100644 --- a/test/e2e/data/shared/capv/main/metadata.yaml +++ b/test/e2e/data/shared/capv/main/metadata.yaml @@ -45,3 +45,6 @@ releaseSeries: - major: 1 minor: 12 contract: v1beta1 + - major: 1 + minor: 13 + contract: v1beta1 diff --git a/test/e2e/data/shared/capv/v1.9/metadata.yaml b/test/e2e/data/shared/capv/v1.12/metadata.yaml similarity index 84% rename from test/e2e/data/shared/capv/v1.9/metadata.yaml rename to test/e2e/data/shared/capv/v1.12/metadata.yaml index 2f6bfcf984..b4c095f206 100644 --- a/test/e2e/data/shared/capv/v1.9/metadata.yaml +++ b/test/e2e/data/shared/capv/v1.12/metadata.yaml @@ -36,4 +36,12 @@ releaseSeries: - major: 1 minor: 9 contract: v1beta1 - + - major: 1 + minor: 10 + contract: v1beta1 + - major: 1 + minor: 11 + contract: v1beta1 + - major: 1 + minor: 12 + contract: v1beta1 diff --git a/test/extension/tilt-provider.yaml b/test/extension/tilt-provider.yaml index 2174c6969c..ea17e544d8 100644 --- a/test/extension/tilt-provider.yaml +++ b/test/extension/tilt-provider.yaml @@ -1,7 +1,7 @@ --- - name: capv-test-extension config: - version: v1.12.99 + version: v1.13.99 image: gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-test-extension live_reload_deps: - main.go diff --git a/test/infrastructure/net-operator/tilt-provider.yaml b/test/infrastructure/net-operator/tilt-provider.yaml index f9931e88a4..1dd699055c 100644 --- a/test/infrastructure/net-operator/tilt-provider.yaml +++ b/test/infrastructure/net-operator/tilt-provider.yaml @@ -1,7 +1,7 @@ --- - name: net-operator config: - version: v1.12.99 + version: v1.13.99 image: gcr.io/k8s-staging-capi-vsphere/cluster-api-net-operator live_reload_deps: - main.go diff --git a/test/infrastructure/vcsim/tilt-provider.yaml b/test/infrastructure/vcsim/tilt-provider.yaml index a128670b93..fae9e032d2 100644 --- a/test/infrastructure/vcsim/tilt-provider.yaml +++ b/test/infrastructure/vcsim/tilt-provider.yaml @@ -1,7 +1,7 @@ --- - name: vcsim config: - version: v1.12.99 + version: v1.13.99 image: gcr.io/k8s-staging-capi-vsphere/cluster-api-vcsim-controller live_reload_deps: - main.go diff --git a/tilt-provider.yaml b/tilt-provider.yaml index 1165865568..50b04a464e 100644 --- a/tilt-provider.yaml +++ b/tilt-provider.yaml @@ -1,7 +1,7 @@ --- - name: vsphere config: - version: v1.12.99 + version: v1.13.99 image: gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller live_reload_deps: - main.go @@ -15,7 +15,7 @@ label: CAPV - name: vsphere-supervisor config: - version: v1.12.99 + version: v1.13.99 image: gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller live_reload_deps: - main.go