diff --git a/.gitignore b/.gitignore index 1e0ca8c622..7203c391c8 100644 --- a/.gitignore +++ b/.gitignore @@ -19,12 +19,12 @@ _artifacts/ test/e2e/data/infrastructure-vsphere-govmomi/main/**/clusterclass-quick-start.yaml test/e2e/data/infrastructure-vsphere-govmomi/main/**/cluster-template*.yaml test/e2e/data/infrastructure-vsphere-govmomi/*/cluster-template*.yaml -test/e2e/data/infrastructure-vsphere-govmomi/*/clusterclass-quick-start.yaml +test/e2e/data/infrastructure-vsphere-govmomi/*/clusterclass-quick-start*.yaml test/e2e/data/infrastructure-vsphere-supervisor/main/**/clusterclass-quick-start-supervisor.yaml test/e2e/data/infrastructure-vsphere-supervisor/main/**/cluster-template*.yaml test/e2e/data/infrastructure-vsphere-supervisor/*/cluster-template*.yaml -test/e2e/data/infrastructure-vsphere-supervisor/*/clusterclass-quick-start-supervisor.yaml +test/e2e/data/infrastructure-vsphere-supervisor/*/clusterclass-quick-start*-supervisor.yaml # env vars file used in getting-started.md and manifests generation envvars.txt diff --git a/Makefile b/Makefile index 14c9395342..efd41091ee 100644 --- a/Makefile +++ b/Makefile @@ -388,8 +388,10 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai # generate clusterclass and cluster topology cp "$(RELEASE_DIR)/main/clusterclass-template.yaml" "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass/clusterclass-quick-start.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass-quick-start.yaml" + "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass-runtimesdk" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass-quick-start-runtimesdk.yaml" cp "$(RELEASE_DIR)/main/cluster-template-topology.yaml" "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology/cluster-template-topology.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-topology.yaml" + "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology-runtimesdk" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-topology-runtimesdk.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/install-on-bootstrap" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-install-on-bootstrap.yaml" # for PCI passthrough template "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/pci" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-pci.yaml" @@ -403,8 +405,10 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/base" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-supervisor.yaml" cp "$(RELEASE_DIR)/main/clusterclass-template-supervisor.yaml" "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/clusterclass/clusterclass-quick-start-supervisor.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/clusterclass" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/clusterclass-quick-start-supervisor.yaml" + "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/clusterclass-runtimesdk" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/clusterclass-quick-start-runtimesdk-supervisor.yaml" cp "$(RELEASE_DIR)/main/cluster-template-topology-supervisor.yaml" "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology/cluster-template-topology-supervisor.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-topology-supervisor.yaml" + "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology-runtimesdk" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-topology-runtimesdk-supervisor.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/conformance" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-conformance-supervisor.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/install-on-bootstrap" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-install-on-bootstrap-supervisor.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/ownerrefs-finalizers" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-ownerrefs-finalizers-supervisor.yaml" diff --git a/test/e2e/cluster_upgrade_runtimesdk_test.go b/test/e2e/cluster_upgrade_runtimesdk_test.go new file mode 100644 index 0000000000..f5f1b370b1 --- /dev/null +++ b/test/e2e/cluster_upgrade_runtimesdk_test.go @@ -0,0 +1,57 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "github.com/blang/semver/v4" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/utils/ptr" + capi_e2e "sigs.k8s.io/cluster-api/test/e2e" + "sigs.k8s.io/cluster-api/test/framework" +) + +var _ = Describe("When upgrading a workload cluster using ClusterClass with RuntimeSDK [supervisor] [ClusterClass]", func() { + const specName = "k8s-upgrade-with-runtimesdk" // aligned to CAPI + Setup(specName, func(testSpecificSettingsGetter func() testSettings) { + capi_e2e.ClusterUpgradeWithRuntimeSDKSpec(ctx, func() capi_e2e.ClusterUpgradeWithRuntimeSDKSpecInput { + version, err := semver.ParseTolerant(e2eConfig.GetVariable(capi_e2e.KubernetesVersionUpgradeFrom)) + Expect(err).ToNot(HaveOccurred(), "Invalid argument, KUBERNETES_VERSION_UPGRADE_FROM is not a valid version") + if version.LT(semver.MustParse("1.24.0")) { + Fail("This test only supports upgrades from Kubernetes >= v1.24.0") + } + + return capi_e2e.ClusterUpgradeWithRuntimeSDKSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: testSpecificSettingsGetter().ClusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + PostUpgrade: func(proxy framework.ClusterProxy, namespace, clusterName string) { + // This check ensures that the resourceVersions are stable, i.e. it verifies there are no + // continuous reconciles when everything should be stable. + framework.ValidateResourceVersionStable(ctx, proxy, namespace, FilterObjectsWithKindAndName(clusterName)) + }, + // "upgrades-runtimesdk" is the same as the "topology" flavor but with an additional RuntimeExtension. + Flavor: ptr.To(testSpecificSettingsGetter().FlavorForMode("topology-runtimesdk")), + ExtensionServiceNamespace: "capv-test-extension", + ExtensionServiceName: "capv-test-extension-webhook-service", + PostNamespaceCreated: testSpecificSettingsGetter().PostNamespaceCreatedFunc, + } + }) + }) +}) diff --git a/test/e2e/config/vsphere.yaml b/test/e2e/config/vsphere.yaml index e0e22ef1d6..cde483885e 100644 --- a/test/e2e/config/vsphere.yaml +++ b/test/e2e/config/vsphere.yaml @@ -16,6 +16,8 @@ images: loadBehavior: mustLoad - name: gcr.io/k8s-staging-capi-vsphere/cluster-api-net-operator-{ARCH}:dev loadBehavior: mustLoad + - name: gcr.io/k8s-staging-capi-vsphere/cluster-api-test-extension-{ARCH}:dev + loadBehavior: mustLoad - name: gcr.io/k8s-staging-capi-vsphere/extra/vm-operator:v1.8.6-0-gde75746a loadBehavior: tryLoad @@ -149,14 +151,18 @@ providers: - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-pci.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-storage-policy.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-topology.yaml" + - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template-topology-runtimesdk.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/cluster-template.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass-quick-start.yaml" + - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass-quick-start-runtimesdk.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-topology-supervisor.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-supervisor.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-quick-start-supervisor.yaml" + - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-quick-start-runtimesdk-supervisor.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-install-on-bootstrap-supervisor.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-conformance-supervisor.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-ownerrefs-finalizers-supervisor.yaml" + - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-topology-runtimesdk-supervisor.yaml" - sourcePath: "../data/shared/capv/main/metadata.yaml" - name: "{go://sigs.k8s.io/cluster-api-provider-vsphere@v1.10}" # supported release in the v1beta1 series value: "https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/releases/download/{go://sigs.k8s.io/cluster-api-provider-vsphere@v1.10}/infrastructure-components.yaml" @@ -215,6 +221,19 @@ providers: - old: "imagePullPolicy: Always" new: "imagePullPolicy: IfNotPresent" + - name: capv-test-extension + type: RuntimeExtensionProvider # test-extension isn't a provider, but we fake it is so it can be handled by the clusterctl machinery. + versions: + - name: v1.11.99 + # Use manifest from source files + value: ../../../../cluster-api-provider-vsphere/test/infrastructure/test-extension/config/default + contract: v1beta1 + files: + - sourcePath: "../data/shared/capv/main/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + variables: # Ensure all Kubernetes versions used here are covered in patch-vsphere-template.yaml KUBERNETES_VERSION: "v1.30.0" @@ -253,11 +272,13 @@ variables: KUBETEST_CONFIGURATION: "./data/kubetest/conformance.yaml" NODE_DRAIN_TIMEOUT: "60s" CLUSTER_TOPOLOGY: "true" + EXP_RUNTIME_SDK: "true" # These IDs correspond to Tesla T4s, they are the decimal representation of the hex values. DEVICE_ID: 7864 VENDOR_ID: 4318 # CAPV feature flags EXP_NODE_ANTI_AFFINITY: "true" + EXP_MACHINE_SET_PREFLIGHT_CHECKS: "true" CAPI_DIAGNOSTICS_ADDRESS: ":8080" CAPI_INSECURE_DIAGNOSTICS: "true" diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass-runtimesdk/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass-runtimesdk/kustomization.yaml new file mode 100644 index 0000000000..4d603d359e --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass-runtimesdk/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../clusterclass +patches: + - target: + kind: ClusterClass + path: ./patch-test-extension.yaml diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass-runtimesdk/patch-test-extension.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass-runtimesdk/patch-test-extension.yaml new file mode 100644 index 0000000000..a4a5379ef1 --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-govmomi/main/clusterclass-runtimesdk/patch-test-extension.yaml @@ -0,0 +1,10 @@ +- op: replace + path: /spec/patches + value: + - name: test-patch + external: + generateExtension: generate-patches.capv-test-extension + validateExtension: validate-topology.capv-test-extension + discoverVariablesExtension: discover-variables.capv-test-extension +- op: remove + path: /spec/variables diff --git a/test/e2e/data/infrastructure-vsphere-govmomi/main/topology-runtimesdk/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-govmomi/main/topology-runtimesdk/kustomization.yaml new file mode 100644 index 0000000000..0aecb2500d --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-govmomi/main/topology-runtimesdk/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../topology diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-runtimesdk/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-runtimesdk/kustomization.yaml new file mode 100644 index 0000000000..4d603d359e --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-runtimesdk/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ../clusterclass +patches: + - target: + kind: ClusterClass + path: ./patch-test-extension.yaml diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-runtimesdk/patch-test-extension.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-runtimesdk/patch-test-extension.yaml new file mode 100644 index 0000000000..a4a5379ef1 --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-runtimesdk/patch-test-extension.yaml @@ -0,0 +1,10 @@ +- op: replace + path: /spec/patches + value: + - name: test-patch + external: + generateExtension: generate-patches.capv-test-extension + validateExtension: validate-topology.capv-test-extension + discoverVariablesExtension: discover-variables.capv-test-extension +- op: remove + path: /spec/variables diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/main/topology-runtimesdk/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/main/topology-runtimesdk/kustomization.yaml new file mode 100644 index 0000000000..0aecb2500d --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-supervisor/main/topology-runtimesdk/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../topology diff --git a/test/go.mod b/test/go.mod index ad42a71f39..79e51bd20f 100644 --- a/test/go.mod +++ b/test/go.mod @@ -19,6 +19,7 @@ require ( ) require ( + github.com/blang/semver/v4 v4.0.0 github.com/dougm/pretty v0.0.0-20171025230240-2ee9d7453c02 github.com/onsi/ginkgo/v2 v2.19.0 github.com/onsi/gomega v1.33.1 @@ -27,6 +28,7 @@ require ( golang.org/x/crypto v0.24.0 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.30.2 + k8s.io/apiextensions-apiserver v0.30.2 k8s.io/apimachinery v0.30.2 k8s.io/client-go v0.30.2 k8s.io/component-base v0.30.2 @@ -54,7 +56,6 @@ require ( github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver v3.5.1+incompatible // indirect - github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cloudflare/circl v1.3.7 // indirect @@ -73,6 +74,7 @@ require ( github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.4 // indirect @@ -164,7 +166,6 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.30.2 // indirect k8s.io/apiserver v0.30.2 // indirect k8s.io/cluster-bootstrap v0.30.2 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect