Merge pull request #2800 from fabriziopandini/vcsim-e2e-supervisor

🌱 Add e2e test using supervisor mode with vcsim

Makefile

@@ -82,7 +82,6 @@ GINKGO_NODES ?= 1
 GINKGO_TIMEOUT ?= 3h
 E2E_CONF_FILE ?= $(abspath test/e2e/config/vsphere.yaml)
 E2E_CONF_OVERRIDE_FILE ?= $(abspath test/e2e/config/config-overrides.yaml)
-E2E_CAPV_MODE ?= govmomi
 E2E_IPAM_KUBECONFIG ?=
 INTEGRATION_CONF_FILE ?= $(abspath test/integration/integration-dev.yaml)
 E2E_TEMPLATE_DIR := $(abspath test/e2e/data/)
@@ -600,7 +599,6 @@ e2e: $(GINKGO) $(KUSTOMIZE) $(KIND) $(GOVC) ## Run e2e tests
 		--e2e.artifacts-folder="$(ARTIFACTS)" \
 		--e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \
 		--e2e.use-existing-cluster="$(USE_EXISTING_CLUSTER)" \
-		--e2e.capv-mode="$(E2E_CAPV_MODE)" \
 		--e2e.ipam-kubeconfig="$(E2E_IPAM_KUBECONFIG)"
 
 ## --------------------------------------

apis/vmware/v1beta1/vspherecluster_types.go

@@ -30,6 +30,7 @@ const (
 
 // VSphereClusterSpec defines the desired state of VSphereCluster.
 type VSphereClusterSpec struct {
+	// +optional
 	ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"`
 }
 

config/supervisor/crd/vmware.infrastructure.cluster.x-k8s.io_vsphereclusters.yaml

@@ -50,8 +50,6 @@ spec:
                 - host
                 - port
                 type: object
-            required:
-            - controlPlaneEndpoint
             type: object
           status:
             description: VSphereClusterStatus defines the observed state of VSphereClusterSpec.

config/supervisor/crd/vmware.infrastructure.cluster.x-k8s.io_vsphereclustertemplates.yaml

@@ -58,8 +58,6 @@ spec:
                         - host
                         - port
                         type: object
-                    required:
-                    - controlPlaneEndpoint
                     type: object
                 required:
                 - spec

hack/e2e.sh

@@ -68,52 +68,76 @@ function login() {
 AUTH=
 E2E_IMAGE_SHA=
 GCR_KEY_FILE="${GCR_KEY_FILE:-}"
-export VSPHERE_SERVER="${GOVC_URL}"
-export VSPHERE_USERNAME="${GOVC_USERNAME}"
-export VSPHERE_PASSWORD="${GOVC_PASSWORD}"
-export VSPHERE_SSH_AUTHORIZED_KEY="${VM_SSH_PUB_KEY}"
+export VSPHERE_SERVER="${GOVC_URL:-}"
+export VSPHERE_USERNAME="${GOVC_USERNAME:-}"
+export VSPHERE_PASSWORD="${GOVC_PASSWORD:-}"
+export VSPHERE_SSH_AUTHORIZED_KEY="${VM_SSH_PUB_KEY:-}"
 export VSPHERE_SSH_PRIVATE_KEY="/root/ssh/.private-key/private-key"
 export E2E_CONF_FILE="${REPO_ROOT}/test/e2e/config/vsphere.yaml"
 export E2E_CONF_OVERRIDE_FILE=""
-export E2E_CAPV_MODE="${CAPV_MODE:-govmomi}"
+export E2E_VM_OPERATOR_VERSION="${VM_OPERATOR_VERSION:-v1.8.1}"
 export ARTIFACTS="${ARTIFACTS:-${REPO_ROOT}/_artifacts}"
 export DOCKER_IMAGE_TAR="/tmp/images/image.tar"
 export GC_KIND="false"
 
 # Make tests run in-parallel
 export GINKGO_NODES=5
+
 # Set the kubeconfig to the IPAM cluster so the e2e tests can claim ip addresses
 # for kube-vip.
 export E2E_IPAM_KUBECONFIG="/root/ipam-conf/capv-services.conf"
 
-# Run the vpn client in container
-docker run --rm -d --name vpn -v "${HOME}/.openvpn/:${HOME}/.openvpn/" \
-  -w "${HOME}/.openvpn/" --cap-add=NET_ADMIN --net=host --device=/dev/net/tun \
-  gcr.io/k8s-staging-capi-vsphere/extra/openvpn:latest
-
-# Tail the vpn logs
-docker logs vpn
-
-# Wait until the VPN connection is active and we are able to reach the ipam cluster
-function wait_for_ipam_reachable() {
-  local n=0
-  until [ $n -ge 30 ]; do
-    kubectl --kubeconfig="${E2E_IPAM_KUBECONFIG}" --request-timeout=2s  get inclusterippools.ipam.cluster.x-k8s.io && RET=$? || RET=$?
-    if [[ "$RET" -eq 0 ]]; then
-      break
-    fi
-    n=$((n + 1))
-    sleep 1
-  done
-  return "$RET"
-}
-wait_for_ipam_reachable
+# Only run the vpn/check for IPAM when we need them
+re='\[vcsim\]'
+if [[ ! "${GINKGO_FOCUS:-}" =~ $re ]]; then
+  # Run the vpn client in container
+  docker run --rm -d --name vpn -v "${HOME}/.openvpn/:${HOME}/.openvpn/" \
+    -w "${HOME}/.openvpn/" --cap-add=NET_ADMIN --net=host --device=/dev/net/tun \
+    gcr.io/k8s-staging-capi-vsphere/extra/openvpn:latest
+
+  # Tail the vpn logs
+  docker logs vpn
+
+  # Wait until the VPN connection is active and we are able to reach the ipam cluster
+  function wait_for_ipam_reachable() {
+    local n=0
+    until [ $n -ge 30 ]; do
+      kubectl --kubeconfig="${E2E_IPAM_KUBECONFIG}" --request-timeout=2s  get inclusterippools.ipam.cluster.x-k8s.io && RET=$? || RET=$?
+      if [[ "$RET" -eq 0 ]]; then
+        break
+      fi
+      n=$((n + 1))
+      sleep 1
+    done
+    return "$RET"
+  }
+  wait_for_ipam_reachable
+fi
 
 make envsubst
 
+# kind:prepullImage pre-pull a docker image if no already present locally.
+# The result will be available in the retVal value which is accessible from the caller.
+kind::prepullImage () {
+  local image=$1
+  image="${image//+/_}"
+
+  if [[ "$(docker images -q "$image" 2> /dev/null)" == "" ]]; then
+    echo "+ Pulling $image"
+    docker pull "$image"
+  else
+    echo "+ image $image already present in the system, skipping pre-pull"
+  fi
+}
+
+if [[ ${GINKGO_FOCUS:-} =~ \\\[supervisor\\\] ]]; then
+   kind::prepullImage "gcr.io/k8s-staging-capi-vsphere/extra/vm-operator:${E2E_VM_OPERATOR_VERSION}"
+fi
+
 ARCH="$(go env GOARCH)"
 
 # Only build and upload the image if we run tests which require it to save some $.
+# NOTE: the image is required for clusterctl upgrade tests, and those test are run only as part of the main e2e test job (without any focus)
 if [[ -z "${GINKGO_FOCUS+x}" ]]; then
   # Save the docker image locally
   make e2e-images

test/e2e/anti_affinity_test.go

@@ -48,7 +48,7 @@ type AntiAffinitySpecInput struct {
 
 var _ = Describe("Cluster creation with anti affined nodes", func() {
 	const specName = "anti-affinity"
-	Setup(specName, func(testSpecificClusterctlConfigPathGetter func() string) {
+	Setup(specName, func(testSpecificSettingsGetter func() testSettings) {
 		var namespace *corev1.Namespace
 
 		BeforeEach(func() {
@@ -72,7 +72,7 @@ var _ = Describe("Cluster creation with anti affined nodes", func() {
 				},
 				Global: GlobalInput{
 					BootstrapClusterProxy: bootstrapClusterProxy,
-					ClusterctlConfigPath:  testSpecificClusterctlConfigPathGetter(),
+					ClusterctlConfigPath:  testSpecificSettingsGetter().ClusterctlConfigPath,
 					E2EConfig:             e2eConfig,
 					ArtifactFolder:        artifactFolder,
 				},

test/e2e/capi_machine_deployment_rollout_test.go

@@ -19,19 +19,22 @@ package e2e
 import (
 	. "github.com/onsi/ginkgo/v2"
 	capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
+	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
 )
 
 var _ = Describe("ClusterAPI Machine Deployment Tests", func() {
 	const specName = "md-rollout" // copied from CAPI
 	Context("Running the MachineDeployment rollout spec", func() {
-		Setup(specName, func(testSpecificClusterctlConfigPathGetter func() string) {
+		Setup(specName, func(testSpecificSettingsGetter func() testSettings) {
 			capi_e2e.MachineDeploymentRolloutSpec(ctx, func() capi_e2e.MachineDeploymentRolloutSpecInput {
 				return capi_e2e.MachineDeploymentRolloutSpecInput{
 					E2EConfig:             e2eConfig,
-					ClusterctlConfigPath:  testSpecificClusterctlConfigPathGetter(),
+					ClusterctlConfigPath:  testSpecificSettingsGetter().ClusterctlConfigPath,
 					BootstrapClusterProxy: bootstrapClusterProxy,
 					ArtifactFolder:        artifactFolder,
 					SkipCleanup:           skipCleanup,
+					Flavor:                testSpecificSettingsGetter().FlavorForMode(clusterctl.DefaultFlavor),
+					PostNamespaceCreated:  testSpecificSettingsGetter().PostNamespaceCreatedFunc,
 				}
 			})
 		})

test/e2e/cluster_upgrade_test.go

@@ -33,7 +33,7 @@ var _ = Describe("When upgrading a workload cluster using ClusterClass and testi
 	// the resolved versions as env vars. This only works without side effects on other tests because we are
 	// running this test in its separate job.
 	const specName = "k8s-upgrade-and-conformance" // copied from CAPI
-	Setup(specName, func(testSpecificClusterctlConfigPathGetter func() string) {
+	Setup(specName, func(testSpecificSettingsGetter func() testSettings) {
 		capi_e2e.ClusterUpgradeConformanceSpec(ctx, func() capi_e2e.ClusterUpgradeConformanceSpecInput {
 			// The Kubernetes versions have to be resolved as they can be defined like this: stable-1.29, ci/latest-1.30.
 			kubernetesVersionUpgradeFrom, err := kubernetesversions.ResolveVersion(ctx, e2eConfig.GetVariable("KUBERNETES_VERSION_UPGRADE_FROM"))
@@ -44,14 +44,15 @@ var _ = Describe("When upgrading a workload cluster using ClusterClass and testi
 			Expect(os.Setenv("KUBERNETES_VERSION_UPGRADE_TO", kubernetesVersionUpgradeTo)).To(Succeed())
 			return capi_e2e.ClusterUpgradeConformanceSpecInput{
 				E2EConfig:             e2eConfig,
-				ClusterctlConfigPath:  testSpecificClusterctlConfigPathGetter(),
+				ClusterctlConfigPath:  testSpecificSettingsGetter().ClusterctlConfigPath,
 				BootstrapClusterProxy: bootstrapClusterProxy,
 				ArtifactFolder:        artifactFolder,
 				SkipCleanup:           skipCleanup,
 				WorkerMachineCount:    ptr.To[int64](5),
 				// Note: install-on-bootstrap will install Kubernetes on bootstrap if the correct Kubernetes version
 				// cannot be detected. This is required to install versions we don't have images for (e.g. ci/latest-1.30).
-				Flavor: ptr.To("install-on-bootstrap"),
+				Flavor:               ptr.To(testSpecificSettingsGetter().FlavorForMode("install-on-bootstrap")),
+				PostNamespaceCreated: testSpecificSettingsGetter().PostNamespaceCreatedFunc,
 			}
 		})
 	})
@@ -61,15 +62,16 @@ var _ = Describe("When upgrading a workload cluster using ClusterClass [ClusterC
 	// Note: This installs a cluster based on KUBERNETES_VERSION_UPGRADE_FROM and then upgrades to
 	// KUBERNETES_VERSION_UPGRADE_TO.
 	const specName = "k8s-upgrade" // aligned to CAPI
-	Setup(specName, func(testSpecificClusterctlConfigPathGetter func() string) {
+	Setup(specName, func(testSpecificSettingsGetter func() testSettings) {
 		capi_e2e.ClusterUpgradeConformanceSpec(ctx, func() capi_e2e.ClusterUpgradeConformanceSpecInput {
 			return capi_e2e.ClusterUpgradeConformanceSpecInput{
 				E2EConfig:             e2eConfig,
-				ClusterctlConfigPath:  testSpecificClusterctlConfigPathGetter(),
+				ClusterctlConfigPath:  testSpecificSettingsGetter().ClusterctlConfigPath,
 				BootstrapClusterProxy: bootstrapClusterProxy,
 				ArtifactFolder:        artifactFolder,
 				SkipCleanup:           skipCleanup,
-				Flavor:                ptr.To("topology"),
+				Flavor:                ptr.To(testSpecificSettingsGetter().FlavorForMode("topology")),
+				PostNamespaceCreated:  testSpecificSettingsGetter().PostNamespaceCreatedFunc,
 				// This test is run in CI in parallel with other tests. To keep the test duration reasonable
 				// the conformance tests are skipped.
 				ControlPlaneMachineCount: ptr.To[int64](1),

test/e2e/clusterclass_changes_test.go

@@ -23,15 +23,16 @@ import (
 
 var _ = Describe("When testing ClusterClass changes [ClusterClass]", func() {
 	const specName = "clusterclass-changes" // copied from CAPI
-	Setup(specName, func(testSpecificClusterctlConfigPathGetter func() string) {
+	Setup(specName, func(testSpecificSettingsGetter func() testSettings) {
 		capie2e.ClusterClassChangesSpec(ctx, func() capie2e.ClusterClassChangesSpecInput {
 			return capie2e.ClusterClassChangesSpecInput{
 				E2EConfig:             e2eConfig,
-				ClusterctlConfigPath:  testSpecificClusterctlConfigPathGetter(),
+				ClusterctlConfigPath:  testSpecificSettingsGetter().ClusterctlConfigPath,
 				BootstrapClusterProxy: bootstrapClusterProxy,
 				ArtifactFolder:        artifactFolder,
 				SkipCleanup:           skipCleanup,
-				Flavor:                "topology",
+				Flavor:                testSpecificSettingsGetter().FlavorForMode("topology"),
+				PostNamespaceCreated:  testSpecificSettingsGetter().PostNamespaceCreatedFunc,
 				ModifyControlPlaneFields: map[string]interface{}{
 					"spec.machineTemplate.nodeDrainTimeout": "10s",
 				},

test/e2e/clusterctl_upgrade_test.go

@@ -23,15 +23,16 @@ import (
 
 var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.9=>current, CAPI 1.6=>1.6) [ClusterClass]", func() {
 	const specName = "clusterctl-upgrade-1.9-current" // prefix (clusterctl-upgrade) copied from CAPI
-	Setup(specName, func(testSpecificClusterctlConfigPathGetter func() string) {
+	Setup(specName, func(testSpecificSettingsGetter func() testSettings) {
 		capi_e2e.ClusterctlUpgradeSpec(ctx, func() capi_e2e.ClusterctlUpgradeSpecInput {
 			return capi_e2e.ClusterctlUpgradeSpecInput{
 				E2EConfig:                         e2eConfig,
-				ClusterctlConfigPath:              testSpecificClusterctlConfigPathGetter(),
+				ClusterctlConfigPath:              testSpecificSettingsGetter().ClusterctlConfigPath,
 				BootstrapClusterProxy:             bootstrapClusterProxy,
 				ArtifactFolder:                    artifactFolder,
 				SkipCleanup:                       skipCleanup,
-				MgmtFlavor:                        "remote-management",
+				MgmtFlavor:                        testSpecificSettingsGetter().FlavorForMode("remote-management"),
+				PostNamespaceCreated:              testSpecificSettingsGetter().PostNamespaceCreatedFunc,
 				InitWithBinary:                    "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.6.1/clusterctl-{OS}-{ARCH}",
 				InitWithCoreProvider:              "cluster-api:v1.6.1",
 				InitWithBootstrapProviders:        []string{"kubeadm:v1.6.1"},
@@ -44,23 +45,24 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.9=
 				// Ensure all Kubernetes versions used here are covered in patch-vsphere-template.yaml
 				InitWithKubernetesVersion: "v1.29.0",
 				WorkloadKubernetesVersion: "v1.29.0",
-				WorkloadFlavor:            "workload",
+				WorkloadFlavor:            testSpecificSettingsGetter().FlavorForMode("workload"),
 			}
 		})
 	}, WithIP("WORKLOAD_CONTROL_PLANE_ENDPOINT_IP"))
 })
 
 var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.8=>current, CAPI 1.5=>1.6) [ClusterClass]", func() {
 	const specName = "clusterctl-upgrade-1.8-current" // prefix (clusterctl-upgrade) copied from CAPI
-	Setup(specName, func(testSpecificClusterctlConfigPathGetter func() string) {
+	Setup(specName, func(testSpecificSettingsGetter func() testSettings) {
 		capi_e2e.ClusterctlUpgradeSpec(ctx, func() capi_e2e.ClusterctlUpgradeSpecInput {
 			return capi_e2e.ClusterctlUpgradeSpecInput{
 				E2EConfig:                         e2eConfig,
-				ClusterctlConfigPath:              testSpecificClusterctlConfigPathGetter(),
+				ClusterctlConfigPath:              testSpecificSettingsGetter().ClusterctlConfigPath,
 				BootstrapClusterProxy:             bootstrapClusterProxy,
 				ArtifactFolder:                    artifactFolder,
 				SkipCleanup:                       skipCleanup,
-				MgmtFlavor:                        "remote-management",
+				MgmtFlavor:                        testSpecificSettingsGetter().FlavorForMode("remote-management"),
+				PostNamespaceCreated:              testSpecificSettingsGetter().PostNamespaceCreatedFunc,
 				InitWithBinary:                    "https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.5.4/clusterctl-{OS}-{ARCH}",
 				InitWithCoreProvider:              "cluster-api:v1.5.4",
 				InitWithBootstrapProviders:        []string{"kubeadm:v1.5.4"},
@@ -73,7 +75,7 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.8=
 				// Ensure all Kubernetes versions used here are covered in patch-vsphere-template.yaml
 				InitWithKubernetesVersion: "v1.28.0",
 				WorkloadKubernetesVersion: "v1.28.0",
-				WorkloadFlavor:            "workload",
+				WorkloadFlavor:            testSpecificSettingsGetter().FlavorForMode("workload"),
 			}
 		})
 	}, WithIP("WORKLOAD_CONTROL_PLANE_ENDPOINT_IP"))

test/e2e/config/vsphere.yaml

@@ -20,6 +20,8 @@ images:
     loadBehavior: mustLoad
   - name: gcr.io/k8s-staging-capi-vsphere/cluster-api-vcsim-controller-{ARCH}:dev
     loadBehavior: mustLoad
+  - name: gcr.io/k8s-staging-capi-vsphere/extra/vm-operator:v1.8.1
+    loadBehavior: tryLoad
   - name: quay.io/jetstack/cert-manager-cainjector:v1.12.2
     loadBehavior: tryLoad
   - name: quay.io/jetstack/cert-manager-webhook:v1.12.2
@@ -166,7 +168,7 @@ providers:
           - sourcePath: "../data/shared/v1.8/v1beta1_provider/metadata.yaml"
 
   - name: vcsim
-    type: InfrastructureProvider
+    type: RuntimeExtensionProvider # vcsim isn't a provider, but we fake it is so it can be handled by the clusterctl machinery.
     versions:
       - name: v1.10.99
         # Use manifest from source files
@@ -176,6 +178,20 @@ providers:
           # Add cluster templates
           - sourcePath: "../data/shared/main/v1beta1_provider/metadata.yaml"
 
+  - name: vm-operator
+    type: RuntimeExtensionProvider # vm-operator isn't a provider, but we fake it is so it can be handled by the clusterctl machinery.
+    versions:
+      - name: v1.8.1
+        # Use manifest from source files
+        value: "https://storage.googleapis.com/artifacts.k8s-staging-capi-vsphere.appspot.com/vm-operator/v1.8.1.yaml"
+        type: "url"
+        contract: v1beta1
+        files:
+          - sourcePath: "../data/shared/main/v1beta1_operator/metadata.yaml"
+        replacements:
+          - old: "imagePullPolicy: Always"
+            new: "imagePullPolicy: IfNotPresent"
+
 variables:
   # Ensure all Kubernetes versions used here are covered in patch-vsphere-template.yaml
   KUBERNETES_VERSION: "v1.29.0"
@@ -210,6 +226,8 @@ variables:
   EXP_NODE_ANTI_AFFINITY: "true"
   CAPI_DIAGNOSTICS_ADDRESS: ":8080"
   CAPI_INSECURE_DIAGNOSTICS: "true"
+  SERVICE_ACCOUNTS_CM_NAME: ""
+  SERVICE_ACCOUNTS_CM_NAMESPACE: ""
 
 intervals:
   default/wait-controllers: ["5m", "10s"]

test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass/patch-vsphere-template.yaml

@@ -4,7 +4,7 @@
     definitions:
     - jsonPatches:
       - op: replace
-        path: /spec/template/spec/template
+        path: /spec/template/spec/imageName
         valueFrom:
           # We have to fall back to v1.29.0 for the conformance latest ci test which uses
           # versions without corresponding templates like "v1.30.0-alpha.0.525+09a5049ca78502".
@@ -17,13 +17,13 @@
             ubuntu-2204-kube-v1.29.0
             {{- end -}}{{- end -}}
       selector:
-        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+        apiVersion: vmware.infrastructure.cluster.x-k8s.io/v1beta1
         kind: VSphereMachineTemplate
         matchResources:
           controlPlane: true
     - jsonPatches:
       - op: replace
-        path: /spec/template/spec/template
+        path: /spec/template/spec/imageName
         valueFrom:
           # We have to fall back to v1.29.0 for the conformance latest ci test which uses
           # versions without corresponding templates like "v1.30.0-alpha.0.525+09a5049ca78502".
@@ -36,7 +36,7 @@
             ubuntu-2204-kube-v1.29.0
             {{- end -}}{{- end -}}
       selector:
-        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+        apiVersion: vmware.infrastructure.cluster.x-k8s.io/v1beta1
         kind: VSphereMachineTemplate
         matchResources:
           machineDeploymentClass: