Merge pull request #2757 from chrischdi/ipam-e2e-test

🌱 test/e2e: implement e2e test for IPAM

Makefile

@@ -359,6 +359,8 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai
 	# for DHCP overrides
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/dhcp-overrides" > "$(E2E_TEMPLATE_DIR)/main/cluster-template-dhcp-overrides.yaml"
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/ownerrefs-finalizers" > "$(E2E_TEMPLATE_DIR)/main/cluster-template-ownerrefs-finalizers.yaml"
+	# for IPAM tests
+	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/ipam" > "$(E2E_TEMPLATE_DIR)/main/cluster-template-ipam.yaml"
 
 .PHONY: generate-e2e-templates-v1.9
 generate-e2e-templates-v1.9: $(KUSTOMIZE)

test/e2e/clusterctl_upgrade_test.go

@@ -38,6 +38,7 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.9=
 				InitWithControlPlaneProviders:     []string{"kubeadm:v1.6.1"},
 				InitWithInfrastructureProviders:   []string{"vsphere:v1.9.0"},
 				InitWithRuntimeExtensionProviders: []string{},
+				InitWithIPAMProviders:             []string{},
 				// InitWithKubernetesVersion should be the highest kubernetes version supported by the init Cluster API version.
 				// This is to guarantee that both, the old and new CAPI version, support the defined version.
 				// Ensure all Kubernetes versions used here are covered in patch-vsphere-template.yaml
@@ -66,6 +67,7 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.8=
 				InitWithControlPlaneProviders:     []string{"kubeadm:v1.5.4"},
 				InitWithInfrastructureProviders:   []string{"vsphere:v1.8.4"},
 				InitWithRuntimeExtensionProviders: []string{},
+				InitWithIPAMProviders:             []string{},
 				// InitWithKubernetesVersion should be the highest kubernetes version supported by the init Cluster API version.
 				// This is to guarantee that both, the old and new CAPI version, support the defined version.
 				// Ensure all Kubernetes versions used here are covered in patch-vsphere-template.yaml

test/e2e/config/vsphere.yaml

@@ -14,6 +14,8 @@ images:
     loadBehavior: tryLoad
   - name: registry.k8s.io/cluster-api/kubeadm-control-plane-controller:v1.6.1
     loadBehavior: tryLoad
+  - name: registry.k8s.io/capi-ipam-ic/cluster-api-ipam-in-cluster-controller:v0.1.0
+    loadBehavior: tryLoad
   - name: gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller-{ARCH}:dev
     loadBehavior: mustLoad
   - name: quay.io/jetstack/cert-manager-cainjector:v1.12.2
@@ -97,6 +99,20 @@ providers:
           - old: "imagePullPolicy: Always"
             new: "imagePullPolicy: IfNotPresent"
 
+  - name: in-cluster
+    type: IPAMProvider
+    versions:
+      - name: v0.1.0
+        # Use manifest from source files
+        value: "https://github.com/kubernetes-sigs/cluster-api-ipam-provider-in-cluster/releases/download/v0.1.0/ipam-components.yaml"
+        type: "url"
+        contract: v1beta1
+        files:
+          - sourcePath: "../data/shared/v1.9/v1beta1_ipam/metadata.yaml"
+        replacements:
+          - old: "imagePullPolicy: Always"
+            new: "imagePullPolicy: IfNotPresent"
+
   - name: vsphere
     type: InfrastructureProvider
     versions:
@@ -107,10 +123,11 @@ providers:
         files:
           # Add a cluster template
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-conformance.yaml"
-          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-install-on-bootstrap.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-dhcp-overrides.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-hw-upgrade.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-ignition.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-install-on-bootstrap.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-ipam.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-kcp-remediation.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-md-remediation.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-node-drain.yaml"

test/e2e/data/infrastructure-vsphere/main/ipam/ipam-patch.yaml

@@ -0,0 +1,37 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: VSphereMachineTemplate
+metadata:
+  name: '${CLUSTER_NAME}'
+  namespace: '${NAMESPACE}'
+spec:
+  template:
+    spec:
+      network:
+        devices:
+        - addressesFromPools:
+          - apiGroup: ipam.cluster.x-k8s.io
+            kind: InClusterIPPool
+            name: ${CLUSTER_NAME}
+          nameservers:
+          # Note: using 8.8.8.8 may require to set an IP as VSPHERE_SERVER
+          - ${IPAM_NAMESERVER:-8.8.8.8}
+          networkName: '${VSPHERE_NETWORK}'
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: VSphereMachineTemplate
+metadata:
+  name: '${CLUSTER_NAME}-worker'
+  namespace: '${NAMESPACE}'
+spec:
+  template:
+    spec:
+      network:
+        devices:
+        - addressesFromPools:
+          - apiGroup: ipam.cluster.x-k8s.io
+            kind: InClusterIPPool
+            name: ${CLUSTER_NAME}
+          nameservers:
+          # Note: using 8.8.8.8 may require to set an IP as VSPHERE_SERVER
+          - ${IPAM_NAMESERVER:-8.8.8.8}
+          networkName: '${VSPHERE_NETWORK}'

test/e2e/data/infrastructure-vsphere/main/ipam/ippool.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: ipam.cluster.x-k8s.io/v1alpha2
+kind: InClusterIPPool
+metadata:
+  name: ${CLUSTER_NAME}
+  namespace: ${NAMESPACE}
+spec:
+  prefix: ${IPAM_PREFIX:-24}
+  gateway: ${IPAM_GATEWAY}
+  addresses:
+  - ${IPAM_IP_1}
+  - ${IPAM_IP_2}

test/e2e/data/infrastructure-vsphere/main/ipam/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../base
+  - ippool.yaml
+patchesStrategicMerge:
+  - ipam-patch.yaml

test/e2e/data/shared/v1.9/v1beta1_ipam/metadata.yaml

@@ -0,0 +1,12 @@
+# maps release series of major.minor to cluster-api contract version
+# the contract version may change between minor or major versions, but *not*
+# between patch versions.
+#
+# update this file only when a new major or minor version is released
+apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3
+kind: Metadata
+releaseSeries:
+  - major: 0
+    minor: 1
+    contract: v1beta1
+

test/e2e/e2e_setup_test.go

@@ -27,10 +27,13 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	. "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions"
 	"sigs.k8s.io/yaml"
+
+	"sigs.k8s.io/cluster-api-provider-vsphere/test/framework/ip"
 )
 
 type setupOptions struct {
 	additionalIPVariableNames []string
+	gatewayIPVariableName     string
 }
 
 // SetupOption is a configuration option supplied to Setup.
@@ -44,6 +47,13 @@ func WithIP(variableName string) SetupOption {
 	}
 }
 
+// WithGateway instructs Setup to store the Gateway IP from IPAM into the provided variableName.
+func WithGateway(variableName string) SetupOption {
+	return func(o *setupOptions) {
+		o.gatewayIPVariableName = variableName
+	}
+}
+
 // Setup for the specific test.
 func Setup(specName string, f func(testSpecificClusterctlConfigPathGetter func() string), opts ...SetupOption) {
 	options := &setupOptions{}
@@ -60,7 +70,7 @@ func Setup(specName string, f func(testSpecificClusterctlConfigPathGetter func()
 		Byf("Setting up test env for %s", specName)
 
 		Byf("Getting IP for %s", strings.Join(append([]string{"CONTROL_PLANE_ENDPOINT_IP"}, options.additionalIPVariableNames...), ","))
-		testSpecificIPAddressClaims, testSpecificVariables = ipAddressManager.ClaimIPs(ctx, options.additionalIPVariableNames...)
+		testSpecificIPAddressClaims, testSpecificVariables = ipAddressManager.ClaimIPs(ctx, ip.WithGateway(options.gatewayIPVariableName), ip.WithIP(options.additionalIPVariableNames...))
 
 		// Create a new clusterctl config file based on the passed file and add the new variables for the IPs.
 		testSpecificClusterctlConfigPath = fmt.Sprintf("%s-%s.yaml", strings.TrimSuffix(clusterctlConfigPath, ".yaml"), specName)

test/e2e/ipam_test.go

@@ -0,0 +1,48 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	"k8s.io/utils/ptr"
+	capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
+)
+
+var _ = Describe("ClusterClass Creation using Cluster API quick-start test and IPAM Provider [ClusterClass]", func() {
+	const specName = "ipam-cluster-class"
+	Setup(specName, func(testSpecificClusterctlConfigPathGetter func() string) {
+		capi_e2e.QuickStartSpec(ctx, func() capi_e2e.QuickStartSpecInput {
+			return capi_e2e.QuickStartSpecInput{
+				E2EConfig:                e2eConfig,
+				ClusterctlConfigPath:     testSpecificClusterctlConfigPathGetter(),
+				BootstrapClusterProxy:    bootstrapClusterProxy,
+				ArtifactFolder:           artifactFolder,
+				SkipCleanup:              skipCleanup,
+				Flavor:                   ptr.To("ipam"),
+				ControlPlaneMachineCount: ptr.To[int64](1),
+				WorkerMachineCount:       ptr.To[int64](1),
+			}
+		})
+	},
+		// Set the WithGateway option to write the gateway ip address to the variable.
+		// This variable is required for creating the InClusterIPPool for the ipam provider.
+		WithGateway("IPAM_GATEWAY"),
+		// Claim two IPs from the CI's IPAM provider to use in the InClusterIPPool of
+		// the ipam provider. The IPs then get claimed during provisioning to configure
+		// static IPs for the control-plane and worker node.
+		WithIP("IPAM_IP_1"), WithIP("IPAM_IP_2"))
+})

test/framework/framework.go

@@ -93,6 +93,7 @@ func InitBootstrapCluster(ctx context.Context, bootstrapClusterProxy framework.C
 		ClusterctlConfigPath:    clusterctlConfig,
 		InfrastructureProviders: config.InfrastructureProviders(),
 		LogFolder:               filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()),
+		IPAMProviders:           config.IPAMProviders(),
 	}, config.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...)
 }
 

test/framework/ip/addressmanager.go

@@ -30,7 +30,7 @@ type AddressManager interface {
 	// ClaimIPs claims IP addresses with the variable name `CONTROL_PLANE_ENDPOINT_IP` and whatever is passed as
 	// additionalIPVariableNames.
 	// It returns a slice of IPAddressClaims namespaced names and corresponding variables.
-	ClaimIPs(ctx context.Context, additionalIPVariableNames ...string) (claims AddressClaims, variables map[string]string)
+	ClaimIPs(ctx context.Context, opts ...ClaimOption) (claims AddressClaims, variables map[string]string)
 
 	// Cleanup deletes the given IPAddressClaims.
 	Cleanup(ctx context.Context, claims AddressClaims) error
@@ -39,3 +39,25 @@ type AddressManager interface {
 	// It identifies IPAddressClaims via labels.
 	Teardown(ctx context.Context, folderName string, vSphereClient *govmomi.Client) error
 }
+
+type claimOptions struct {
+	additionalIPVariableNames []string
+	gatewayIPVariableName     string
+}
+
+type ClaimOption func(*claimOptions)
+
+// WithIP instructs Setup to allocate another IP and store it into the provided variableName
+// NOTE: Setup always allocate an IP for CONTROL_PLANE_ENDPOINT_IP.
+func WithIP(variableName ...string) ClaimOption {
+	return func(o *claimOptions) {
+		o.additionalIPVariableNames = append(o.additionalIPVariableNames, variableName...)
+	}
+}
+
+// WithGateway instructs Setup to store the Gateway IP from IPAM into the provided variableName.
+func WithGateway(variableName string) ClaimOption {
+	return func(o *claimOptions) {
+		o.gatewayIPVariableName = variableName
+	}
+}

test/framework/ip/incluster.go

@@ -84,21 +84,33 @@ func InClusterAddressManager(e2eIPAMKubeconfig string, labels map[string]string,
 	}, nil
 }
 
-func (h *inCluster) ClaimIPs(ctx context.Context, additionalIPVariableNames ...string) (AddressClaims, map[string]string) {
+func (h *inCluster) ClaimIPs(ctx context.Context, opts ...ClaimOption) (AddressClaims, map[string]string) {
+	options := &claimOptions{}
+	for _, o := range opts {
+		o(options)
+	}
+
 	variables := map[string]string{}
 
 	ipAddressClaims := AddressClaims{}
 
 	// Claim an IP per variable.
-	for _, variable := range append(additionalIPVariableNames, controlPlaneEndpointVariable) {
+	for _, variable := range append(options.additionalIPVariableNames, controlPlaneEndpointVariable) {
 		ip, ipAddressClaim, err := h.claimIPAddress(ctx)
 		Expect(err).ToNot(HaveOccurred())
 		ipAddressClaims = append(ipAddressClaims, types.NamespacedName{
 			Namespace: ipAddressClaim.Namespace,
 			Name:      ipAddressClaim.Name,
 		})
-		Byf("Setting clusterctl variable %s to %s", variable, ip)
-		variables[variable] = ip
+		Byf("Setting clusterctl variable %s to %s", variable, ip.Spec.Address)
+		variables[variable] = ip.Spec.Address
+		if variable == controlPlaneEndpointVariable && options.gatewayIPVariableName != "" {
+			// Set the gateway variable if requested to the gateway of the control plane IP.
+			// This is required in ipam scenarios, otherwise the VMs will not be able to
+			// connect to the public internet to pull images.
+			Byf("Setting clusterctl variable %s to %s", variable, ip.Spec.Gateway)
+			variables[options.gatewayIPVariableName] = ip.Spec.Gateway
+		}
 	}
 
 	return ipAddressClaims, variables
@@ -271,7 +283,7 @@ func getVirtualMachineIPAddresses(ctx context.Context, folderName string, vSpher
 	return virtualMachineIPAddresses, nil
 }
 
-func (h *inCluster) claimIPAddress(ctx context.Context) (_ string, _ *ipamv1.IPAddressClaim, err error) {
+func (h *inCluster) claimIPAddress(ctx context.Context) (_ *ipamv1.IPAddress, _ *ipamv1.IPAddressClaim, err error) {
 	claim := &ipamv1.IPAddressClaim{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        "ipclaim-" + rand.String(32),
@@ -295,7 +307,7 @@ func (h *inCluster) claimIPAddress(ctx context.Context) (_ string, _ *ipamv1.IPA
 	// Create an IPAddressClaim
 	Byf("Creating IPAddressClaim %s", klog.KObj(claim))
 	if err := h.client.Create(ctx, claim); err != nil {
-		return "", nil, err
+		return nil, nil, err
 	}
 	// Store claim inside the service so the cleanup function knows what to delete.
 	ip := &ipamv1.IPAddress{}
@@ -328,8 +340,8 @@ func (h *inCluster) claimIPAddress(ctx context.Context) (_ string, _ *ipamv1.IPA
 	if retryError != nil {
 		// Try best effort deletion of the unused claim before returning an error.
 		_ = h.client.Delete(ctx, claim)
-		return "", nil, retryError
+		return nil, nil, retryError
 	}
 
-	return ip.Spec.Address, claim, nil
+	return ip, claim, nil
 }

test/framework/ip/noop.go

@@ -27,7 +27,7 @@ var _ AddressManager = &noop{}
 
 type noop struct{}
 
-func (h *noop) ClaimIPs(_ context.Context, _ ...string) (AddressClaims, map[string]string) {
+func (h *noop) ClaimIPs(_ context.Context, _ ...ClaimOption) (AddressClaims, map[string]string) {
 	return nil, nil
 }