diff --git a/Makefile b/Makefile index 81ef9f0769..44d7f571aa 100644 --- a/Makefile +++ b/Makefile @@ -359,6 +359,8 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai # for DHCP overrides "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/dhcp-overrides" > "$(E2E_TEMPLATE_DIR)/main/cluster-template-dhcp-overrides.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/ownerrefs-finalizers" > "$(E2E_TEMPLATE_DIR)/main/cluster-template-ownerrefs-finalizers.yaml" + # for IPAM tests + "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/ipam" > "$(E2E_TEMPLATE_DIR)/main/cluster-template-ipam.yaml" .PHONY: generate-e2e-templates-v1.9 generate-e2e-templates-v1.9: $(KUSTOMIZE) diff --git a/test/e2e/clusterctl_upgrade_test.go b/test/e2e/clusterctl_upgrade_test.go index 20d3ceb0fa..aaf6f92064 100644 --- a/test/e2e/clusterctl_upgrade_test.go +++ b/test/e2e/clusterctl_upgrade_test.go @@ -38,6 +38,7 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.9= InitWithControlPlaneProviders: []string{"kubeadm:v1.6.1"}, InitWithInfrastructureProviders: []string{"vsphere:v1.9.0"}, InitWithRuntimeExtensionProviders: []string{}, + InitWithIPAMProviders: []string{}, // InitWithKubernetesVersion should be the highest kubernetes version supported by the init Cluster API version. // This is to guarantee that both, the old and new CAPI version, support the defined version. // Ensure all Kubernetes versions used here are covered in patch-vsphere-template.yaml @@ -66,6 +67,7 @@ var _ = Describe("When testing clusterctl upgrades using ClusterClass (CAPV 1.8= InitWithControlPlaneProviders: []string{"kubeadm:v1.5.4"}, InitWithInfrastructureProviders: []string{"vsphere:v1.8.4"}, InitWithRuntimeExtensionProviders: []string{}, + InitWithIPAMProviders: []string{}, // InitWithKubernetesVersion should be the highest kubernetes version supported by the init Cluster API version. // This is to guarantee that both, the old and new CAPI version, support the defined version. // Ensure all Kubernetes versions used here are covered in patch-vsphere-template.yaml diff --git a/test/e2e/config/vsphere.yaml b/test/e2e/config/vsphere.yaml index a4aa6ff06c..6eb994d42b 100644 --- a/test/e2e/config/vsphere.yaml +++ b/test/e2e/config/vsphere.yaml @@ -14,6 +14,8 @@ images: loadBehavior: tryLoad - name: registry.k8s.io/cluster-api/kubeadm-control-plane-controller:v1.6.1 loadBehavior: tryLoad + - name: registry.k8s.io/capi-ipam-ic/cluster-api-ipam-in-cluster-controller:v0.1.0 + loadBehavior: tryLoad - name: gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller-{ARCH}:dev loadBehavior: mustLoad - name: quay.io/jetstack/cert-manager-cainjector:v1.12.2 @@ -97,6 +99,20 @@ providers: - old: "imagePullPolicy: Always" new: "imagePullPolicy: IfNotPresent" + - name: in-cluster + type: IPAMProvider + versions: + - name: v0.1.0 + # Use manifest from source files + value: "https://github.com/kubernetes-sigs/cluster-api-ipam-provider-in-cluster/releases/download/v0.1.0/ipam-components.yaml" + type: "url" + contract: v1beta1 + files: + - sourcePath: "../data/shared/v1.9/v1beta1_ipam/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - name: vsphere type: InfrastructureProvider versions: @@ -107,10 +123,11 @@ providers: files: # Add a cluster template - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-conformance.yaml" - - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-install-on-bootstrap.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-dhcp-overrides.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-hw-upgrade.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-ignition.yaml" + - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-install-on-bootstrap.yaml" + - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-ipam.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-kcp-remediation.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-md-remediation.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-node-drain.yaml" diff --git a/test/e2e/data/infrastructure-vsphere/main/ipam/ipam-patch.yaml b/test/e2e/data/infrastructure-vsphere/main/ipam/ipam-patch.yaml new file mode 100644 index 0000000000..f5837082fc --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere/main/ipam/ipam-patch.yaml @@ -0,0 +1,37 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: VSphereMachineTemplate +metadata: + name: '${CLUSTER_NAME}' + namespace: '${NAMESPACE}' +spec: + template: + spec: + network: + devices: + - addressesFromPools: + - apiGroup: ipam.cluster.x-k8s.io + kind: InClusterIPPool + name: ${CLUSTER_NAME} + nameservers: + # Note: using 8.8.8.8 may require to set an IP as VSPHERE_SERVER + - ${IPAM_NAMESERVER:-8.8.8.8} + networkName: '${VSPHERE_NETWORK}' +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: VSphereMachineTemplate +metadata: + name: '${CLUSTER_NAME}-worker' + namespace: '${NAMESPACE}' +spec: + template: + spec: + network: + devices: + - addressesFromPools: + - apiGroup: ipam.cluster.x-k8s.io + kind: InClusterIPPool + name: ${CLUSTER_NAME} + nameservers: + # Note: using 8.8.8.8 may require to set an IP as VSPHERE_SERVER + - ${IPAM_NAMESERVER:-8.8.8.8} + networkName: '${VSPHERE_NETWORK}' \ No newline at end of file diff --git a/test/e2e/data/infrastructure-vsphere/main/ipam/ippool.yaml b/test/e2e/data/infrastructure-vsphere/main/ipam/ippool.yaml new file mode 100644 index 0000000000..5d50d2acbe --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere/main/ipam/ippool.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: ipam.cluster.x-k8s.io/v1alpha2 +kind: InClusterIPPool +metadata: + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +spec: + prefix: ${IPAM_PREFIX:-24} + gateway: ${IPAM_GATEWAY} + addresses: + - ${IPAM_IP_1} + - ${IPAM_IP_2} diff --git a/test/e2e/data/infrastructure-vsphere/main/ipam/kustomization.yaml b/test/e2e/data/infrastructure-vsphere/main/ipam/kustomization.yaml new file mode 100644 index 0000000000..6baa21e4b6 --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere/main/ipam/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../base + - ippool.yaml +patchesStrategicMerge: + - ipam-patch.yaml diff --git a/test/e2e/data/shared/v1.9/v1beta1_ipam/metadata.yaml b/test/e2e/data/shared/v1.9/v1beta1_ipam/metadata.yaml new file mode 100644 index 0000000000..56d61ac6ca --- /dev/null +++ b/test/e2e/data/shared/v1.9/v1beta1_ipam/metadata.yaml @@ -0,0 +1,12 @@ +# maps release series of major.minor to cluster-api contract version +# the contract version may change between minor or major versions, but *not* +# between patch versions. +# +# update this file only when a new major or minor version is released +apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 +kind: Metadata +releaseSeries: + - major: 0 + minor: 1 + contract: v1beta1 + diff --git a/test/e2e/e2e_setup_test.go b/test/e2e/e2e_setup_test.go index 16cd36f6b6..6aadc53c67 100644 --- a/test/e2e/e2e_setup_test.go +++ b/test/e2e/e2e_setup_test.go @@ -27,10 +27,13 @@ import ( "k8s.io/apimachinery/pkg/types" . "sigs.k8s.io/cluster-api/test/framework/ginkgoextensions" "sigs.k8s.io/yaml" + + "sigs.k8s.io/cluster-api-provider-vsphere/test/framework/ip" ) type setupOptions struct { additionalIPVariableNames []string + gatewayIPVariableName string } // SetupOption is a configuration option supplied to Setup. @@ -44,6 +47,13 @@ func WithIP(variableName string) SetupOption { } } +// WithGateway instructs Setup to store the Gateway IP from IPAM into the provided variableName. +func WithGateway(variableName string) SetupOption { + return func(o *setupOptions) { + o.gatewayIPVariableName = variableName + } +} + // Setup for the specific test. func Setup(specName string, f func(testSpecificClusterctlConfigPathGetter func() string), opts ...SetupOption) { options := &setupOptions{} @@ -60,7 +70,7 @@ func Setup(specName string, f func(testSpecificClusterctlConfigPathGetter func() Byf("Setting up test env for %s", specName) Byf("Getting IP for %s", strings.Join(append([]string{"CONTROL_PLANE_ENDPOINT_IP"}, options.additionalIPVariableNames...), ",")) - testSpecificIPAddressClaims, testSpecificVariables = ipAddressManager.ClaimIPs(ctx, options.additionalIPVariableNames...) + testSpecificIPAddressClaims, testSpecificVariables = ipAddressManager.ClaimIPs(ctx, ip.WithGateway(options.gatewayIPVariableName), ip.WithIP(options.additionalIPVariableNames...)) // Create a new clusterctl config file based on the passed file and add the new variables for the IPs. testSpecificClusterctlConfigPath = fmt.Sprintf("%s-%s.yaml", strings.TrimSuffix(clusterctlConfigPath, ".yaml"), specName) diff --git a/test/e2e/ipam_test.go b/test/e2e/ipam_test.go new file mode 100644 index 0000000000..440b8f1e28 --- /dev/null +++ b/test/e2e/ipam_test.go @@ -0,0 +1,48 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + . "github.com/onsi/ginkgo/v2" + "k8s.io/utils/ptr" + capi_e2e "sigs.k8s.io/cluster-api/test/e2e" +) + +var _ = Describe("ClusterClass Creation using Cluster API quick-start test and IPAM Provider [ClusterClass]", func() { + const specName = "ipam-cluster-class" + Setup(specName, func(testSpecificClusterctlConfigPathGetter func() string) { + capi_e2e.QuickStartSpec(ctx, func() capi_e2e.QuickStartSpecInput { + return capi_e2e.QuickStartSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: testSpecificClusterctlConfigPathGetter(), + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + Flavor: ptr.To("ipam"), + ControlPlaneMachineCount: ptr.To[int64](1), + WorkerMachineCount: ptr.To[int64](1), + } + }) + }, + // Set the WithGateway option to write the gateway ip address to the variable. + // This variable is required for creating the InClusterIPPool for the ipam provider. + WithGateway("IPAM_GATEWAY"), + // Claim two IPs from the CI's IPAM provider to use in the InClusterIPPool of + // the ipam provider. The IPs then get claimed during provisioning to configure + // static IPs for the control-plane and worker node. + WithIP("IPAM_IP_1"), WithIP("IPAM_IP_2")) +}) diff --git a/test/framework/framework.go b/test/framework/framework.go index d02cfdfdad..b248b46fa6 100644 --- a/test/framework/framework.go +++ b/test/framework/framework.go @@ -93,6 +93,7 @@ func InitBootstrapCluster(ctx context.Context, bootstrapClusterProxy framework.C ClusterctlConfigPath: clusterctlConfig, InfrastructureProviders: config.InfrastructureProviders(), LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), + IPAMProviders: config.IPAMProviders(), }, config.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...) } diff --git a/test/framework/ip/addressmanager.go b/test/framework/ip/addressmanager.go index b59606bedc..3d4b908ffb 100644 --- a/test/framework/ip/addressmanager.go +++ b/test/framework/ip/addressmanager.go @@ -30,7 +30,7 @@ type AddressManager interface { // ClaimIPs claims IP addresses with the variable name `CONTROL_PLANE_ENDPOINT_IP` and whatever is passed as // additionalIPVariableNames. // It returns a slice of IPAddressClaims namespaced names and corresponding variables. - ClaimIPs(ctx context.Context, additionalIPVariableNames ...string) (claims AddressClaims, variables map[string]string) + ClaimIPs(ctx context.Context, opts ...ClaimOption) (claims AddressClaims, variables map[string]string) // Cleanup deletes the given IPAddressClaims. Cleanup(ctx context.Context, claims AddressClaims) error @@ -39,3 +39,25 @@ type AddressManager interface { // It identifies IPAddressClaims via labels. Teardown(ctx context.Context, folderName string, vSphereClient *govmomi.Client) error } + +type claimOptions struct { + additionalIPVariableNames []string + gatewayIPVariableName string +} + +type ClaimOption func(*claimOptions) + +// WithIP instructs Setup to allocate another IP and store it into the provided variableName +// NOTE: Setup always allocate an IP for CONTROL_PLANE_ENDPOINT_IP. +func WithIP(variableName ...string) ClaimOption { + return func(o *claimOptions) { + o.additionalIPVariableNames = append(o.additionalIPVariableNames, variableName...) + } +} + +// WithGateway instructs Setup to store the Gateway IP from IPAM into the provided variableName. +func WithGateway(variableName string) ClaimOption { + return func(o *claimOptions) { + o.gatewayIPVariableName = variableName + } +} diff --git a/test/framework/ip/incluster.go b/test/framework/ip/incluster.go index 4ec9338273..da3339269a 100644 --- a/test/framework/ip/incluster.go +++ b/test/framework/ip/incluster.go @@ -84,21 +84,33 @@ func InClusterAddressManager(e2eIPAMKubeconfig string, labels map[string]string, }, nil } -func (h *inCluster) ClaimIPs(ctx context.Context, additionalIPVariableNames ...string) (AddressClaims, map[string]string) { +func (h *inCluster) ClaimIPs(ctx context.Context, opts ...ClaimOption) (AddressClaims, map[string]string) { + options := &claimOptions{} + for _, o := range opts { + o(options) + } + variables := map[string]string{} ipAddressClaims := AddressClaims{} // Claim an IP per variable. - for _, variable := range append(additionalIPVariableNames, controlPlaneEndpointVariable) { + for _, variable := range append(options.additionalIPVariableNames, controlPlaneEndpointVariable) { ip, ipAddressClaim, err := h.claimIPAddress(ctx) Expect(err).ToNot(HaveOccurred()) ipAddressClaims = append(ipAddressClaims, types.NamespacedName{ Namespace: ipAddressClaim.Namespace, Name: ipAddressClaim.Name, }) - Byf("Setting clusterctl variable %s to %s", variable, ip) - variables[variable] = ip + Byf("Setting clusterctl variable %s to %s", variable, ip.Spec.Address) + variables[variable] = ip.Spec.Address + if variable == controlPlaneEndpointVariable && options.gatewayIPVariableName != "" { + // Set the gateway variable if requested to the gateway of the control plane IP. + // This is required in ipam scenarios, otherwise the VMs will not be able to + // connect to the public internet to pull images. + Byf("Setting clusterctl variable %s to %s", variable, ip.Spec.Gateway) + variables[options.gatewayIPVariableName] = ip.Spec.Gateway + } } return ipAddressClaims, variables @@ -271,7 +283,7 @@ func getVirtualMachineIPAddresses(ctx context.Context, folderName string, vSpher return virtualMachineIPAddresses, nil } -func (h *inCluster) claimIPAddress(ctx context.Context) (_ string, _ *ipamv1.IPAddressClaim, err error) { +func (h *inCluster) claimIPAddress(ctx context.Context) (_ *ipamv1.IPAddress, _ *ipamv1.IPAddressClaim, err error) { claim := &ipamv1.IPAddressClaim{ ObjectMeta: metav1.ObjectMeta{ Name: "ipclaim-" + rand.String(32), @@ -295,7 +307,7 @@ func (h *inCluster) claimIPAddress(ctx context.Context) (_ string, _ *ipamv1.IPA // Create an IPAddressClaim Byf("Creating IPAddressClaim %s", klog.KObj(claim)) if err := h.client.Create(ctx, claim); err != nil { - return "", nil, err + return nil, nil, err } // Store claim inside the service so the cleanup function knows what to delete. ip := &ipamv1.IPAddress{} @@ -328,8 +340,8 @@ func (h *inCluster) claimIPAddress(ctx context.Context) (_ string, _ *ipamv1.IPA if retryError != nil { // Try best effort deletion of the unused claim before returning an error. _ = h.client.Delete(ctx, claim) - return "", nil, retryError + return nil, nil, retryError } - return ip.Spec.Address, claim, nil + return ip, claim, nil } diff --git a/test/framework/ip/noop.go b/test/framework/ip/noop.go index 546aef3477..ae056f8055 100644 --- a/test/framework/ip/noop.go +++ b/test/framework/ip/noop.go @@ -27,7 +27,7 @@ var _ AddressManager = &noop{} type noop struct{} -func (h *noop) ClaimIPs(_ context.Context, _ ...string) (AddressClaims, map[string]string) { +func (h *noop) ClaimIPs(_ context.Context, _ ...ClaimOption) (AddressClaims, map[string]string) { return nil, nil }