Merge pull request #2201 from Nordix/test-ci-failure/adil

🌱 Remove CP scaling from e2e-feature-test

scripts/environment.sh

@@ -78,6 +78,13 @@ if [[ ${GINKGO_FOCUS:-} == "integration" || ${GINKGO_FOCUS:-} == "basic" ]]; the
   export WORKER_MACHINE_COUNT=${WORKER_MACHINE_COUNT:-"1"}
 fi
 
+# IPReuse feature test environment vars and config
+if [[ ${GINKGO_FOCUS:-} == "features" && ${GINKGO_SKIP:-} == "pivoting remediation" ]]; then
+  export NUM_NODES="5"
+  export CONTROL_PLANE_MACHINE_COUNT=${CONTROL_PLANE_MACHINE_COUNT:-"3"}
+  export WORKER_MACHINE_COUNT=${WORKER_MACHINE_COUNT:-"2"}
+fi
+
 # Exported to the cluster templates
 # Generate user ssh key
 if [ ! -f "${HOME}/.ssh/id_rsa" ]; then

test/e2e/basic_integration_test.go

@@ -23,7 +23,21 @@ var _ = Describe("When testing basic cluster creation [basic]", Label("basic"),
 		By("Fetching cluster configuration")
 		k8sVersion := e2eConfig.GetVariable("KUBERNETES_VERSION")
 		By("Provision Workload cluster")
-		targetCluster, _ = createTargetCluster(k8sVersion)
+		targetCluster, _ = CreateTargetCluster(ctx, func() CreateTargetClusterInput {
+			return CreateTargetClusterInput{
+				E2EConfig:             e2eConfig,
+				BootstrapClusterProxy: bootstrapClusterProxy,
+				SpecName:              specName,
+				ClusterName:           clusterName,
+				K8sVersion:            k8sVersion,
+				KCPMachineCount:       int64(numberOfControlplane),
+				WorkerMachineCount:    int64(numberOfWorkers),
+				ClusterctlLogFolder:   clusterctlLogFolder,
+				ClusterctlConfigPath:  clusterctlConfigPath,
+				OSType:                osType,
+				Namespace:             namespace,
+			}
+		})
 	})
 
 	AfterEach(func() {

test/e2e/common.go

@@ -32,6 +32,7 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/types"
 	kerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/client-go/kubernetes"
@@ -997,3 +998,53 @@ func CreateOrUpdateWithNamespace(ctx context.Context, p framework.ClusterProxy,
 	}
 	return kerrors.NewAggregate(retErrs)
 }
+
+type CreateTargetClusterInput struct {
+	E2EConfig             *clusterctl.E2EConfig
+	BootstrapClusterProxy framework.ClusterProxy
+	SpecName              string
+	ClusterName           string
+	K8sVersion            string
+	KCPMachineCount       int64
+	WorkerMachineCount    int64
+	ClusterctlLogFolder   string
+	ClusterctlConfigPath  string
+	OSType                string
+	Namespace             string
+}
+
+func CreateTargetCluster(ctx context.Context, inputGetter func() CreateTargetClusterInput) (framework.ClusterProxy, *clusterctl.ApplyClusterTemplateAndWaitResult) {
+	By("Creating a high available cluster")
+	input := inputGetter()
+	imageURL, imageChecksum := EnsureImage(input.K8sVersion)
+	os.Setenv("IMAGE_RAW_CHECKSUM", imageChecksum)
+	os.Setenv("IMAGE_RAW_URL", imageURL)
+	controlPlaneMachineCount := input.KCPMachineCount
+	workerMachineCount := input.WorkerMachineCount
+	result := clusterctl.ApplyClusterTemplateAndWaitResult{}
+	clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
+		ClusterProxy: input.BootstrapClusterProxy,
+		ConfigCluster: clusterctl.ConfigClusterInput{
+			LogFolder:                input.ClusterctlLogFolder,
+			ClusterctlConfigPath:     input.ClusterctlConfigPath,
+			KubeconfigPath:           input.BootstrapClusterProxy.GetKubeconfigPath(),
+			InfrastructureProvider:   clusterctl.DefaultInfrastructureProvider,
+			Flavor:                   input.OSType,
+			Namespace:                input.Namespace,
+			ClusterName:              input.ClusterName,
+			KubernetesVersion:        input.K8sVersion,
+			ControlPlaneMachineCount: &controlPlaneMachineCount,
+			WorkerMachineCount:       &workerMachineCount,
+		},
+		WaitForClusterIntervals:      input.E2EConfig.GetIntervals(input.SpecName, "wait-cluster"),
+		WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(input.SpecName, "wait-control-plane"),
+		WaitForMachineDeployments:    input.E2EConfig.GetIntervals(input.SpecName, "wait-worker-nodes"),
+	}, &result)
+	targetCluster := input.BootstrapClusterProxy.GetWorkloadCluster(ctx, input.Namespace, result.Cluster.Name)
+	framework.WaitForPodListCondition(ctx, framework.WaitForPodListConditionInput{
+		Lister:      targetCluster.GetClient(),
+		ListOptions: &client.ListOptions{LabelSelector: labels.Everything(), Namespace: "kube-system"},
+		Condition:   framework.PhasePodCondition(corev1.PodRunning),
+	}, input.E2EConfig.GetIntervals(input.SpecName, "wait-all-pod-to-be-running-on-target-cluster")...)
+	return targetCluster, &result
+}

test/e2e/integration_test.go

@@ -25,7 +25,21 @@ var _ = Describe("When testing integration [integration]", Label("integration"),
 		numberOfControlplane = int(*e2eConfig.GetInt32PtrVariable("CONTROL_PLANE_MACHINE_COUNT"))
 		k8sVersion := e2eConfig.GetVariable("KUBERNETES_VERSION")
 		By("Provision Workload cluster")
-		targetCluster, _ = createTargetCluster(k8sVersion)
+		targetCluster, _ = CreateTargetCluster(ctx, func() CreateTargetClusterInput {
+			return CreateTargetClusterInput{
+				E2EConfig:             e2eConfig,
+				BootstrapClusterProxy: bootstrapClusterProxy,
+				SpecName:              specName,
+				ClusterName:           clusterName,
+				K8sVersion:            k8sVersion,
+				KCPMachineCount:       int64(numberOfControlplane),
+				WorkerMachineCount:    int64(numberOfWorkers),
+				ClusterctlLogFolder:   clusterctlLogFolder,
+				ClusterctlConfigPath:  clusterctlConfigPath,
+				OSType:                osType,
+				Namespace:             namespace,
+			}
+		})
 		By("Pivot objects to target cluster")
 		pivoting(ctx, func() PivotingInput {
 			return PivotingInput{

test/e2e/ip_reuse.go

@@ -30,51 +30,53 @@ func IPReuse(ctx context.Context, inputGetter func() IPReuseInput) {
 	targetClusterClient := input.TargetCluster.GetClient()
 	managementClusterClient := input.BootstrapClusterProxy.GetClient()
 	fromK8sVersion := input.E2EConfig.GetVariable("FROM_K8S_VERSION")
-	kubernetesVersion := input.E2EConfig.GetVariable("KUBERNETES_VERSION")
+	toK8sVersion := input.E2EConfig.GetVariable("KUBERNETES_VERSION")
+	numberOfControlplane := *input.E2EConfig.GetInt32PtrVariable("CONTROL_PLANE_MACHINE_COUNT")
+	numberOfWorkers := *input.E2EConfig.GetInt32PtrVariable("WORKER_MACHINE_COUNT")
+	numberOfAllBmh := numberOfControlplane + numberOfWorkers
 
-	// scale down KCP to 1
-	By("Scale the controlplane down to 1")
-	ScaleKubeadmControlPlane(ctx, managementClusterClient, client.ObjectKey{Namespace: input.Namespace, Name: input.ClusterName}, 1)
-	Byf("Wait until controlplane is scaled down and %d BMHs are Available", 2)
-	WaitForNumBmhInState(ctx, bmov1alpha1.StateAvailable, WaitForNumInput{
+	// Download node image
+	Byf("Download image %s", toK8sVersion)
+	imageURL, imageChecksum := EnsureImage(toK8sVersion)
+
+	// Upgrade KCP
+	By("Create new KCP Metal3MachineTemplate with upgraded image to boot")
+	KCPm3MachineTemplateName := fmt.Sprintf("%s-controlplane", input.ClusterName)
+	KCPNewM3MachineTemplateName := fmt.Sprintf("%s-new-controlplane", input.ClusterName)
+	CreateNewM3MachineTemplate(ctx, input.Namespace, KCPNewM3MachineTemplateName, KCPm3MachineTemplateName, managementClusterClient, imageURL, imageChecksum)
+
+	Byf("Update KCP to upgrade k8s version and binaries from %s to %s", fromK8sVersion, toK8sVersion)
+	kcpObj := framework.GetKubeadmControlPlaneByCluster(ctx, framework.GetKubeadmControlPlaneByClusterInput{
+		Lister:      managementClusterClient,
+		ClusterName: input.ClusterName,
+		Namespace:   input.Namespace,
+	})
+	helper, err := patch.NewHelper(kcpObj, managementClusterClient)
+	Expect(err).NotTo(HaveOccurred())
+	kcpObj.Spec.MachineTemplate.InfrastructureRef.Name = KCPNewM3MachineTemplateName
+	kcpObj.Spec.Version = toK8sVersion
+	kcpObj.Spec.RolloutStrategy.RollingUpdate.MaxSurge.IntVal = 0
+
+	Expect(helper.Patch(ctx, kcpObj)).To(Succeed())
+
+	Byf("Wait until %d Control Plane machines become running and updated with the new %s k8s version", numberOfControlplane, toK8sVersion)
+	runningAndUpgraded := func(machine clusterv1.Machine) bool {
+		running := machine.Status.GetTypedPhase() == clusterv1.MachinePhaseRunning
+		upgraded := *machine.Spec.Version == toK8sVersion
+		return (running && upgraded)
+	}
+	WaitForNumMachines(ctx, runningAndUpgraded, WaitForNumInput{
 		Client:    managementClusterClient,
 		Options:   []client.ListOption{client.InNamespace(input.Namespace)},
-		Replicas:  2,
-		Intervals: input.E2EConfig.GetIntervals(input.SpecName, "wait-cp-available"),
+		Replicas:  int(numberOfControlplane),
+		Intervals: input.E2EConfig.GetIntervals(input.Namespace, "wait-machine-running"),
 	})
 
 	ListBareMetalHosts(ctx, managementClusterClient, client.InNamespace(input.Namespace))
 	ListMetal3Machines(ctx, managementClusterClient, client.InNamespace(input.Namespace))
 	ListMachines(ctx, managementClusterClient, client.InNamespace(input.Namespace))
 	ListNodes(ctx, targetClusterClient)
 
-	// scale up MD to 3
-	By("Scale the worker up to 3")
-	ScaleMachineDeployment(ctx, managementClusterClient, input.ClusterName, input.Namespace, 3)
-	By("Waiting for one BMH to become provisioning")
-	WaitForNumBmhInState(ctx, bmov1alpha1.StateProvisioning, WaitForNumInput{
-		Client:    managementClusterClient,
-		Options:   []client.ListOption{client.InNamespace(input.Namespace)},
-		Replicas:  2,
-		Intervals: input.E2EConfig.GetIntervals(input.SpecName, "wait-machine-remediation"),
-	})
-
-	By("Waiting for all BMHs to become provisioned")
-	WaitForNumBmhInState(ctx, bmov1alpha1.StateProvisioned, WaitForNumInput{
-		Client:    managementClusterClient,
-		Options:   []client.ListOption{client.InNamespace(input.Namespace)},
-		Replicas:  4,
-		Intervals: input.E2EConfig.GetIntervals(input.SpecName, "wait-machine-remediation"),
-	})
-
-	By("Waiting for all Machines to be Running")
-	WaitForNumMachinesInState(ctx, clusterv1.MachinePhaseRunning, WaitForNumInput{
-		Client:    managementClusterClient,
-		Options:   []client.ListOption{client.InNamespace(input.Namespace)},
-		Replicas:  4,
-		Intervals: input.E2EConfig.GetIntervals(input.SpecName, "wait-machine-remediation"),
-	})
-
 	By("Get the IPPools in the cluster")
 	baremetalv4Pool, provisioningPool := GetIPPools(ctx, managementClusterClient, input.ClusterName, input.Namespace)
 	Expect(baremetalv4Pool).To(HaveLen(1))
@@ -115,29 +117,25 @@ func IPReuse(ctx context.Context, inputGetter func() IPReuseInput) {
 	Expect(machineDeployments).To(HaveLen(1), "Expected exactly 1 MachineDeployment")
 	md := machineDeployments[0]
 
-	// Download node image
-	Byf("Download image %s", kubernetesVersion)
-	imageURL, imageChecksum := EnsureImage(kubernetesVersion)
-
 	By("Create new worker Metal3MachineTemplate with upgraded image to boot")
 	m3MachineTemplateName := md.Spec.Template.Spec.InfrastructureRef.Name
 	newM3MachineTemplateName := fmt.Sprintf("%s-new", m3MachineTemplateName)
 	CreateNewM3MachineTemplate(ctx, input.Namespace, newM3MachineTemplateName, m3MachineTemplateName, managementClusterClient, imageURL, imageChecksum)
 
-	Byf("Update MachineDeployment maxUnavailable to number of workers and k8s version from %s to %s", fromK8sVersion, kubernetesVersion)
-	helper, err := patch.NewHelper(md, managementClusterClient)
+	Byf("Update MachineDeployment maxUnavailable to number of workers and k8s version from %s to %s", fromK8sVersion, toK8sVersion)
+	helper, err = patch.NewHelper(md, managementClusterClient)
 	Expect(err).NotTo(HaveOccurred())
 	md.Spec.Template.Spec.InfrastructureRef.Name = newM3MachineTemplateName
-	md.Spec.Template.Spec.Version = &kubernetesVersion
+	md.Spec.Template.Spec.Version = &toK8sVersion
 	md.Spec.Strategy.RollingUpdate.MaxSurge.IntVal = 0
-	md.Spec.Strategy.RollingUpdate.MaxUnavailable.IntVal = 3
+	md.Spec.Strategy.RollingUpdate.MaxUnavailable.IntVal = numberOfWorkers
 	Expect(helper.Patch(ctx, md)).To(Succeed())
 
-	Byf("Wait until %d BMH(s) in deprovisioning state", 3)
+	Byf("Wait until %d BMH(s) in deprovisioning state", numberOfWorkers)
 	WaitForNumBmhInState(ctx, bmov1alpha1.StateDeprovisioning, WaitForNumInput{
 		Client:    managementClusterClient,
 		Options:   []client.ListOption{client.InNamespace(input.Namespace)},
-		Replicas:  3,
+		Replicas:  int(numberOfWorkers),
 		Intervals: input.E2EConfig.GetIntervals(input.SpecName, "wait-bmh-deprovisioning"),
 	})
 
@@ -146,11 +144,11 @@ func IPReuse(ctx context.Context, inputGetter func() IPReuseInput) {
 	ListMachines(ctx, managementClusterClient, client.InNamespace(input.Namespace))
 	ListNodes(ctx, targetClusterClient)
 
-	Byf("Wait until all %d machine(s) become(s) running", 4)
+	Byf("Wait until all %d machine(s) become(s) running", numberOfAllBmh)
 	WaitForNumMachinesInState(ctx, clusterv1.MachinePhaseRunning, WaitForNumInput{
 		Client:    managementClusterClient,
 		Options:   []client.ListOption{client.InNamespace(input.Namespace)},
-		Replicas:  4,
+		Replicas:  int(numberOfAllBmh),
 		Intervals: input.E2EConfig.GetIntervals(input.SpecName, "wait-machine-running"),
 	})
 

test/e2e/ip_reuse_test.go

@@ -21,8 +21,22 @@ var _ = Describe("When testing ip reuse [ip-reuse] [features]", Label("ip-reuse"
 		clusterctlLogFolder = filepath.Join(os.TempDir(), "target_cluster_logs", bootstrapClusterProxy.GetName())
 	})
 	It("Should create a workload cluster then verify ip allocation reuse while upgrading k8s", func() {
+		targetCluster, _ = CreateTargetCluster(ctx, func() CreateTargetClusterInput {
+			return CreateTargetClusterInput{
+				E2EConfig:             e2eConfig,
+				BootstrapClusterProxy: bootstrapClusterProxy,
+				SpecName:              specName,
+				ClusterName:           clusterName,
+				K8sVersion:            e2eConfig.GetVariable("FROM_K8S_VERSION"),
+				KCPMachineCount:       int64(numberOfControlplane),
+				WorkerMachineCount:    int64(numberOfWorkers),
+				ClusterctlLogFolder:   clusterctlLogFolder,
+				ClusterctlConfigPath:  clusterctlConfigPath,
+				OSType:                osType,
+				Namespace:             namespace,
+			}
+		})
 		IPReuse(ctx, func() IPReuseInput {
-			targetCluster, _ = createTargetCluster(e2eConfig.GetVariable("FROM_K8S_VERSION"))
 			return IPReuseInput{
 				E2EConfig:             e2eConfig,
 				BootstrapClusterProxy: bootstrapClusterProxy,

test/e2e/pivoting_based_feature_test.go

@@ -8,22 +8,17 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/labels"
 	"sigs.k8s.io/cluster-api/test/framework"
-	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 var (
-	ctx                      = context.TODO()
-	specName                 = "metal3"
-	namespace                = "metal3"
-	clusterName              = "test1"
-	clusterctlLogFolder      string
-	targetCluster            framework.ClusterProxy
-	controlPlaneMachineCount int64
-	workerMachineCount       int64
+	ctx                 = context.TODO()
+	specName            = "metal3"
+	namespace           = "metal3"
+	clusterName         = "test1"
+	clusterctlLogFolder string
+	targetCluster       framework.ClusterProxy
 )
 
 /*
@@ -91,7 +86,21 @@ var _ = Describe("Testing features in ephemeral or target cluster [pivoting] [fe
 		})
 
 		It("Should get a management cluster then test cert rotation and node reuse", func() {
-			targetCluster, _ = createTargetCluster(e2eConfig.GetVariable("FROM_K8S_VERSION"))
+			targetCluster, _ = CreateTargetCluster(ctx, func() CreateTargetClusterInput {
+				return CreateTargetClusterInput{
+					E2EConfig:             e2eConfig,
+					BootstrapClusterProxy: bootstrapClusterProxy,
+					SpecName:              specName,
+					ClusterName:           clusterName,
+					K8sVersion:            e2eConfig.GetVariable("FROM_K8S_VERSION"),
+					KCPMachineCount:       int64(numberOfControlplane),
+					WorkerMachineCount:    int64(numberOfWorkers),
+					ClusterctlLogFolder:   clusterctlLogFolder,
+					ClusterctlConfigPath:  clusterctlConfigPath,
+					OSType:                osType,
+					Namespace:             namespace,
+				}
+			})
 			managementCluster := bootstrapClusterProxy
 			// If not running ephemeral test, use the target cluster for management
 			if !ephemeralTest {
@@ -169,38 +178,3 @@ var _ = Describe("Testing features in ephemeral or target cluster [pivoting] [fe
 		})
 
 	})
-
-func createTargetCluster(k8sVersion string) (framework.ClusterProxy, *clusterctl.ApplyClusterTemplateAndWaitResult) {
-	By("Creating a high available cluster")
-	imageURL, imageChecksum := EnsureImage(k8sVersion)
-	os.Setenv("IMAGE_RAW_CHECKSUM", imageChecksum)
-	os.Setenv("IMAGE_RAW_URL", imageURL)
-	controlPlaneMachineCount = int64(numberOfControlplane)
-	workerMachineCount = int64(numberOfWorkers)
-	result := clusterctl.ApplyClusterTemplateAndWaitResult{}
-	clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
-		ClusterProxy: bootstrapClusterProxy,
-		ConfigCluster: clusterctl.ConfigClusterInput{
-			LogFolder:                clusterctlLogFolder,
-			ClusterctlConfigPath:     clusterctlConfigPath,
-			KubeconfigPath:           bootstrapClusterProxy.GetKubeconfigPath(),
-			InfrastructureProvider:   clusterctl.DefaultInfrastructureProvider,
-			Flavor:                   osType,
-			Namespace:                namespace,
-			ClusterName:              clusterName,
-			KubernetesVersion:        k8sVersion,
-			ControlPlaneMachineCount: &controlPlaneMachineCount,
-			WorkerMachineCount:       &workerMachineCount,
-		},
-		WaitForClusterIntervals:      e2eConfig.GetIntervals(specName, "wait-cluster"),
-		WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
-		WaitForMachineDeployments:    e2eConfig.GetIntervals(specName, "wait-worker-nodes"),
-	}, &result)
-	targetCluster := bootstrapClusterProxy.GetWorkloadCluster(ctx, namespace, clusterName)
-	framework.WaitForPodListCondition(ctx, framework.WaitForPodListConditionInput{
-		Lister:      targetCluster.GetClient(),
-		ListOptions: &client.ListOptions{LabelSelector: labels.Everything(), Namespace: "kube-system"},
-		Condition:   framework.PhasePodCondition(corev1.PodRunning),
-	}, e2eConfig.GetIntervals(specName, "wait-all-pod-to-be-running-on-target-cluster")...)
-	return targetCluster, &result
-}

test/e2e/remediation_based_feature_test.go

@@ -72,8 +72,21 @@ var _ = Describe("Testing nodes remediation [remediation] [features]", Label("re
 
 	It("Should create a cluster and run remediation based tests", func() {
 		By("Creating target cluster")
-		targetCluster, _ = createTargetCluster(e2eConfig.GetVariable("KUBERNETES_VERSION"))
-
+		targetCluster, _ = CreateTargetCluster(ctx, func() CreateTargetClusterInput {
+			return CreateTargetClusterInput{
+				E2EConfig:             e2eConfig,
+				BootstrapClusterProxy: bootstrapClusterProxy,
+				SpecName:              specName,
+				ClusterName:           clusterName,
+				K8sVersion:            e2eConfig.GetVariable("FROM_K8S_VERSION"),
+				KCPMachineCount:       int64(numberOfControlplane),
+				WorkerMachineCount:    int64(numberOfWorkers),
+				ClusterctlLogFolder:   clusterctlLogFolder,
+				ClusterctlConfigPath:  clusterctlConfigPath,
+				OSType:                osType,
+				Namespace:             namespace,
+			}
+		})
 		// Run Metal3Remediation test first, doesn't work after remediation...
 		By("Running node remediation tests")
 		nodeRemediation(ctx, func() NodeRemediation {