Add Scalability test

Signed-off-by: Mohammed Boukhalfa <[email protected]>
metal3-io · Oct 21, 2024 · 1099be2 · 1099be2
1 parent 627f9ba
commit 1099be2
Show file tree

Hide file tree

Showing 18 changed files with 1,022 additions and 22 deletions.
diff --git a/Makefile b/Makefile
@@ -173,6 +173,7 @@ E2E_TEMPLATES_DIR ?= $(ROOT_DIR)/test/e2e/data/infrastructure-metal3
 cluster-templates: $(KUSTOMIZE) ## Generate cluster templates
 	$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-ubuntu > $(E2E_OUT_DIR)/cluster-template-ubuntu.yaml
 	$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-centos > $(E2E_OUT_DIR)/cluster-template-centos.yaml
+	$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-centos-fake > $(E2E_OUT_DIR)/cluster-template-centos-fake.yaml
 	$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-upgrade-workload > $(E2E_OUT_DIR)/cluster-template-upgrade-workload.yaml
 	$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-centos-md-remediation > $(E2E_OUT_DIR)/cluster-template-centos-md-remediation.yaml
 	$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-ubuntu-md-remediation > $(E2E_OUT_DIR)/cluster-template-ubuntu-md-remediation.yaml

diff --git a/docs/ip_reuse.md b/docs/ip_reuse.md
@@ -56,7 +56,7 @@ included) to the `preAllocations` field in the `IPPool`, i.e:
 apiVersion: ipam.metal3.io/v1alpha1
 kind: IPPool
 metadata:
-  name: baremetalv4-pool
+  name: baremetalv4-pool-${CLUSTER_NAME}
   namespace: metal3
 spec:
   clusterName: test1

diff --git a/go.mod b/go.mod
@@ -27,6 +27,8 @@ require (
 
 replace github.com/metal3-io/cluster-api-provider-metal3/api => ./api
 
+replace sigs.k8s.io/cluster-api/test => github.com/Nordix/cluster-api/test v1.0.1-0.20241021093551-8634871d5229
+
 require (
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
 	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect

diff --git a/scripts/ci-e2e.sh b/scripts/ci-e2e.sh
@@ -20,8 +20,8 @@ source "${REPO_ROOT}/scripts/environment.sh"
 # Clone dev-env repo
 sudo mkdir -p ${WORKING_DIR}
 sudo chown "${USER}":"${USER}" ${WORKING_DIR}
-M3_DEV_ENV_REPO="https://github.com/metal3-io/metal3-dev-env.git"
-M3_DEV_ENV_BRANCH=main
+M3_DEV_ENV_REPO="https://github.com/Nordix/metal3-dev-env.git"
+M3_DEV_ENV_BRANCH=Integrate-fakeIPA/mohammed
 M3_DEV_ENV_PATH="${M3_DEV_ENV_PATH:-${WORKING_DIR}/metal3-dev-env}"
 clone_repo "${M3_DEV_ENV_REPO}" "${M3_DEV_ENV_BRANCH}" "${M3_DEV_ENV_PATH}"
 
@@ -42,6 +42,10 @@ if [[ ${GINKGO_FOCUS:-} == "features" ]]; then
     mkdir -p "$CAPI_CONFIG_FOLDER"
     echo "ENABLE_BMH_NAME_BASED_PREALLOCATION: true" >"$CAPI_CONFIG_FOLDER/clusterctl.yaml"
 fi
+# if running a scalability test run DevEnv with fakeIPA
+if [[ ${GINKGO_FOCUS:-} == "scalability" ]]; then
+    echo 'export NODES_PLATFORM="fake"' >>"${M3_DEV_ENV_PATH}/config_${USER}.sh"
+fi
 # Run make devenv to boot the source cluster
 pushd "${M3_DEV_ENV_PATH}" || exit 1
 make

diff --git a/scripts/environment.sh b/scripts/environment.sh
@@ -62,6 +62,14 @@ if [[ ${GINKGO_FOCUS:-} == "clusterctl-upgrade" ]]; then
   export NUM_NODES="5"
 fi
 
+# Scalability test environment vars and config
+if [[ ${GINKGO_FOCUS:-} == "scalability" ]]; then
+  export NUM_NODES=${NUM_NODES:-"40"}
+  export BMH_BATCH_SIZE=${BMH_BATCH_SIZE:-"5"}
+  export CONTROL_PLANE_MACHINE_COUNT=${CONTROL_PLANE_MACHINE_COUNT:-"1"}
+  export WORKER_MACHINE_COUNT=${WORKER_MACHINE_COUNT:-"1"}
+fi
+
 # Integration test environment vars and config
 if [[ ${GINKGO_FOCUS:-} == "integration" || ${GINKGO_FOCUS:-} == "basic" ]]; then
   export NUM_NODES=${NUM_NODES:-"2"}

diff --git a/test/e2e/config/e2e_conf.yaml b/test/e2e/config/e2e_conf.yaml
@@ -150,6 +150,8 @@ providers:
       targetName: "cluster-template-ubuntu.yaml"
     - sourcePath: "../_out/cluster-template-centos.yaml"
       targetName: "cluster-template-centos.yaml"
+    - sourcePath: "../_out/cluster-template-centos-fake.yaml"
+      targetName: "cluster-template-centos-fake.yaml"
     - sourcePath: "../_out/clusterclass.yaml"
       targetName: "clusterclass-test-clusterclass.yaml"
     - sourcePath: "../_out/cluster-template-centos-md-remediation.yaml"
@@ -210,7 +212,8 @@ variables:
   BMO_RELEASE_0.5: "data/bmo-deployment/overlays/release-0.5"
   BMO_RELEASE_0.6: "data/bmo-deployment/overlays/release-0.6"
   BMO_RELEASE_LATEST: "data/bmo-deployment/overlays/release-latest"
-
+  FKAS_RELEASE_LATEST: "data/fkas"
+
 intervals:
   default/wait-controllers: ["10m", "10s"]
   default/wait-cluster: ["20m", "30s"] # The second time to check the availibility of the cluster should happen late, so kcp object has time to be created

diff --git a/test/e2e/data/fkas/kustomization.yaml b/test/e2e/data/fkas/kustomization.yaml
@@ -0,0 +1,2 @@
+resources:
+- resources.yaml
diff --git a/test/e2e/data/fkas/resources.yaml b/test/e2e/data/fkas/resources.yaml
@@ -0,0 +1,77 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: fkas-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: metal3-fkas-sa
+  namespace: fkas-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: metal3-fkas-role
+rules:
+- apiGroups: ["metal3.io", "infrastructure.cluster.x-k8s.io"]
+  resources: ["baremetalhosts", "metal3machines"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["cluster.x-k8s.io"]
+  resources: ["machines"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metal3-fkas-rolebinding
+subjects:
+- kind: ServiceAccount
+  name: metal3-fkas-sa
+  namespace: fkas-system
+roleRef:
+  kind: ClusterRole
+  name: metal3-fkas-role
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: metal3-fkas-system
+  namespace: fkas-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: metal3-fkas-system
+  template:
+    metadata:
+      labels:
+        app: metal3-fkas-system
+    spec:
+      serviceAccountName: metal3-fkas-sa
+      hostNetwork: true
+      containers:
+      - name: metal3-fkas-reconciler
+        image: 192.168.111.1:5000/localimages/api-server 
+        imagePullPolicy: IfNotPresent
+        command: ["/reconciler"]
+        env:
+        - name: DEBUG
+          value: "true"
+      - image: 192.168.111.1:5000/localimages/api-server 
+        imagePullPolicy: IfNotPresent
+        ports:
+        - containerPort: 3333
+        env:
+        - name: POD_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        - name: DEBUG
+          value: "true"
+        name: metal3-fkas
diff --git a/test/e2e/data/infrastructure-metal3/bases/cluster/cluster-with-kcp.yaml b/test/e2e/data/infrastructure-metal3/bases/cluster/cluster-with-kcp.yaml
@@ -82,7 +82,7 @@ spec:
   metaData:
     ipAddressesFromIPPool:
     - key: provisioningIP
-      name: provisioning-pool
+      name: provisioning-pool-${CLUSTER_NAME}
     objectNames:
     - key: name
       object: machine
@@ -92,7 +92,7 @@ spec:
       object: machine
     prefixesFromIPPool:
     - key: provisioningCIDR
-      name: provisioning-pool
+      name: provisioning-pool-${CLUSTER_NAME}
   networkData:
     links:
       ethernets:
@@ -107,11 +107,11 @@ spec:
     networks:
       ipv4:
       - id: baremetalv4
-        ipAddressFromIPPool: baremetalv4-pool
+        ipAddressFromIPPool: baremetalv4-pool-${CLUSTER_NAME}
         link: enp2s0
         routes:
         - gateway:
-            fromIPPool: baremetalv4-pool
+            fromIPPool: baremetalv4-pool-${CLUSTER_NAME}
           network: 0.0.0.0
           prefix: 0
     services:

diff --git a/test/e2e/data/infrastructure-metal3/bases/cluster/md.yaml b/test/e2e/data/infrastructure-metal3/bases/cluster/md.yaml
@@ -59,7 +59,7 @@ spec:
   metaData:
     ipAddressesFromIPPool:
     - key: provisioningIP
-      name: provisioning-pool
+      name: provisioning-pool-${CLUSTER_NAME}
     objectNames:
     - key: name
       object: machine
@@ -69,7 +69,7 @@ spec:
       object: machine
     prefixesFromIPPool:
     - key: provisioningCIDR
-      name: provisioning-pool
+      name: provisioning-pool-${CLUSTER_NAME}
   networkData:
     links:
       ethernets:
@@ -84,11 +84,11 @@ spec:
     networks:
       ipv4:
       - id: baremetalv4
-        ipAddressFromIPPool: baremetalv4-pool
+        ipAddressFromIPPool: baremetalv4-pool-${CLUSTER_NAME}
         link: enp2s0
         routes:
         - gateway:
-            fromIPPool: baremetalv4-pool
+            fromIPPool: baremetalv4-pool-${CLUSTER_NAME}
           network: 0.0.0.0
           prefix: 0
     services:

diff --git a/test/e2e/data/infrastructure-metal3/bases/clusterclass-cluster/cluster-with-kcp.yaml b/test/e2e/data/infrastructure-metal3/bases/clusterclass-cluster/cluster-with-kcp.yaml
@@ -43,7 +43,7 @@ spec:
   metaData:
     ipAddressesFromIPPool:
     - key: provisioningIP
-      name: provisioning-pool
+      name: provisioning-pool-${CLUSTER_NAME}
     objectNames:
     - key: name
       object: machine
@@ -53,7 +53,7 @@ spec:
       object: machine
     prefixesFromIPPool:
     - key: provisioningCIDR
-      name: provisioning-pool
+      name: provisioning-pool-${CLUSTER_NAME}
   networkData:
     links:
       ethernets:
@@ -68,11 +68,11 @@ spec:
     networks:
       ipv4:
       - id: baremetalv4
-        ipAddressFromIPPool: baremetalv4-pool
+        ipAddressFromIPPool: baremetalv4-pool-${CLUSTER_NAME}
         link: enp2s0
         routes:
         - gateway:
-            fromIPPool: baremetalv4-pool
+            fromIPPool: baremetalv4-pool-${CLUSTER_NAME}
           network: 0.0.0.0
           prefix: 0
     services:

diff --git a/test/e2e/data/infrastructure-metal3/bases/clusterclass-cluster/md.yaml b/test/e2e/data/infrastructure-metal3/bases/clusterclass-cluster/md.yaml
@@ -24,7 +24,7 @@ spec:
   metaData:
     ipAddressesFromIPPool:
     - key: provisioningIP
-      name: provisioning-pool
+      name: provisioning-pool-${CLUSTER_NAME}
     objectNames:
     - key: name
       object: machine
@@ -34,7 +34,7 @@ spec:
       object: machine
     prefixesFromIPPool:
     - key: provisioningCIDR
-      name: provisioning-pool
+      name: provisioning-pool-${CLUSTER_NAME}
   networkData:
     links:
       ethernets:
@@ -49,11 +49,11 @@ spec:
     networks:
       ipv4:
       - id: baremetalv4
-        ipAddressFromIPPool: baremetalv4-pool
+        ipAddressFromIPPool: baremetalv4-pool-${CLUSTER_NAME}
         link: enp2s0
         routes:
         - gateway:
-            fromIPPool: baremetalv4-pool
+            fromIPPool: baremetalv4-pool-${CLUSTER_NAME}
           network: 0.0.0.0
           prefix: 0
     services:

diff --git a/test/e2e/data/infrastructure-metal3/bases/ippool/ippool.yaml b/test/e2e/data/infrastructure-metal3/bases/ippool/ippool.yaml
@@ -2,7 +2,7 @@
 apiVersion: ipam.metal3.io/v1alpha1
 kind: IPPool
 metadata:
-  name: provisioning-pool
+  name: provisioning-pool-${CLUSTER_NAME}
   namespace: ${NAMESPACE}
 spec:
   clusterName: ${CLUSTER_NAME}
@@ -15,7 +15,7 @@ spec:
 apiVersion: ipam.metal3.io/v1alpha1
 kind: IPPool
 metadata:
-  name: baremetalv4-pool
+  name: baremetalv4-pool-${CLUSTER_NAME}
   namespace: ${NAMESPACE}
 spec:
   clusterName: ${CLUSTER_NAME}

diff --git a/test/e2e/data/infrastructure-metal3/cluster-template-centos-fake/kustomization.yaml b/test/e2e/data/infrastructure-metal3/cluster-template-centos-fake/kustomization.yaml
@@ -0,0 +1,3 @@
+resources:
+- ../bases/ippool
+- ../bases/centos-kubeadm-config