Skip to content

Commit

Permalink
Added basic e2e tests
Browse files Browse the repository at this point in the history
Signed-off-by: Patryk Strusiewicz-Surmacki <[email protected]>
  • Loading branch information
p-strusiewiczsurmacki-mobica committed Apr 11, 2024
1 parent 95d5b65 commit 854ef91
Show file tree
Hide file tree
Showing 18 changed files with 865 additions and 2 deletions.
35 changes: 35 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,10 @@ test: lint unit ## Run tests
test-e2e: ## Run e2e tests with capi e2e testing framework
./scripts/ci-e2e.sh

.PHONY: test-clusterclass-e2e
test-clusterclass-e2e: ## Run e2e tests with capi e2e testing framework
CLUSTER_TOPOLOGY=true GINKGO_FOCUS=basic ./scripts/ci-e2e.sh

GINKGO_NOCOLOR ?= false
ARTIFACTS ?= $(ROOT_DIR)/_artifacts
E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/e2e_conf.yaml
Expand Down Expand Up @@ -168,6 +172,14 @@ cluster-templates: $(KUSTOMIZE) ## Generate cluster templates
$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-ubuntu > $(E2E_OUT_DIR)/cluster-template-ubuntu.yaml
$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-centos > $(E2E_OUT_DIR)/cluster-template-centos.yaml
$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-upgrade-workload > $(E2E_OUT_DIR)/cluster-template-upgrade-workload.yaml
touch $(E2E_OUT_DIR)/clusterclass.yaml

.PHONY: clusterclass-templates
clusterclass-templates: $(KUSTOMIZE) ## Generate cluster templates
$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/clusterclass-template-ubuntu > $(E2E_OUT_DIR)/cluster-template-ubuntu.yaml
$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/clusterclass-template-centos > $(E2E_OUT_DIR)/cluster-template-centos.yaml
$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/clusterclass-template-upgrade-workload > $(E2E_OUT_DIR)/cluster-template-upgrade-workload.yaml
$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/clusterclass > $(E2E_OUT_DIR)/clusterclass.yaml

## --------------------------------------
## E2E Testing
Expand Down Expand Up @@ -204,6 +216,29 @@ e2e-tests: $(GINKGO) e2e-substitutions cluster-templates # This target should be

rm $(E2E_CONF_FILE_ENVSUBST)

.PHONY: e2e-clusterclass-tests
e2e-clusterclass-tests: CONTAINER_RUNTIME?=docker # Env variable can override this default
export CONTAINER_RUNTIME

e2e-clusterclass-tests: $(GINKGO) e2e-substitutions clusterclass-templates # This target should be called from scripts/ci-e2e.sh
for image in $(E2E_CONTAINERS); do \
$(CONTAINER_RUNTIME) pull $$image; \
done

$(GINKGO) --timeout=$(GINKGO_TIMEOUT) -v --trace --tags=e2e \
--show-node-events --no-color=$(GINKGO_NOCOLOR) \
--fail-fast="$(KEEP_TEST_ENV)" \
--junit-report="junit.e2e_suite.1.xml" \
--focus="$(GINKGO_FOCUS)" $(_SKIP_ARGS) "$(ROOT_DIR)/$(TEST_DIR)/e2e/" -- \
-e2e.artifacts-folder="$(ARTIFACTS)" \
-e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \
-e2e.skip-resource-cleanup=$(SKIP_CLEANUP) \
-e2e.keep-test-environment=$(KEEP_TEST_ENV) \
-e2e.trigger-ephemeral-test=$(EPHEMERAL_TEST) \
-e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER)

rm $(E2E_CONF_FILE_ENVSUBST)

## --------------------------------------
## Build
## --------------------------------------
Expand Down
2 changes: 1 addition & 1 deletion config/ipam/image_patch.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,4 @@ spec:
containers:
# Change the value of image field below to your controller image URL
- image: quay.io/metal3-io/ip-address-manager:v1.7.0-beta.0
name: manager
name: manager
8 changes: 7 additions & 1 deletion scripts/ci-e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -120,4 +120,10 @@ export CONTRACT_TO="v1beta1"
export LIVE_ISO_IMAGE="https://artifactory.nordix.org/artifactory/metal3/images/iso/minimal_linux_live-v2.iso"

# run e2e tests
make e2e-tests
if [ -n "${CLUSTER_TOPOLOGY:-}" ]; then
export CLUSTER_TOPOLOGY=true
make e2e-clusterclass-tests
else
make e2e-tests
fi

2 changes: 2 additions & 0 deletions test/e2e/config/e2e_conf.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,8 @@ providers:
targetName: "cluster-template-ubuntu.yaml"
- sourcePath: "../_out/cluster-template-centos.yaml"
targetName: "cluster-template-centos.yaml"
- sourcePath: "../_out/clusterclass.yaml"
targetName: "clusterclass-test-clusterclass.yaml"

variables:
CNI: "/tmp/calico.yaml"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,244 @@
---
apiVersion: controlplane.cluster.x-k8s.io/${CAPI_VERSION}
kind: KubeadmControlPlaneTemplate
metadata:
name: ${CLUSTER_NAME}
namespace: ${NAMESPACE}
spec:
template:
spec:
kubeadmConfigSpec:
clusterConfiguration: {}
users:
- name: ${IMAGE_USERNAME}
sshAuthorizedKeys:
- ${SSH_PUB_KEY_CONTENT}
sudo: ALL=(ALL) NOPASSWD:ALL
files:
- content: |
#!/bin/bash
set -e
url="$1"
dst="$2"
filename="$(basename $url)"
tmpfile="/tmp/$filename"
curl -sSL -w "%{http_code}" "$url" | sed "s:/usr/bin:/usr/local/bin:g" > /tmp/"$filename"
http_status=$(cat "$tmpfile" | tail -n 1)
if [ "$http_status" != "200" ]; then
echo "Error: unable to retrieve $filename file";
exit 1;
else
cat "$tmpfile"| sed '$d' > "$dst";
fi
owner: root:root
path: /usr/local/bin/retrieve.configuration.files.sh
permissions: "0755"
- path: /etc/keepalived/keepalived.conf
content: |
! Configuration File for keepalived
global_defs {
notification_email {
[email protected]
[email protected]
}
notification_email_from [email protected]
smtp_server localhost
smtp_connect_timeout 30
}
vrrp_script k8s_api_check {
script "curl -sk https://127.0.0.1:6443/healthz"
interval 5
timeout 5
rise 3
fall 3
}
vrrp_instance VI_1 {
state MASTER
interface eth1
virtual_router_id 1
priority 101
advert_int 1
virtual_ipaddress {
${CLUSTER_APIENDPOINT_HOST}
}
track_script {
k8s_api_check
}
}
- path: /etc/NetworkManager/system-connections/eth0.nmconnection
owner: root:root
permissions: '0600'
content: |
[connection]
id=eth0
type=ethernet
interface-name=eth0
master=ironicendpoint
slave-type=bridge
- content: |
[connection]
id=ironicendpoint
type=bridge
interface-name=ironicendpoint
autoconnect=yes
autoconnect-priority=1
[bridge]
stp=false
interface-name=ironicendpoint
[ipv4]
address1={{ ds.meta_data.provisioningIP }}/{{ ds.meta_data.provisioningCIDR }}
method=manual
[ipv6]
addr-gen-mode=eui64
method=ignore
path: /etc/NetworkManager/system-connections/ironicendpoint.nmconnection
owner: root:root
permissions: '0600'
- content: |
[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=0
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
owner: root:root
path: /etc/yum.repos.d/kubernetes.repo
permissions: '0644'
- content: |
[registries.search]
registries = ['docker.io']
[registries.insecure]
registries = ['${REGISTRY}']
path: /etc/containers/registries.conf
initConfiguration:
nodeRegistration:
kubeletExtraArgs:
cgroup-driver: systemd
container-runtime-endpoint: unix:///var/run/crio/crio.sock
feature-gates: AllAlpha=false
node-labels: metal3.io/uuid={{ ds.meta_data.uuid }}
provider-id: ${PROVIDER_ID_FORMAT}
runtime-request-timeout: 5m
name: '{{ ds.meta_data.name }}'
joinConfiguration:
controlPlane: {}
nodeRegistration:
kubeletExtraArgs:
cgroup-driver: systemd
container-runtime-endpoint: unix:///var/run/crio/crio.sock
feature-gates: AllAlpha=false
node-labels: metal3.io/uuid={{ ds.meta_data.uuid }}
provider-id: ${PROVIDER_ID_FORMAT}
runtime-request-timeout: 5m
name: '{{ ds.meta_data.name }}'
preKubeadmCommands:
- rm /etc/cni/net.d/*
- systemctl restart NetworkManager.service
- nmcli connection load /etc/NetworkManager/system-connections/ironicendpoint.nmconnection
- nmcli connection up ironicendpoint
- nmcli connection load /etc/NetworkManager/system-connections/eth0.nmconnection
- nmcli connection up eth0
- systemctl enable --now keepalived
- sleep 60
- systemctl enable --now crio kubelet
postKubeadmCommands:
- mkdir -p /home/${IMAGE_USERNAME}/.kube
- chown ${IMAGE_USERNAME}:${IMAGE_USERNAME} /home/${IMAGE_USERNAME}/.kube
- cp /etc/kubernetes/admin.conf /home/${IMAGE_USERNAME}/.kube/config
- chown ${IMAGE_USERNAME}:${IMAGE_USERNAME} /home/${IMAGE_USERNAME}/.kube/config

---
apiVersion: bootstrap.cluster.x-k8s.io/${CAPI_VERSION}
kind: KubeadmConfigTemplate
metadata:
name: ${CLUSTER_NAME}-workers
namespace: ${NAMESPACE}
spec:
template:
spec:
files:
- content: |
#!/bin/bash
set -e
url="$1"
dst="$2"
filename="$(basename $url)"
tmpfile="/tmp/$filename"
curl -sSL -w "%{http_code}" "$url" | sed "s:/usr/bin:/usr/local/bin:g" > /tmp/"$filename"
http_status=$(cat "$tmpfile" | tail -n 1)
if [ "$http_status" != "200" ]; then
echo "Error: unable to retrieve $filename file";
exit 1;
else
cat "$tmpfile"| sed '$d' > "$dst";
fi
owner: root:root
path: /usr/local/bin/retrieve.configuration.files.sh
permissions: "0755"
- content: |
[connection]
id=eth0
type=ethernet
interface-name=eth0
master=ironicendpoint
slave-type=bridge
autoconnect=yes
autoconnect-priority=999
path: /etc/NetworkManager/system-connections/eth0.nmconnection
owner: root:root
permissions: '0600'
- path: /etc/NetworkManager/system-connections/ironicendpoint.nmconnection
owner: root:root
permissions: '0600'
content: |
[connection]
id=ironicendpoint
type=bridge
interface-name=ironicendpoint
[bridge]
stp=false
[ipv4]
address1={{ ds.meta_data.provisioningIP }}/{{ ds.meta_data.provisioningCIDR }}
method=manual
[ipv6]
addr-gen-mode=eui64
method=ignore
- path: /etc/yum.repos.d/kubernetes.repo
owner: root:root
permissions: '0644'
content: |
[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=0
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
- path : /etc/containers/registries.conf
owner: root:root
permissions: '0644'
content: |
[registries.search]
registries = ['docker.io']
[registries.insecure]
registries = ['${REGISTRY}']
preKubeadmCommands:
- rm /etc/cni/net.d/*
- systemctl restart NetworkManager.service
- nmcli connection load /etc/NetworkManager/system-connections/ironicendpoint.nmconnection
- nmcli connection up ironicendpoint
- nmcli connection load /etc/NetworkManager/system-connections/eth0.nmconnection
- nmcli connection up eth0
- systemctl enable --now crio kubelet
- sleep 120
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
resources:
- ../clusterclass-cluster
patchesStrategicMerge:
- clusterclass-centos-kubeadm-config.yaml
Loading

0 comments on commit 854ef91

Please sign in to comment.