From 368ad80a284c462c49eeacb1abc52f76bc3a305b Mon Sep 17 00:00:00 2001 From: Alex Kalenyuk Date: Fri, 26 Jul 2024 07:25:15 +0300 Subject: [PATCH] Update kubevirtci, 1.30 lanes (#3343) * Bump kvci to latest Signed-off-by: Alex Kalenyuk * Switch testing lanes to 1.30/1.29 Signed-off-by: Alex Kalenyuk * Revert "Add W/A for NFS OOMKills" https://github.com/kubevirt/kubevirt/issues/10822#issuecomment-2220620883 This reverts commit aba780300f863942346903efa61bde8e9ddbaef7. Signed-off-by: Alex Kalenyuk --------- Signed-off-by: Alex Kalenyuk --- automation/ceph-wffc.sh | 2 +- automation/ceph.sh | 2 +- automation/destructive.sh | 2 +- automation/istio.sh | 2 +- automation/latest-hpp.sh | 2 +- automation/nfs.sh | 2 +- automation/non-csi-hpp.sh | 2 +- automation/previous-hpp.sh | 2 +- automation/upgrade.sh | 2 +- cluster-sync/sync.sh | 6 -- cluster-up/check.sh | 2 +- .../cluster/ephemeral-provider-common.sh | 6 +- .../{k8s-1.27 => k8s-1.31}/provider.sh | 0 cluster-up/cluster/k8s-provider-common.sh | 64 +++++++++++++- cluster-up/cluster/kind-1.27-vgpu/provider.sh | 3 - .../kind-1.30-vgpu/config_vgpu_cluster.sh | 19 ++++ .../cluster/kind-1.30-vgpu/conformance.json | 47 ++++++++++ cluster-up/cluster/kind-1.30-vgpu/image | 1 + cluster-up/cluster/kind-1.30-vgpu/provider.sh | 58 ++++++++++++ cluster-up/cluster/kind-1.30-vgpu/version | 1 + .../cluster/kind-1.30-vgpu/vgpu-node/node.sh | 32 +++++++ cluster-up/cluster/kind-ovn/README.md | 29 ++++++ cluster-up/cluster/kind-ovn/install-ovn.sh | 47 ++++++++++ cluster-up/cluster/kind-ovn/provider.sh | 88 +++++++++++++++++++ cluster-up/cluster/kind/common.sh | 2 +- cluster-up/hack/common.sh | 9 +- cluster-up/up.sh | 6 -- hack/update-kubevirtci.sh | 2 +- 28 files changed, 407 insertions(+), 33 deletions(-) rename cluster-up/cluster/{k8s-1.27 => k8s-1.31}/provider.sh (100%) create mode 100755 cluster-up/cluster/kind-1.30-vgpu/config_vgpu_cluster.sh create mode 100644 cluster-up/cluster/kind-1.30-vgpu/conformance.json create mode 100644 cluster-up/cluster/kind-1.30-vgpu/image create mode 100755 cluster-up/cluster/kind-1.30-vgpu/provider.sh create mode 100644 cluster-up/cluster/kind-1.30-vgpu/version create mode 100644 cluster-up/cluster/kind-1.30-vgpu/vgpu-node/node.sh create mode 100644 cluster-up/cluster/kind-ovn/README.md create mode 100755 cluster-up/cluster/kind-ovn/install-ovn.sh create mode 100755 cluster-up/cluster/kind-ovn/provider.sh diff --git a/automation/ceph-wffc.sh b/automation/ceph-wffc.sh index 3e665c33fc..9ec2edace3 100755 --- a/automation/ceph-wffc.sh +++ b/automation/ceph-wffc.sh @@ -18,7 +18,7 @@ # set -ex -export TARGET=k8s-1.28 +export TARGET=k8s-1.29 #ensure no hard coded cdi cr in tests. export RANDOM_CR=true export KUBEVIRT_STORAGE=rook-ceph-default diff --git a/automation/ceph.sh b/automation/ceph.sh index b6d41a4971..7575d97f46 100755 --- a/automation/ceph.sh +++ b/automation/ceph.sh @@ -18,7 +18,7 @@ # set -ex -export TARGET=k8s-1.28 +export TARGET=k8s-1.29 #ensure no hard coded cdi cr in tests. export RANDOM_CR=true export KUBEVIRT_STORAGE=rook-ceph-default diff --git a/automation/destructive.sh b/automation/destructive.sh index 50a25f01d1..c3a0044e8d 100755 --- a/automation/destructive.sh +++ b/automation/destructive.sh @@ -18,7 +18,7 @@ # set -ex -export TARGET=k8s-1.29 +export TARGET=k8s-1.30 export KUBEVIRT_STORAGE=hpp export KUBEVIRT_DEPLOY_PROMETHEUS=true export CDI_E2E_FOCUS=Destructive diff --git a/automation/istio.sh b/automation/istio.sh index 17b9d19d14..f2f5077a86 100755 --- a/automation/istio.sh +++ b/automation/istio.sh @@ -18,7 +18,7 @@ # set -ex -export TARGET=k8s-1.29 +export TARGET=k8s-1.30 export KUBEVIRT_STORAGE=hpp export CDI_E2E_SKIP=Destructive export KUBEVIRT_DEPLOY_ISTIO=true diff --git a/automation/latest-hpp.sh b/automation/latest-hpp.sh index 21ac1124b2..1e3ab41dc2 100755 --- a/automation/latest-hpp.sh +++ b/automation/latest-hpp.sh @@ -18,7 +18,7 @@ # set -ex -export TARGET=k8s-1.29 +export TARGET=k8s-1.30 export KUBEVIRT_STORAGE=hpp export CDI_E2E_SKIP=Destructive automation/test.sh diff --git a/automation/nfs.sh b/automation/nfs.sh index 55d5edb6c7..074754a8eb 100755 --- a/automation/nfs.sh +++ b/automation/nfs.sh @@ -18,7 +18,7 @@ # set -ex -export TARGET=k8s-1.29 +export TARGET=k8s-1.30 export KUBEVIRT_DEPLOY_NFS_CSI=true export KUBEVIRT_STORAGE=nfs export CDI_E2E_SKIP=Destructive diff --git a/automation/non-csi-hpp.sh b/automation/non-csi-hpp.sh index a3b596f1ae..c0035b32a3 100755 --- a/automation/non-csi-hpp.sh +++ b/automation/non-csi-hpp.sh @@ -18,7 +18,7 @@ # set -ex -export TARGET=k8s-1.29 +export TARGET=k8s-1.30 export KUBEVIRT_STORAGE=hpp export HPP_CLASSIC=true export CDI_E2E_SKIP=Destructive diff --git a/automation/previous-hpp.sh b/automation/previous-hpp.sh index 943ecfc49d..52dc477276 100755 --- a/automation/previous-hpp.sh +++ b/automation/previous-hpp.sh @@ -18,7 +18,7 @@ # set -ex -export TARGET=k8s-1.28 +export TARGET=k8s-1.29 export KUBEVIRT_STORAGE=hpp export KUBEVIRT_DEPLOY_PROMETHEUS=true export CDI_E2E_SKIP=Destructive diff --git a/automation/upgrade.sh b/automation/upgrade.sh index 2001f88d8f..a8b96a2538 100755 --- a/automation/upgrade.sh +++ b/automation/upgrade.sh @@ -18,7 +18,7 @@ # set -ex -export TARGET=k8s-1.29 +export TARGET=k8s-1.30 export KUBEVIRT_STORAGE=hpp export MULTI_UPGRADE=true export CDI_E2E_SKIP=Destructive diff --git a/cluster-sync/sync.sh b/cluster-sync/sync.sh index da0cc9c2d0..6f9e2ae41f 100755 --- a/cluster-sync/sync.sh +++ b/cluster-sync/sync.sh @@ -275,12 +275,6 @@ if [ "${KUBEVIRT_PROVIDER}" != "external" ]; then configure_uploadproxy_override # Tell prometheus to watch our namespace configure_prometheus - if [ "$KUBEVIRT_STORAGE" == "nfs" ]; then - # nfs writing started to consistently breach the dirty_ratio, causing OOMKills - # we think the problem sits somewhere around the fsync calls to writeout to nfs being slow/failing - # https://github.com/kubevirt/containerized-data-importer/pull/3023#issuecomment-1913529241 - _kubectl patch cdi ${CR_NAME} --type merge -p '{"spec":{"config":{"podResourceRequirements": {"limits": {"cpu": "750m", "memory": "1Gi"}, "requests": {"cpu": "100m", "memory": "60M"}}}}}' - fi fi # Grab all the CDI crds so we can check if they are structural schemas diff --git a/cluster-up/check.sh b/cluster-up/check.sh index dc64452aa4..6d7348bcf2 100755 --- a/cluster-up/check.sh +++ b/cluster-up/check.sh @@ -38,7 +38,7 @@ function is_enabled() { if [ "$1" == "1" ]; then return 0 fi - if [ "$1" == "Y" ] || [ "$1" == "y"]; then + if [ "$1" == "Y" ] || [ "$1" == "y" ]; then return 0 fi return 1 diff --git a/cluster-up/cluster/ephemeral-provider-common.sh b/cluster-up/cluster/ephemeral-provider-common.sh index 577131c479..808e7b79a8 100644 --- a/cluster-up/cluster/ephemeral-provider-common.sh +++ b/cluster-up/cluster/ephemeral-provider-common.sh @@ -87,7 +87,7 @@ function _registry_volume() { function _add_common_params() { # shellcheck disable=SC2155 - local params="--nodes ${KUBEVIRT_NUM_NODES} --memory ${KUBEVIRT_MEMORY_SIZE} --cpu 6 --secondary-nics ${KUBEVIRT_NUM_SECONDARY_NICS} --random-ports --background --prefix $provider_prefix ${KUBEVIRT_PROVIDER} ${KUBEVIRT_PROVIDER_EXTRA_ARGS}" + local params="--nodes ${KUBEVIRT_NUM_NODES} --memory ${KUBEVIRT_MEMORY_SIZE} --numa ${KUBEVIRT_NUM_NUMA_NODES} --cpu ${KUBEVIRT_NUM_VCPU} --secondary-nics ${KUBEVIRT_NUM_SECONDARY_NICS} --random-ports --background --prefix $provider_prefix ${KUBEVIRT_PROVIDER} ${KUBEVIRT_PROVIDER_EXTRA_ARGS}" params=" --dns-port $KUBEVIRT_DNS_HOST_PORT $params" @@ -169,6 +169,10 @@ function _add_common_params() { params=" --hugepages-2m $KUBEVIRT_HUGEPAGES_2M $params" fi + if [ -n "$KUBEVIRT_HUGEPAGES_1G" ]; then + params=" --hugepages-1g $KUBEVIRT_HUGEPAGES_1G $params" + fi + if [ -n "$KUBEVIRT_REALTIME_SCHEDULER" ]; then params=" --enable-realtime-scheduler $params" fi diff --git a/cluster-up/cluster/k8s-1.27/provider.sh b/cluster-up/cluster/k8s-1.31/provider.sh similarity index 100% rename from cluster-up/cluster/k8s-1.27/provider.sh rename to cluster-up/cluster/k8s-1.31/provider.sh diff --git a/cluster-up/cluster/k8s-provider-common.sh b/cluster-up/cluster/k8s-provider-common.sh index c70a4bf8d9..cec5dd2880 100644 --- a/cluster-up/cluster/k8s-provider-common.sh +++ b/cluster-up/cluster/k8s-provider-common.sh @@ -65,6 +65,15 @@ function deploy_cnao() { fi } +function deploy_kwok() { + if [[ ${KUBEVIRT_DEPLOY_KWOK} == "true" ]]; then + $kubectl create -f /opt/kwok/kwok.yaml + $kubectl create -f /opt/kwok/stage-fast.yaml + + $kubectl apply -k /opt/kwok/kubevirt + fi +} + function create_network_addons_config() { local nac="/opt/cnao/network-addons-config-example.cr.yaml" if [ "$KUBEVIRT_WITH_MULTUS_V3" == "true" ]; then @@ -144,8 +153,10 @@ function deploy_cdi() { $ssh node01 -- 'sudo sed --regexp-extended -i s/v[0-9]+\.[0-9]+\.[0-9]+\(.*\)?$/'"$KUBEVIRT_CUSTOM_CDI_VERSION"'/g /opt/cdi-*-operator.yaml' fi - $kubectl create -f /opt/cdi-*-operator.yaml - $kubectl create -f /opt/cdi-*-cr.yaml + LATEST_CDI_OPERATOR=$($ssh node01 -- 'ls -rt /opt/cdi-*-operator.yaml | tail -n 1') + LATEST_CDI_CR=$($ssh node01 -- 'ls -rt /opt/cdi-*-cr.yaml | tail -n 1') + $kubectl create -f $LATEST_CDI_OPERATOR + $kubectl create -f $LATEST_CDI_CR fi } @@ -166,6 +177,50 @@ function configure_prometheus() { fi } +function deploy_aaq() { + if [ "$KUBEVIRT_DEPLOY_AAQ" == "true" ]; then + if [ -n "${KUBEVIRT_CUSTOM_AAQ_VERSION}" ]; then + $ssh node01 -- 'sudo sed --regexp-extended -i s/v[0-9]+\.[0-9]+\.[0-9]+\(.*\)?$/'"$KUBEVIRT_CUSTOM_AAQ_VERSION"'/g /opt/aaq/aaq-*-operator.yaml' + fi + + $kubectl create -f /opt/aaq/aaq-*-operator.yaml + $kubectl create -f /opt/aaq/aaq-*-cr.yaml + fi +} + +function wait_for_aaq_ready() { + if [ "$KUBEVIRT_DEPLOY_AAQ" == "true" ]; then + while [ "$($kubectl get pods --namespace aaq | grep -c 'aaq-')" -lt 4 ]; do + $kubectl get pods --namespace aaq + sleep 10 + done + $kubectl wait --for=condition=Ready pod --timeout=180s --all --namespace aaq + fi +} + +function wait_for_kwok_ready() { + if [ "KUBEVIRT_DEPLOY_KWOK" == "true" ]; then + $kubectl wait deployment -n kube-system kwok-controller --for condition=Available --timeout=200s + fi +} + +function configure_cpu_manager() { + if [ ${KUBEVIRT_CPU_MANAGER_POLICY} == "static" ]; then + for node in $($kubectl get nodes -l "node-role.kubernetes.io/worker" --no-headers -o custom-columns=":metadata.name" | tr -d '\r'); do + # FIXME Replace with kubelet config drop ins once all providers are using k8s >= 1.28 + # https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/#kubelet-conf-d + $kubectl drain ${node} + $ssh ${node} -- sudo systemctl stop kubelet + # FIXME ${ssh} is broken when using HereDocs, fix and replace this mess if possible. + # https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/#configuration + $ssh ${node} -- "sudo rm -f /var/lib/kubelet/cpu_manager_state && sudo echo -e 'cpuManagerPolicy: static\nkubeReserved:\n cpu: \"1\"\n memory: \"1Gi\"\ncpuManagerPolicyOptions:\n full-pcpus-only: \"true\"' | sudo tee -a /var/lib/kubelet/config.yaml && sudo sed -i 's/cpuManagerReconcilePeriod\:\ 0s/cpuManagerReconcilePeriod\:\ 5s/g' /var/lib/kubelet/config.yaml" + $ssh ${node} -- sudo systemctl start kubelet + $kubectl label --overwrite node/${node} cpumanager=true + $kubectl uncordon ${node} + done + fi +} + function up() { params=$(_add_common_params) if echo "$params" | grep -q ERROR; then @@ -207,13 +262,16 @@ function up() { configure_prometheus configure_memory_overcommitment_behavior + configure_cpu_manager deploy_cnao deploy_multus deploy_istio deploy_cdi + deploy_aaq + deploy_kwok - until wait_for_cnao_ready && wait_for_istio_ready && wait_for_cdi_ready && wait_for_multus_ready; do + until wait_for_cnao_ready && wait_for_istio_ready && wait_for_cdi_ready && wait_for_multus_ready && wait_for_aaq_ready && wait_for_kwok_ready; do echo "Waiting for cluster components..." sleep 5 done diff --git a/cluster-up/cluster/kind-1.27-vgpu/provider.sh b/cluster-up/cluster/kind-1.27-vgpu/provider.sh index 9d41c47172..3ff5ade6f0 100755 --- a/cluster-up/cluster/kind-1.27-vgpu/provider.sh +++ b/cluster-up/cluster/kind-1.27-vgpu/provider.sh @@ -34,9 +34,6 @@ function configure_registry_proxy() { } function up() { - # load the vfio_mdev module - /usr/sbin/modprobe vfio_mdev - # print hardware info for easier debugging based on logs echo 'Available cards' ${CRI_BIN} run --rm --cap-add=SYS_RAWIO quay.io/phoracek/lspci@sha256:0f3cacf7098202ef284308c64e3fc0ba441871a846022bb87d65ff130c79adb1 sh -c "lspci -k | grep -EA2 'VGA|3D'" diff --git a/cluster-up/cluster/kind-1.30-vgpu/config_vgpu_cluster.sh b/cluster-up/cluster/kind-1.30-vgpu/config_vgpu_cluster.sh new file mode 100755 index 0000000000..87087c6de4 --- /dev/null +++ b/cluster-up/cluster/kind-1.30-vgpu/config_vgpu_cluster.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +[ $(id -u) -ne 0 ] && echo "FATAL: this script requires sudo privileges" >&2 && exit 1 + +set -xe + +SCRIPT_PATH=$(dirname "$(realpath "$0")") + +source ${SCRIPT_PATH}/vgpu-node/node.sh +echo "_kubectl: " ${_kubectl} +echo "KUBEVIRTCI_PATH: " ${KUBEVIRTCI_PATH} +source ${KUBEVIRTCI_PATH}/cluster/kind/common.sh +echo "_kubectl: " ${_kubectl} + +nodes=($(_kubectl get nodes -o custom-columns=:.metadata.name --no-headers)) +node::remount_sysfs "${nodes[*]}" +node::discover_host_gpus + +_kubectl get nodes diff --git a/cluster-up/cluster/kind-1.30-vgpu/conformance.json b/cluster-up/cluster/kind-1.30-vgpu/conformance.json new file mode 100644 index 0000000000..2ff6e83a5b --- /dev/null +++ b/cluster-up/cluster/kind-1.30-vgpu/conformance.json @@ -0,0 +1,47 @@ +{ + "Description": "DEFAULT", + "UUID": "", + "Version": "v0.56.9", + "ResultsDir": "/tmp/sonobuoy/results", + "Resources": null, + "Filters": { + "Namespaces": ".*", + "LabelSelector": "" + }, + "Limits": { + "PodLogs": { + "Namespaces": "kube-system", + "SonobuoyNamespace": true, + "FieldSelectors": [], + "LabelSelector": "", + "Previous": false, + "SinceSeconds": null, + "SinceTime": null, + "Timestamps": false, + "TailLines": null, + "LimitBytes": null + } + }, + "QPS": 30, + "Burst": 50, + "Server": { + "bindaddress": "0.0.0.0", + "bindport": 8080, + "advertiseaddress": "", + "timeoutseconds": 21600 + }, + "Plugins": null, + "PluginSearchPath": [ + "./plugins.d", + "/etc/sonobuoy/plugins.d", + "~/sonobuoy/plugins.d" + ], + "Namespace": "sonobuoy", + "WorkerImage": "sonobuoy/sonobuoy:v0.56.9", + "ImagePullPolicy": "IfNotPresent", + "ImagePullSecrets": "", + "AggregatorPermissions": "clusterAdmin", + "ServiceAccountName": "sonobuoy-serviceaccount", + "ProgressUpdatesPort": "8099", + "SecurityContextMode": "nonroot" +} diff --git a/cluster-up/cluster/kind-1.30-vgpu/image b/cluster-up/cluster/kind-1.30-vgpu/image new file mode 100644 index 0000000000..babda2aca0 --- /dev/null +++ b/cluster-up/cluster/kind-1.30-vgpu/image @@ -0,0 +1 @@ +kindest/node:v1.30.0@sha256:047357ac0cfea04663786a612ba1eaba9702bef25227a794b52890dd8bcd692e diff --git a/cluster-up/cluster/kind-1.30-vgpu/provider.sh b/cluster-up/cluster/kind-1.30-vgpu/provider.sh new file mode 100755 index 0000000000..3ff5ade6f0 --- /dev/null +++ b/cluster-up/cluster/kind-1.30-vgpu/provider.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +set -e + +DEFAULT_CLUSTER_NAME="vgpu" +DEFAULT_HOST_PORT=5000 +ALTERNATE_HOST_PORT=5001 +export CLUSTER_NAME=${CLUSTER_NAME:-$DEFAULT_CLUSTER_NAME} + +if [ $CLUSTER_NAME == $DEFAULT_CLUSTER_NAME ]; then + export HOST_PORT=$DEFAULT_HOST_PORT +else + export HOST_PORT=$ALTERNATE_HOST_PORT +fi + +function set_kind_params() { + version=$(cat cluster-up/cluster/$KUBEVIRT_PROVIDER/version) + export KIND_VERSION="${KIND_VERSION:-$version}" + + image=$(cat cluster-up/cluster/$KUBEVIRT_PROVIDER/image) + export KIND_NODE_IMAGE="${KIND_NODE_IMAGE:-$image}" +} + +function configure_registry_proxy() { + [ "$CI" != "true" ] && return + + echo "Configuring cluster nodes to work with CI mirror-proxy..." + + local -r ci_proxy_hostname="docker-mirror-proxy.kubevirt-prow.svc" + local -r kind_binary_path="${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/.kind" + local -r configure_registry_proxy_script="${KUBEVIRTCI_PATH}/cluster/kind/configure-registry-proxy.sh" + + KIND_BIN="$kind_binary_path" PROXY_HOSTNAME="$ci_proxy_hostname" $configure_registry_proxy_script +} + +function up() { + # print hardware info for easier debugging based on logs + echo 'Available cards' + ${CRI_BIN} run --rm --cap-add=SYS_RAWIO quay.io/phoracek/lspci@sha256:0f3cacf7098202ef284308c64e3fc0ba441871a846022bb87d65ff130c79adb1 sh -c "lspci -k | grep -EA2 'VGA|3D'" + echo "" + + cp $KIND_MANIFESTS_DIR/kind.yaml ${KUBEVIRTCI_CONFIG_PATH}/$KUBEVIRT_PROVIDER/kind.yaml + _add_extra_mounts + kind_up + + configure_registry_proxy + + # remove the rancher.io kind default storageClass + _kubectl delete sc standard + + ${KUBEVIRTCI_PATH}/cluster/$KUBEVIRT_PROVIDER/config_vgpu_cluster.sh + + echo "$KUBEVIRT_PROVIDER cluster '$CLUSTER_NAME' is ready" +} + +set_kind_params + +source ${KUBEVIRTCI_PATH}/cluster/kind/common.sh diff --git a/cluster-up/cluster/kind-1.30-vgpu/version b/cluster-up/cluster/kind-1.30-vgpu/version new file mode 100644 index 0000000000..ca222b7cf3 --- /dev/null +++ b/cluster-up/cluster/kind-1.30-vgpu/version @@ -0,0 +1 @@ +0.23.0 diff --git a/cluster-up/cluster/kind-1.30-vgpu/vgpu-node/node.sh b/cluster-up/cluster/kind-1.30-vgpu/vgpu-node/node.sh new file mode 100644 index 0000000000..4a69c54a09 --- /dev/null +++ b/cluster-up/cluster/kind-1.30-vgpu/vgpu-node/node.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +function node::discover_host_gpus() { + local -r gpu_types=( $(find /sys/class/mdev_bus/*/mdev_supported_types) ) + [ "${#gpu_types[@]}" -eq 0 ] && echo "FATAL: Could not find available GPUs on host" >&2 && return 1 + + local gpu_addr + local gpu_addresses=() + for path in "${gpu_types}"; do + gpu_addr="${gpu_types#/sys/class/mdev_bus/}" + gpu_addr=${gpu_addr%/*} + + gpu_addresses+=( $gpu_addr ) + done + + echo "${gpu_addresses[@]}" +} + +function node::remount_sysfs() { + local -r nodes_array=($1) + local node_exec + + for node in "${nodes_array[@]}"; do + + # KIND mounts sysfs as read-only by default, remount as R/W" + node_exec="${CRI_BIN} exec $node" + $node_exec mount -o remount,rw /sys + $node_exec chmod 666 /dev/vfio/vfio + + done +} + diff --git a/cluster-up/cluster/kind-ovn/README.md b/cluster-up/cluster/kind-ovn/README.md new file mode 100644 index 0000000000..11630ef883 --- /dev/null +++ b/cluster-up/cluster/kind-ovn/README.md @@ -0,0 +1,29 @@ +# OVN K8S in a Kind cluster + +Provides a k8s cluster that runs using [KinD](https://github.com/kubernetes-sigs/kind) +The cluster is completely ephemeral and is recreated on every cluster restart. The KubeVirt containers are built on the +local machine and are then pushed to a registry which is exposed at +`localhost:5000`. + +## Bringing the cluster up + +```bash +export KUBEVIRT_PROVIDER=kind-ovn +make cluster-up +``` + +## Bringing the cluster down + +```bash +export KUBEVIRT_PROVIDER=kind-ovn +make cluster-down +``` + +## FAQ + +In case the cluster deployment fails, you need to make sure you have enough watches +add those to /etc/sysctl.conf, and apply it `sysctl -p /etc/sysctl.conf`. +``` +sysctl fs.inotify.max_user_watches=1048576 +sysctl fs.inotify.max_user_instances=512 +``` diff --git a/cluster-up/cluster/kind-ovn/install-ovn.sh b/cluster-up/cluster/kind-ovn/install-ovn.sh new file mode 100755 index 0000000000..f4a448eda1 --- /dev/null +++ b/cluster-up/cluster/kind-ovn/install-ovn.sh @@ -0,0 +1,47 @@ +#!/bin/bash -e +# +# This file is part of the KubeVirt project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright 2024 Red Hat, Inc. +# + +export OVNK_COMMIT=c77ee8c38c6a6d9e55131a1272db5fad5b606e44 + +OVNK_REPO='https://github.com/ovn-org/ovn-kubernetes.git' +CLUSTER_PATH=${CLUSTER_PATH:-"${KUBEVIRTCI_CONFIG_PATH}/${KUBEVIRT_PROVIDER}/_ovnk"} + +function cluster::_get_repo() { + git --git-dir ${CLUSTER_PATH}/.git config --get remote.origin.url +} + +function cluster::_get_sha() { + git --git-dir ${CLUSTER_PATH}/.git rev-parse HEAD +} + +function cluster::install() { + if [ -d ${CLUSTER_PATH} ]; then + if [ $(cluster::_get_repo) != ${OVNK_REPO} -o $(cluster::_get_sha) != ${OVNK_COMMIT} ]; then + rm -rf ${CLUSTER_PATH} + fi + fi + + if [ ! -d ${CLUSTER_PATH} ]; then + git clone ${OVNK_REPO} ${CLUSTER_PATH} + ( + cd ${CLUSTER_PATH} + git checkout ${OVNK_COMMIT} + ) + fi +} diff --git a/cluster-up/cluster/kind-ovn/provider.sh b/cluster-up/cluster/kind-ovn/provider.sh new file mode 100755 index 0000000000..2e15a33872 --- /dev/null +++ b/cluster-up/cluster/kind-ovn/provider.sh @@ -0,0 +1,88 @@ +#!/bin/bash -ex +# This file is part of the KubeVirt project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright 2024 Red Hat, Inc. +# + +KIND_VERSION=0.19.0 +export KIND_IMAGE=kindest/node +export K8S_VERSION=v1.28.0@sha256:dad5a6238c5e41d7cac405fae3b5eda2ad1de6f1190fa8bfc64ff5bb86173213 + +KIND_PATH=${KIND_PATH:-"${KUBEVIRTCI_CONFIG_PATH}/${KUBEVIRT_PROVIDER}/_kind"} +CLUSTER_PATH=${CLUSTER_PATH:-"${KUBEVIRTCI_CONFIG_PATH}/${KUBEVIRT_PROVIDER}/_ovnk"} +CLUSTER_NAME=${KUBEVIRT_PROVIDER} + +function calculate_mtu() { + overlay_overhead=58 + current_mtu=$(cat /sys/class/net/$(ip route | grep "default via" | head -1 | awk '{print $5}')/mtu) + expr $current_mtu - $overlay_overhead +} + +MTU=${MTU:-$(calculate_mtu)} + +PLATFORM=$(uname -m) +case ${PLATFORM} in +x86_64* | i?86_64* | amd64*) + ARCH="amd64" + ;; +aarch64* | arm64*) + ARCH="arm64" + ;; +*) + echo "invalid Arch, only support x86_64, aarch64" + exit 1 + ;; +esac + +function fetch_kind() { + mkdir -p $KIND_PATH + current_kind_version=$($KIND_PATH/kind --version |& awk '{print $3}') + if [[ $current_kind_version != $KIND_VERSION ]]; then + echo "Downloading kind v$KIND_VERSION" + curl -LSs https://github.com/kubernetes-sigs/kind/releases/download/v$KIND_VERSION/kind-linux-${ARCH} -o "$KIND_PATH/kind" + chmod +x "$KIND_PATH/kind" + fi + export PATH=$KIND_PATH:$PATH +} + +function prepare_config() { + echo "STEP: Prepare provider config" + cat >$KUBEVIRTCI_CONFIG_PATH/$KUBEVIRT_PROVIDER/config-provider-$KUBEVIRT_PROVIDER.sh <