diff --git a/Makefile b/Makefile index 6f9185175a..dac0529e39 100644 --- a/Makefile +++ b/Makefile @@ -58,6 +58,7 @@ BIN_DIR := bin BUILD_DIR := .build TEST_DIR := test VCSIM_DIR := test/infrastructure/vcsim +NETOP_DIR := test/infrastructure/net-operator TOOLS_DIR := hack/tools TOOLS_BIN_DIR := $(abspath $(TOOLS_DIR)/$(BIN_DIR)) FLAVOR_DIR := $(ROOT_DIR)/templates @@ -223,9 +224,15 @@ VM_OPERATOR_IMAGE_NAME ?= extra/vm-operator VM_OPERATOR_CONTROLLER_IMG ?= $(STAGING_REGISTRY)/$(VM_OPERATOR_IMAGE_NAME) VM_OPERATOR_DIR := test/infrastructure/vm-operator VM_OPERATOR_TMP_DIR ?= vm-operator.tmp -VM_OPERATOR_VERSION ?= v1.8.1 +# note: this is the commit from 1.8.6 tag +VM_OPERATOR_COMMIT ?= de75746a9505ef3161172d99b735d6593c54f0c5 +VM_OPERATOR_VERSION ?= v1.8.6-0-gde75746a VM_OPERATOR_ALL_ARCH = amd64 arm64 +# net operator +NET_OPERATOR_IMAGE_NAME ?= cluster-api-net-operator +NET_OPERATOR_IMG ?= $(STAGING_REGISTRY)/$(NET_OPERATOR_IMAGE_NAME) + # It is set by Prow GIT_TAG, a git-based tag of the form vYYYYMMDD-hash, e.g., v20210120-v0.3.10-308-gc61521971 TAG ?= dev @@ -255,6 +262,7 @@ VCSIM_CRD_ROOT ?= $(VCSIM_DIR)/config/crd/bases WEBHOOK_ROOT ?= $(MANIFEST_ROOT)/webhook RBAC_ROOT ?= $(MANIFEST_ROOT)/rbac VCSIM_RBAC_ROOT ?= $(VCSIM_DIR)/config/rbac +NETOP_RBAC_ROOT ?= $(NETOP_DIR)/config/rbac VERSION ?= $(shell cat clusterctl-settings.json | jq .config.nextVersion -r) OVERRIDES_DIR := $(HOME)/.cluster-api/overrides/infrastructure-vsphere/$(VERSION) @@ -294,16 +302,21 @@ generate-manifests: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc. paths=./apis/vmware/v1beta1 \ crd:crdVersions=v1 \ output:crd:dir=$(SUPERVISOR_CRD_ROOT) - # vm-operator crds are loaded to be used for integration tests. + # vm-operator crds are used for test. $(CONTROLLER_GEN) \ paths=github.com/vmware-tanzu/vm-operator/api/v1alpha1/... \ crd:crdVersions=v1 \ output:crd:dir=$(VMOP_CRD_ROOT) + # net-operator is used for tests + $(CONTROLLER_GEN) \ + paths=./$(NETOP_DIR)/controllers/... \ + output:rbac:dir=$(NETOP_RBAC_ROOT) \ + rbac:roleName=manager-role # vcsim crds are used for tests. $(CONTROLLER_GEN) \ - paths=./$(VCSIM_DIR)/api/v1alpha1 \ - crd:crdVersions=v1 \ - output:crd:dir=$(VCSIM_CRD_ROOT) + paths=./$(VCSIM_DIR)/api/v1alpha1 \ + crd:crdVersions=v1 \ + output:crd:dir=$(VCSIM_CRD_ROOT) $(CONTROLLER_GEN) \ paths=./$(VCSIM_DIR)/ \ paths=./$(VCSIM_DIR)/controllers/... \ @@ -317,8 +330,8 @@ generate-go-deepcopy: $(CONTROLLER_GEN) ## Generate deepcopy go code for core object:headerFile=./hack/boilerplate/boilerplate.generatego.txt \ paths=./apis/... $(CONTROLLER_GEN) \ - object:headerFile=./hack/boilerplate/boilerplate.generatego.txt \ - paths=./$(VCSIM_DIR)/api/... + object:headerFile=./hack/boilerplate/boilerplate.generatego.txt \ + paths=./$(VCSIM_DIR)/api/... .PHONY: generate-go-conversions generate-go-conversions: $(CONTROLLER_GEN) $(CONVERSION_GEN) ## Runs Go related generate targets @@ -537,6 +550,15 @@ docker-build-vcsim: docker-pull-prerequisites ## Build the docker image for vcsi $(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./$(VCSIM_DIR)/config/default/manager_pull_policy.yaml"; \ fi +.PHONY: docker-build-net-operator +docker-build-net-operator: docker-pull-prerequisites ## Build the docker image for net-operator controller manager +## reads Dockerfile from stdin to avoid an incorrectly cached Dockerfile (https://github.com/moby/buildkit/issues/1368) + cat $(NETOP_DIR)/Dockerfile | DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg ldflags="$(LDFLAGS)" . -t $(NET_OPERATOR_IMG)-$(ARCH):$(TAG) --file - + @if [ "${DOCKER_BUILD_MODIFY_MANIFESTS}" = "true" ]; then \ + $(MAKE) set-manifest-image MANIFEST_IMG=$(NET_OPERATOR_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./$(NETOP_DIR)/config/default/manager_image_patch.yaml"; \ + $(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./$(NETOP_DIR)/config/default/manager_pull_policy.yaml"; \ + fi + ## -------------------------------------- ## Testing ## -------------------------------------- @@ -584,6 +606,7 @@ e2e-images: ## Build the e2e manager image # also the same settings must exist in e2e.sh $(MAKE) REGISTRY=gcr.io/k8s-staging-capi-vsphere PULL_POLICY=IfNotPresent TAG=dev docker-build $(MAKE) REGISTRY=gcr.io/k8s-staging-capi-vsphere PULL_POLICY=IfNotPresent TAG=dev docker-build-vcsim + $(MAKE) REGISTRY=gcr.io/k8s-staging-capi-vsphere PULL_POLICY=IfNotPresent TAG=dev docker-build-net-operator .PHONY: e2e e2e: e2e-images generate-e2e-templates @@ -783,13 +806,15 @@ vm-operator-checkout: @if [ -z "${VM_OPERATOR_VERSION}" ]; then echo "VM_OPERATOR_VERSION is not set"; exit 1; fi @if [ -d "$(VM_OPERATOR_TMP_DIR)" ]; then \ echo "$(VM_OPERATOR_TMP_DIR) exists, skipping clone"; \ - cd "$(VM_OPERATOR_TMP_DIR)"; \ - if [ "$$(git describe --match "v[0-9]*")" != "$(VM_OPERATOR_VERSION)" ]; then \ - echo "ERROR: checked out version $$(git describe --match "v[0-9]*") does not match expected version $(VM_OPERATOR_VERSION)"; \ - exit 1; \ - fi \ else \ - git clone --depth 1 --branch "$(VM_OPERATOR_VERSION)" "https://github.com/vmware-tanzu/vm-operator.git" "$(VM_OPERATOR_TMP_DIR)"; \ + git clone "https://github.com/vmware-tanzu/vm-operator.git" "$(VM_OPERATOR_TMP_DIR)"; \ + cd "$(VM_OPERATOR_TMP_DIR)"; \ + git checkout "$(VM_OPERATOR_COMMIT)"; \ + fi + @cd "$(ROOT_DIR)/$(VM_OPERATOR_TMP_DIR)"; \ + if [ "$$(git describe --dirty 2> /dev/null)" != "$(VM_OPERATOR_VERSION)" ]; then \ + echo "ERROR: checked out version $$(git describe --dirty 2> /dev/null) does not match expected version $(VM_OPERATOR_VERSION)"; \ + exit 1; \ fi .PHONY: vm-operator-manifest-build diff --git a/hack/e2e.sh b/hack/e2e.sh index 20c3a5ab7d..a9c18bbf24 100755 --- a/hack/e2e.sh +++ b/hack/e2e.sh @@ -83,7 +83,7 @@ export VSPHERE_SSH_AUTHORIZED_KEY="${VM_SSH_PUB_KEY:-}" export VSPHERE_SSH_PRIVATE_KEY="/root/ssh/.private-key/private-key" export E2E_CONF_FILE="${REPO_ROOT}/test/e2e/config/vsphere.yaml" export E2E_CONF_OVERRIDE_FILE="" -export E2E_VM_OPERATOR_VERSION="${VM_OPERATOR_VERSION:-v1.8.1}" +export E2E_VM_OPERATOR_VERSION="${VM_OPERATOR_VERSION:-v1.8.6-0-gde75746a}" export ARTIFACTS="${ARTIFACTS:-${REPO_ROOT}/_artifacts}" export DOCKER_IMAGE_TAR="/tmp/images/image.tar" export GC_KIND="false" diff --git a/internal/test/helpers/vcsim/model.go b/internal/test/helpers/vcsim/model.go index eadc865f0a..4e0ab41484 100644 --- a/internal/test/helpers/vcsim/model.go +++ b/internal/test/helpers/vcsim/model.go @@ -89,3 +89,8 @@ func NetworkFolderName(datacenter int) string { func NetworkPath(datacenter int, network string) string { return fmt.Sprintf("/%s/%s", NetworkFolderName(datacenter), network) } + +// DistributedPortGroupName provide a function to compute vcsim distribute port group names in a datacenter. +func DistributedPortGroupName(datacenter int, distributedPortGroup int) string { + return fmt.Sprintf("%s_DVPG%d", DatacenterName(datacenter), distributedPortGroup) +} diff --git a/internal/test/helpers/vcsim/model_test.go b/internal/test/helpers/vcsim/model_test.go index 53131fc72a..bf27658f52 100644 --- a/internal/test/helpers/vcsim/model_test.go +++ b/internal/test/helpers/vcsim/model_test.go @@ -28,6 +28,7 @@ func Test_vcsim_NamesAndPath(t *testing.T) { datacenter := 5 cluster := 3 datastore := 7 + distributedPortGroup := 4 g.Expect(DatacenterName(datacenter)).To(Equal("DC5")) g.Expect(ClusterName(datacenter, cluster)).To(Equal("DC5_C3")) @@ -39,4 +40,5 @@ func Test_vcsim_NamesAndPath(t *testing.T) { g.Expect(VMPath(datacenter, "my-mv")).To(Equal("/DC5/vm/my-mv")) g.Expect(NetworkFolderName(datacenter)).To(Equal("DC5/network")) g.Expect(NetworkPath(datacenter, "my-network")).To(Equal("/DC5/network/my-network")) + g.Expect(DistributedPortGroupName(datacenter, distributedPortGroup)).To(Equal("DC5_DVPG4")) } diff --git a/test/e2e/config/vsphere.yaml b/test/e2e/config/vsphere.yaml index 23ce28a712..a40e9cc900 100644 --- a/test/e2e/config/vsphere.yaml +++ b/test/e2e/config/vsphere.yaml @@ -20,7 +20,9 @@ images: loadBehavior: mustLoad - name: gcr.io/k8s-staging-capi-vsphere/cluster-api-vcsim-controller-{ARCH}:dev loadBehavior: mustLoad - - name: gcr.io/k8s-staging-capi-vsphere/extra/vm-operator:v1.8.1 + - name: gcr.io/k8s-staging-capi-vsphere/cluster-api-net-operator-{ARCH}:dev + loadBehavior: mustLoad + - name: gcr.io/k8s-staging-capi-vsphere/extra/vm-operator:v1.8.6-0-gde75746a loadBehavior: tryLoad - name: quay.io/jetstack/cert-manager-cainjector:v1.12.2 loadBehavior: tryLoad @@ -211,9 +213,9 @@ providers: - name: vm-operator type: RuntimeExtensionProvider # vm-operator isn't a provider, but we fake it is so it can be handled by the clusterctl machinery. versions: - - name: v1.8.1 + - name: v1.8.6-0-gde75746a # Use manifest from source files - value: "https://storage.googleapis.com/artifacts.k8s-staging-capi-vsphere.appspot.com/vm-operator/v1.8.1.yaml" + value: "https://storage.googleapis.com/artifacts.k8s-staging-capi-vsphere.appspot.com/vm-operator/v1.8.6-0-gde75746a.yaml" type: "url" contract: v1beta1 files: @@ -222,6 +224,19 @@ providers: - old: "imagePullPolicy: Always" new: "imagePullPolicy: IfNotPresent" + - name: net-operator + type: RuntimeExtensionProvider # net-operator isn't a provider, but we fake it is so it can be handled by the clusterctl machinery. + versions: + - name: v1.10.99 + # Use manifest from source files + value: ../../../../cluster-api-provider-vsphere/test/infrastructure/net-operator/config/default + contract: v1beta1 + files: + - sourcePath: "../data/shared/capv/main/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + variables: # Ensure all Kubernetes versions used here are covered in patch-vsphere-template.yaml KUBERNETES_VERSION: "v1.29.0" @@ -247,8 +262,10 @@ variables: VSPHERE_MACHINE_CLASS_CPU: "4" VSPHERE_MACHINE_CLASS_MEMORY: "8Gi" VSPHERE_CONTENT_LIBRARY: "capv" + VSPHERE_CONTENT_LIBRARY_ITEMS: "ubuntu-2204-kube-v1.28.0,ubuntu-2204-kube-v1.29.0" VSPHERE_IMAGE_NAME: "ubuntu-2204-kube-v1.29.0" VSPHERE_NETWORK: "sddc-cgw-network-6" + VSPHERE_DISTRIBUTED_PORT_GROUP: "/SDDC-Datacenter/network/sddc-cgw-network-6" VSPHERE_TEMPLATE: "ubuntu-2204-kube-v1.29.0" FLATCAR_VSPHERE_TEMPLATE: "flatcar-stable-3602.2.3-kube-v1.29.0" VSPHERE_INSECURE_CSI: "true" diff --git a/test/e2e/e2e_setup_test.go b/test/e2e/e2e_setup_test.go index 26792e6390..2c18bef7d1 100644 --- a/test/e2e/e2e_setup_test.go +++ b/test/e2e/e2e_setup_test.go @@ -273,7 +273,7 @@ func setupNamespaceWithVMOperatorDependenciesVCenter(managementClusterProxy fram // NOTE: when running on vCenter the vm-operator automatically creates VirtualMachine objects for the content library. Items: []vcsimv1.ContentLibraryItemConfig{}, }, - NetworkName: e2eConfig.GetVariable("VSPHERE_NETWORK"), + DistributedPortGroupName: e2eConfig.GetVariable("VSPHERE_DISTRIBUTED_PORT_GROUP"), }, StorageClasses: []vcsimv1.StorageClass{ { @@ -291,6 +291,16 @@ func setupNamespaceWithVMOperatorDependenciesVCenter(managementClusterProxy fram }, } + items := e2eConfig.GetVariable("VSPHERE_CONTENT_LIBRARY_ITEMS") + if items != "" { + for _, i := range strings.Split(e2eConfig.GetVariable("VSPHERE_CONTENT_LIBRARY_ITEMS"), ",") { + dependenciesConfig.Spec.VCenter.ContentLibrary.Items = append(dependenciesConfig.Spec.VCenter.ContentLibrary.Items, vcsimv1.ContentLibraryItemConfig{ + Name: i, + ItemType: "ovf", + }) + } + } + err := vmoperator.ReconcileDependencies(ctx, c, dependenciesConfig) Expect(err).ToNot(HaveOccurred(), "Failed to reconcile VMOperatorDependencies") } diff --git a/test/framework/framework.go b/test/framework/framework.go index 2f20ca26f7..5eb482e708 100644 --- a/test/framework/framework.go +++ b/test/framework/framework.go @@ -122,6 +122,21 @@ func LoadE2EConfig(ctx context.Context, configPath string, configOverridesPath, break } } + + Byf("Dropping net-operator from the e2e config") + for i := range config.Providers { + if config.Providers[i].Name == "net-operator" { + config.Providers = append(config.Providers[:i], config.Providers[i+1:]...) + break + } + } + + for i := range config.Images { + if strings.Contains(config.Images[i].Name, "net-operator") { + config.Images = append(config.Images[:i], config.Images[i+1:]...) + break + } + } } else { // In case we are testing supervisor, change the folder we build manifest from Byf("Overriding source folder for vsphere provider to /config/supervisor in the e2e config") diff --git a/test/framework/vmoperator/vmoperator.go b/test/framework/vmoperator/vmoperator.go index adb16dd865..7a7c3e8a40 100644 --- a/test/framework/vmoperator/vmoperator.go +++ b/test/framework/vmoperator/vmoperator.go @@ -22,6 +22,7 @@ import ( "fmt" "net" "net/url" + "sort" "strings" "time" @@ -50,22 +51,34 @@ import ( const DefaultNamespace = "vmware-system-vmop" const ( - // NOTE: const below are copied from pkg/vmprovider/providers/vsphere/config/config.go. - // int the vm-operator project. - - providerConfigMapName = "vsphere.provider.config.vmoperator.vmware.com" - vcPNIDKey = "VcPNID" - vcPortKey = "VcPort" - vcCredsSecretNameKey = "VcCredsSecretName" //nolint:gosec - datacenterKey = "Datacenter" - resourcePoolKey = "ResourcePool" - folderKey = "Folder" - datastoreKey = "Datastore" - networkNameKey = "Network" - scRequiredKey = "StorageClassRequired" - useInventoryKey = "UseInventoryAsContentSource" - insecureSkipTLSVerifyKey = "InsecureSkipTLSVerify" - caFilePathKey = "CAFilePath" + // NOTE: ConfigMapName/ConfigMapKey values below must match what defined in pkg/vmprovider/providers/vsphere/config/config.go. + + configMapName = "vsphere.provider.config.vmoperator.vmware.com" + hostConfigMapKey = "VcPNID" // vcenter host + portConfigMapKey = "VcPort" + credentialSecretNameConfigMapKey = "VcCredsSecretName" //nolint:gosec + datacenterConfigMapKey = "Datacenter" + resourcePoolConfigMapKey = "ResourcePool" + folderConfigMapKey = "Folder" + storageClassRequiredConfigMapKey = "StorageClassRequired" + useInventoryConfigMapKey = "UseInventoryAsContentSource" + insecureSkipTLSVerifyConfigMapKey = "InsecureSkipTLSVerify" + + // Additional ConfigMapKey we are adding to the vm-operator config map for sake of convenience (not supported in vm-operator). + + serverURLConfigMapKey = "CAPV-TEST-Server" + datacenterNameConfigMapKey = "CAPV-TEST-DatacenterName" + distributedPortGroupConfigMapKey = "CAPV-TEST-PortGroup" + + // Const for the VcCredsSecret (hard-coded in vm-operator). + vmOperatorSecretName = "vsphere.provider.credentials.vmoperator.vmware.com" + + usernameSecretKey = "username" + passwordSecretKey = "password" + + // Additional key we are adding to the VcCredsSecret for sake of convenience (not supported in vm-operator). + + thumbprintSecretKey = "CAPV-TEST-Thumbprint" //nolint:gosec ) // ReconcileDependencies reconciles dependencies for the vm-operator. @@ -264,12 +277,15 @@ func ReconcileDependencies(ctx context.Context, c client.Client, dependenciesCon // This secret contains credentials to access vCenter the vm-operator acts on. secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: providerConfigMapName, // using the same name of the config map for consistency. + Name: vmOperatorSecretName, Namespace: config.Spec.OperatorRef.Namespace, }, Data: map[string][]byte{ - "username": []byte(config.Spec.VCenter.Username), - "password": []byte(config.Spec.VCenter.Password), + usernameSecretKey: []byte(config.Spec.VCenter.Username), + passwordSecretKey: []byte(config.Spec.VCenter.Password), + + // Additional key we are adding to the VcCredsSecret for sake of convenience (not supported in vm-operator) + thumbprintSecretKey: []byte(config.Spec.VCenter.Thumbprint), }, Type: corev1.SecretTypeOpaque, } @@ -302,22 +318,27 @@ func ReconcileDependencies(ctx context.Context, c client.Client, dependenciesCon providerConfigMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: providerConfigMapName, + Name: configMapName, Namespace: config.Spec.OperatorRef.Namespace, }, Data: map[string]string{ - caFilePathKey: "", // Leaving this empty because we don't have (yet) a solution to inject a CA file into the vm-operator pod. - datastoreKey: "", // It seems it is ok to leave it empty. - datacenterKey: datacenter.Reference().Value, - folderKey: folder.Reference().Value, - insecureSkipTLSVerifyKey: "true", // Using this given that we don't have (yet) a solution to inject a CA file into the vm-operator pod. - networkNameKey: config.Spec.VCenter.NetworkName, // It seems it is ok to leave it empty. - resourcePoolKey: resourcePool.Reference().Value, - scRequiredKey: "true", - useInventoryKey: "false", - vcCredsSecretNameKey: secret.Name, - vcPNIDKey: host, - vcPortKey: port, + // caFilePathConfigMapKey: "", // Leaving this empty because we don't have (yet) a solution to inject a CA file into the vm-operator pod. + // datastoreConfigMapKey: "", // It seems it is ok to leave it empty. + datacenterConfigMapKey: datacenter.Reference().Value, + folderConfigMapKey: folder.Reference().Value, + insecureSkipTLSVerifyConfigMapKey: "true", // Using this given that we don't have (yet) a solution to inject a CA file into the vm-operator pod. + // NetworkNameConfigMapKey: config.Spec.VCenter.NetworkName, // It seems it is ok to leave it empty. + resourcePoolConfigMapKey: resourcePool.Reference().Value, + storageClassRequiredConfigMapKey: "true", + useInventoryConfigMapKey: "false", + credentialSecretNameConfigMapKey: secret.Name, + hostConfigMapKey: host, + portConfigMapKey: port, + + // Additional key we are adding to the vm-operator config map for sake of convenience (not supported in vm-operator) + serverURLConfigMapKey: config.Spec.VCenter.ServerURL, + datacenterNameConfigMapKey: config.Spec.VCenter.Datacenter, + distributedPortGroupConfigMapKey: config.Spec.VCenter.DistributedPortGroupName, }, } _ = wait.PollUntilContextTimeout(ctx, 250*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { @@ -343,10 +364,10 @@ func ReconcileDependencies(ctx context.Context, c client.Client, dependenciesCon for _, vmc := range config.Spec.VirtualMachineClasses { vmClass := &vmoprv1.VirtualMachineClass{ ObjectMeta: metav1.ObjectMeta{ - Name: vmc.Name, + Name: vmc.Name, + Namespace: config.Namespace, }, Spec: vmoprv1.VirtualMachineClassSpec{ - // TODO: figure out if to make vm class configurable via API Hardware: vmoprv1.VirtualMachineClassHardware{ Cpus: vmc.Cpus, Memory: vmc.Memory, @@ -571,7 +592,8 @@ func ReconcileDependencies(ctx context.Context, c client.Client, dependenciesCon virtualMachineImage := &vmoprv1.VirtualMachineImage{ ObjectMeta: metav1.ObjectMeta{ - Name: libraryItem.Name, + Name: libraryItem.Name, + Namespace: config.Namespace, }, Spec: vmoprv1.VirtualMachineImageSpec{ ImageID: libraryItemID, @@ -579,9 +601,10 @@ func ReconcileDependencies(ctx context.Context, c client.Client, dependenciesCon Type: "ovf", ProviderRef: vmoprv1.ContentProviderReference{ APIVersion: vmoprv1.SchemeGroupVersion.String(), - Kind: "ContentLibraryProvider", - Name: contentLibraryProvider.Name, - Namespace: contentLibraryProvider.Namespace, + Kind: "ContentLibraryItem", + // Not 100% sure about following values now that Kind is required to be ContentLibraryItem, but this doesn't seem to be an issue + Name: contentLibraryProvider.Name, + Namespace: contentLibraryProvider.Namespace, }, ProductInfo: vmoprv1.VirtualMachineImageProductInfo{ FullVersion: item.ProductInfo, @@ -614,6 +637,22 @@ func ReconcileDependencies(ctx context.Context, c client.Client, dependenciesCon return retryError } + // Fakes reconciliation of virtualMachineImage by setting required status field for the image to be considered ready. + virtualMachineImageReconciled := virtualMachineImage.DeepCopy() + virtualMachineImageReconciled.Status.ImageName = virtualMachineImage.Name + Set(virtualMachineImageReconciled, TrueCondition(ReadyConditionType)) + _ = wait.PollUntilContextTimeout(ctx, 250*time.Millisecond, 5*time.Second, true, func(ctx context.Context) (bool, error) { + retryError = nil + if err := c.Status().Patch(ctx, virtualMachineImageReconciled, client.MergeFrom(virtualMachineImage)); err != nil { + retryError = errors.Wrapf(err, "failed to patch vm-operator VirtualMachineImage %s", virtualMachineImage.Name) + } + log.Info("Patched vm-operator VirtualMachineImage", "ContentSource", klog.KObj(contentSource), "ContentLibraryProvider", klog.KObj(contentLibraryProvider), "VirtualMachineImage", klog.KObj(virtualMachineImage)) + return true, nil + }) + if retryError != nil { + return retryError + } + existingFiles, err := libMgr.ListLibraryItemFiles(ctx, libraryItemID) if err != nil { return errors.Wrapf(err, "failed to list files for vm-operator libraryItem %s", libraryItem.Name) @@ -669,3 +708,162 @@ func ReconcileDependencies(ctx context.Context, c client.Client, dependenciesCon return nil } + +// GetVCenterSession returns a VCenter session from vm-operator config. +func GetVCenterSession(ctx context.Context, c client.Client, enableKeepAlive bool, keepAliveDuration time.Duration) (*session.Session, error) { + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapName, + Namespace: DefaultNamespace, // This is where tilt/E2E deploy the vm-operator + }, + } + if err := c.Get(ctx, client.ObjectKeyFromObject(configMap), configMap); err != nil { + return nil, errors.Wrapf(err, "failed to get vm-operator ConfigMap %s", configMap.Name) + } + + serverURL := configMap.Data[serverURLConfigMapKey] + if serverURL == "" { + return nil, errors.Errorf("%s value is missing from the vm-operator ConfigMap %s", serverURLConfigMapKey, configMap.Name) + } + datacenter := configMap.Data[datacenterNameConfigMapKey] + if datacenter == "" { + return nil, errors.Errorf("%s value is missing from the vm-operator ConfigMap %s", datacenterNameConfigMapKey, configMap.Name) + } + secretName := configMap.Data[credentialSecretNameConfigMapKey] + if secretName == "" { + return nil, errors.Errorf("%s value is missing from the vm-operator ConfigMap %s", credentialSecretNameConfigMapKey, configMap.Name) + } + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: configMap.Namespace, // This is where tilt deploys the vm-operator + }, + } + if err := c.Get(ctx, client.ObjectKeyFromObject(secret), secret); err != nil { + return nil, errors.Wrapf(err, "failed to get vm-operator Credential Secret %s", secret.Name) + } + username := string(secret.Data[usernameSecretKey]) + if username == "" { + return nil, errors.Errorf("%s value is missing from the vm-operator Secret %s", usernameSecretKey, secret.Name) + } + password := string(secret.Data[passwordSecretKey]) + if password == "" { + return nil, errors.Errorf("%s value is missing from the vm-operator Secret %s", passwordSecretKey, secret.Name) + } + thumbprint := string(secret.Data[thumbprintSecretKey]) + if thumbprint == "" { + return nil, errors.Errorf("%s value is missing from the vm-operator Secret %s", thumbprintSecretKey, secret.Name) + } + + params := session.NewParams(). + WithServer(serverURL). + WithDatacenter(datacenter). + WithUserInfo(username, password). + WithThumbprint(thumbprint). + WithFeatures(session.Feature{ + EnableKeepAlive: enableKeepAlive, + KeepAliveDuration: keepAliveDuration, + }) + + return session.GetOrCreate(ctx, params) +} + +// GetDistributedPortGroup returns a the DistributedPortGroup from vm-operator config. +func GetDistributedPortGroup(ctx context.Context, c client.Client) (string, error) { + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapName, + Namespace: DefaultNamespace, // This is where tilt/E2E deploy the vm-operator + }, + } + if err := c.Get(ctx, client.ObjectKeyFromObject(configMap), configMap); err != nil { + return "", errors.Wrapf(err, "failed to get vm-operator ConfigMap %s", configMap.Name) + } + + distributedPortGroup := configMap.Data[distributedPortGroupConfigMapKey] + if distributedPortGroup == "" { + return "", errors.Errorf("%s value is missing from the vm-operator ConfigMap %s", distributedPortGroupConfigMapKey, configMap.Name) + } + + return distributedPortGroup, nil +} + +// NOTE: code below is a fork of vm-operator's pkg/conditions (so we can avoid to import the entire project). + +const ( + ReadyConditionType = "Ready" +) + +type Getter interface { + client.Object + + // GetConditions returns the list of conditions for a cluster API object. + GetConditions() vmoprv1.Conditions +} + +type Setter interface { + Getter + SetConditions(vmoprv1.Conditions) +} + +func Set(to Setter, condition *vmoprv1.Condition) { + if to == nil || condition == nil { + return + } + + // Check if the new conditions already exists, and change it only if there is a status + // transition (otherwise we should preserve the current last transition time)- + conditions := to.GetConditions() + exists := false + for i := range conditions { + existingCondition := conditions[i] + if existingCondition.Type == condition.Type { + exists = true + if !hasSameState(&existingCondition, condition) { + condition.LastTransitionTime = metav1.NewTime(time.Now().UTC().Truncate(time.Second)) + conditions[i] = *condition + break + } + condition.LastTransitionTime = existingCondition.LastTransitionTime + break + } + } + + // If the condition does not exist, add it, setting the transition time only if not already set + if !exists { + if condition.LastTransitionTime.IsZero() { + condition.LastTransitionTime = metav1.NewTime(time.Now().UTC().Truncate(time.Second)) + } + conditions = append(conditions, *condition) + } + + // Sorts conditions for convenience of the consumer, i.e. kubectl. + sort.Slice(conditions, func(i, j int) bool { + return lexicographicLess(&conditions[i], &conditions[j]) + }) + + to.SetConditions(conditions) +} + +func lexicographicLess(i, j *vmoprv1.Condition) bool { + return (i.Type == ReadyConditionType || i.Type < j.Type) && j.Type != ReadyConditionType +} + +func hasSameState(i, j *vmoprv1.Condition) bool { + return i.Type == j.Type && + i.Status == j.Status && + i.Reason == j.Reason && + i.Message == j.Message +} + +func TrueCondition(t vmoprv1.ConditionType) *vmoprv1.Condition { + return &vmoprv1.Condition{ + Type: t, + Status: corev1.ConditionTrue, + // This is a non-empty field in metav1.Conditions, when it was not in our v1a1 Conditions. This + // really doesn't work with how we've defined our conditions so do something to make things + // work for now. + Reason: string(corev1.ConditionTrue), + } +} diff --git a/test/go.mod b/test/go.mod index 479e57e17d..2c3127f475 100644 --- a/test/go.mod +++ b/test/go.mod @@ -15,6 +15,7 @@ require ( github.com/onsi/gomega v1.32.0 github.com/pkg/errors v0.9.1 github.com/spf13/pflag v1.0.5 + github.com/vmware-tanzu/net-operator-api v0.0.0-20231019160108-42131d6e8360 github.com/vmware-tanzu/vm-operator/api v1.8.5 github.com/vmware-tanzu/vm-operator/external/tanzu-topology v0.0.0-20231214185006-5477585eebfd github.com/vmware/govmomi v0.36.3 @@ -124,7 +125,6 @@ require ( github.com/stoewer/go-strcase v1.2.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/valyala/fastjson v1.6.4 // indirect - github.com/vmware-tanzu/net-operator-api v0.0.0-20231019160108-42131d6e8360 // indirect github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20231214185006-5477585eebfd // indirect go.etcd.io/etcd/api/v3 v3.5.13 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.13 // indirect diff --git a/test/infrastructure/net-operator/Dockerfile b/test/infrastructure/net-operator/Dockerfile new file mode 100644 index 0000000000..9cf947fdc5 --- /dev/null +++ b/test/infrastructure/net-operator/Dockerfile @@ -0,0 +1,83 @@ +# syntax=docker/dockerfile:1.4 + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build the manager binary +# Run this with docker build --build-arg builder_image= +ARG builder_image + +# Build architecture +ARG ARCH + +# Ignore Hadolint rule "Always tag the version of an image explicitly." +# It's an invalid finding since the image is explicitly set in the Makefile. +# https://github.com/hadolint/hadolint/wiki/DL3006 +# hadolint ignore=DL3006 +FROM ${builder_image} as builder +WORKDIR /workspace + +# Run this with docker build --build-arg goproxy=$(go env GOPROXY) to override the goproxy +ARG goproxy=https://proxy.golang.org +ENV GOPROXY=$goproxy + +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum + +# Change directories into the test go module +WORKDIR /workspace/test + +# Copy the Go Modules manifests +COPY test/go.mod go.mod +COPY test/go.sum go.sum + +# Cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN --mount=type=cache,target=/go/pkg/mod \ + go mod download + +# This needs to build with the entire CAPV context +WORKDIR /workspace + +# Copy the sources (which includes the test/infrastructure/vcsim subdirectory) +COPY ./ ./ + +# Change directories into net-operator +WORKDIR /workspace/test/infrastructure/net-operator + +# Cache the go build into the Go’s compiler cache folder so we take benefits of compiler caching across docker build calls +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + go build . + +# Build +ARG package=. +ARG ARCH +ARG ldflags + +# Do not force rebuild of up-to-date packages (do not use -a) and use the compiler cache folder +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \ + go build -trimpath -ldflags "${ldflags} -extldflags '-static'" \ + -o manager ${package} + + +FROM gcr.io/distroless/static:nonroot-${ARCH} +WORKDIR / +COPY --from=builder /workspace/test/infrastructure/net-operator/manager . +# Use uid of nonroot user (65532) because kubernetes expects numeric user when applying pod security policies +USER 65532 +ENTRYPOINT ["/manager"] diff --git a/test/infrastructure/net-operator/README.md b/test/infrastructure/net-operator/README.md new file mode 100644 index 0000000000..156eb08599 --- /dev/null +++ b/test/infrastructure/net-operator/README.md @@ -0,0 +1,3 @@ +# net-operator + +Provide a minimal implementation of the net-operator. See [vm-operator](../vm-operator/README.md) for more details. diff --git a/test/infrastructure/net-operator/config/certmanager/certificate.yaml b/test/infrastructure/net-operator/config/certmanager/certificate.yaml new file mode 100644 index 0000000000..4079986e89 --- /dev/null +++ b/test/infrastructure/net-operator/config/certmanager/certificate.yaml @@ -0,0 +1,24 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize + dnsNames: + - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc + - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: $(SERVICE_NAME)-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/test/infrastructure/net-operator/config/certmanager/kustomization.yaml b/test/infrastructure/net-operator/config/certmanager/kustomization.yaml new file mode 100644 index 0000000000..95f333f3f7 --- /dev/null +++ b/test/infrastructure/net-operator/config/certmanager/kustomization.yaml @@ -0,0 +1,5 @@ +resources: + - certificate.yaml + +configurations: + - kustomizeconfig.yaml diff --git a/test/infrastructure/net-operator/config/certmanager/kustomizeconfig.yaml b/test/infrastructure/net-operator/config/certmanager/kustomizeconfig.yaml new file mode 100644 index 0000000000..c6a6c0f1e0 --- /dev/null +++ b/test/infrastructure/net-operator/config/certmanager/kustomizeconfig.yaml @@ -0,0 +1,19 @@ +# This configuration is for teaching kustomize how to update name ref and var substitution +nameReference: + - kind: Issuer + group: cert-manager.io + fieldSpecs: + - kind: Certificate + group: cert-manager.io + path: spec/issuerRef/name + +varReference: + - kind: Certificate + group: cert-manager.io + path: spec/commonName + - kind: Certificate + group: cert-manager.io + path: spec/dnsNames + - kind: Certificate + group: cert-manager.io + path: spec/secretName diff --git a/test/infrastructure/net-operator/config/default/kustomization.yaml b/test/infrastructure/net-operator/config/default/kustomization.yaml new file mode 100644 index 0000000000..22e2d4470e --- /dev/null +++ b/test/infrastructure/net-operator/config/default/kustomization.yaml @@ -0,0 +1,54 @@ +namespace: vmware-system-netop + +namePrefix: vmware-system-netop- + +commonLabels: + # capvsim is not a provider, but by adding this label + # we can get this installed by Cluster APIs Tiltfile and by the clusterctl machinery we use in E2E tests. + cluster.x-k8s.io/provider: "runtime-extension-net-operator" + +resources: + - namespace.yaml + +bases: + - ../rbac + - ../manager + - ../webhook + - ../certmanager + +patchesStrategicMerge: + # Provide customizable hook for make targets. + - manager_image_patch.yaml + - manager_pull_policy.yaml + - manager_webhook_patch.yaml + +vars: + - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR + objref: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + fieldref: + fieldpath: metadata.namespace + - name: CERTIFICATE_NAME + objref: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + - name: SERVICE_NAMESPACE # namespace of the service + objref: + kind: Service + version: v1 + name: webhook-service + fieldref: + fieldpath: metadata.namespace + - name: SERVICE_NAME + objref: + kind: Service + version: v1 + name: webhook-service + +configurations: + - kustomizeconfig.yaml diff --git a/test/infrastructure/net-operator/config/default/kustomizeconfig.yaml b/test/infrastructure/net-operator/config/default/kustomizeconfig.yaml new file mode 100644 index 0000000000..eb191e64d0 --- /dev/null +++ b/test/infrastructure/net-operator/config/default/kustomizeconfig.yaml @@ -0,0 +1,4 @@ +# This configuration is for teaching kustomize how to update name ref and var substitution +varReference: +- kind: Deployment + path: spec/template/spec/volumes/secret/secretName diff --git a/test/infrastructure/net-operator/config/default/manager_image_patch.yaml b/test/infrastructure/net-operator/config/default/manager_image_patch.yaml new file mode 100644 index 0000000000..5468918c69 --- /dev/null +++ b/test/infrastructure/net-operator/config/default/manager_image_patch.yaml @@ -0,0 +1,11 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - image: gcr.io/k8s-staging-capi-vsphere/cluster-api-net-operator:main + name: manager diff --git a/test/infrastructure/net-operator/config/default/manager_pull_policy.yaml b/test/infrastructure/net-operator/config/default/manager_pull_policy.yaml new file mode 100644 index 0000000000..74a0879c60 --- /dev/null +++ b/test/infrastructure/net-operator/config/default/manager_pull_policy.yaml @@ -0,0 +1,11 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + imagePullPolicy: Always diff --git a/test/infrastructure/net-operator/config/default/manager_webhook_patch.yaml b/test/infrastructure/net-operator/config/default/manager_webhook_patch.yaml new file mode 100644 index 0000000000..f18fd10f99 --- /dev/null +++ b/test/infrastructure/net-operator/config/default/manager_webhook_patch.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + secretName: $(SERVICE_NAME)-cert # this secret will not be prefixed, since it's not managed by kustomize + diff --git a/test/infrastructure/net-operator/config/default/namespace.yaml b/test/infrastructure/net-operator/config/default/namespace.yaml new file mode 100644 index 0000000000..8b55c3cd89 --- /dev/null +++ b/test/infrastructure/net-operator/config/default/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + name: system diff --git a/test/infrastructure/net-operator/config/manager/kustomization.yaml b/test/infrastructure/net-operator/config/manager/kustomization.yaml new file mode 100644 index 0000000000..5c5f0b84cb --- /dev/null +++ b/test/infrastructure/net-operator/config/manager/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- manager.yaml diff --git a/test/infrastructure/net-operator/config/manager/manager.yaml b/test/infrastructure/net-operator/config/manager/manager.yaml new file mode 100644 index 0000000000..62dadf6a05 --- /dev/null +++ b/test/infrastructure/net-operator/config/manager/manager.yaml @@ -0,0 +1,60 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + control-plane: controller-manager +spec: + selector: + matchLabels: + control-plane: controller-manager + replicas: 1 + template: + metadata: + labels: + control-plane: controller-manager + spec: + containers: + - command: + - /manager + args: + - "--leader-elect" + - "--diagnostics-address=${CAPI_DIAGNOSTICS_ADDRESS:=:8443}" + - "--insecure-diagnostics=${CAPI_INSECURE_DIAGNOSTICS:=false}" + image: controller:latest + name: manager + ports: + - containerPort: 9440 + name: healthz + protocol: TCP + - containerPort: 8443 + name: metrics + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: healthz + livenessProbe: + httpGet: + path: /healthz + port: healthz + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + runAsUser: 65532 + runAsGroup: 65532 + terminationGracePeriodSeconds: 10 + serviceAccountName: manager + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault diff --git a/test/infrastructure/net-operator/config/rbac/kustomization.yaml b/test/infrastructure/net-operator/config/rbac/kustomization.yaml new file mode 100644 index 0000000000..e82521ffdc --- /dev/null +++ b/test/infrastructure/net-operator/config/rbac/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- role.yaml +- role_binding.yaml +- service_account.yaml +- leader_election_role.yaml +- leader_election_role_binding.yaml diff --git a/test/infrastructure/net-operator/config/rbac/leader_election_role.yaml b/test/infrastructure/net-operator/config/rbac/leader_election_role.yaml new file mode 100644 index 0000000000..23055e187d --- /dev/null +++ b/test/infrastructure/net-operator/config/rbac/leader_election_role.yaml @@ -0,0 +1,24 @@ +# permissions to do leader election. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: leader-election-role +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create +- apiGroups: + - "coordination.k8s.io" + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete diff --git a/test/infrastructure/net-operator/config/rbac/leader_election_role_binding.yaml b/test/infrastructure/net-operator/config/rbac/leader_election_role_binding.yaml new file mode 100644 index 0000000000..d5e0044679 --- /dev/null +++ b/test/infrastructure/net-operator/config/rbac/leader_election_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: leader-election-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election-role +subjects: +- kind: ServiceAccount + name: manager + namespace: system diff --git a/test/infrastructure/net-operator/config/rbac/role.yaml b/test/infrastructure/net-operator/config/rbac/role.yaml new file mode 100644 index 0000000000..881758e75d --- /dev/null +++ b/test/infrastructure/net-operator/config/rbac/role.yaml @@ -0,0 +1,41 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: manager-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - get + - list + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - get + - list + - watch +- apiGroups: + - netoperator.vmware.com + resources: + - networkinterfaces + verbs: + - get + - list + - patch + - watch +- apiGroups: + - netoperator.vmware.com + resources: + - networkinterfaces/status + verbs: + - get + - patch + - update diff --git a/test/infrastructure/net-operator/config/rbac/role_binding.yaml b/test/infrastructure/net-operator/config/rbac/role_binding.yaml new file mode 100644 index 0000000000..5a95f66d6f --- /dev/null +++ b/test/infrastructure/net-operator/config/rbac/role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: manager-role +subjects: +- kind: ServiceAccount + name: manager + namespace: system diff --git a/test/infrastructure/net-operator/config/rbac/service_account.yaml b/test/infrastructure/net-operator/config/rbac/service_account.yaml new file mode 100644 index 0000000000..77f747b53c --- /dev/null +++ b/test/infrastructure/net-operator/config/rbac/service_account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: manager + namespace: system diff --git a/test/infrastructure/net-operator/config/webhook/kustomization.yaml b/test/infrastructure/net-operator/config/webhook/kustomization.yaml new file mode 100644 index 0000000000..66157d5d5f --- /dev/null +++ b/test/infrastructure/net-operator/config/webhook/kustomization.yaml @@ -0,0 +1,5 @@ +resources: +- service.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/test/infrastructure/net-operator/config/webhook/kustomizeconfig.yaml b/test/infrastructure/net-operator/config/webhook/kustomizeconfig.yaml new file mode 100644 index 0000000000..25e21e3c96 --- /dev/null +++ b/test/infrastructure/net-operator/config/webhook/kustomizeconfig.yaml @@ -0,0 +1,25 @@ +# the following config is for teaching kustomize where to look at when substituting vars. +# It requires kustomize v2.1.0 or newer to work properly. +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + - kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + +namespace: +- kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true +- kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true + +varReference: +- path: metadata/annotations diff --git a/test/infrastructure/net-operator/config/webhook/service.yaml b/test/infrastructure/net-operator/config/webhook/service.yaml new file mode 100644 index 0000000000..711977f54f --- /dev/null +++ b/test/infrastructure/net-operator/config/webhook/service.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Service +metadata: + name: webhook-service + namespace: system +spec: + ports: + - port: 443 + targetPort: webhook-server diff --git a/test/infrastructure/net-operator/controllers/doc.go b/test/infrastructure/net-operator/controllers/doc.go new file mode 100644 index 0000000000..ab6f42354b --- /dev/null +++ b/test/infrastructure/net-operator/controllers/doc.go @@ -0,0 +1,18 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package controllers implements reconcilers for the net-operator controller. +package controllers diff --git a/test/infrastructure/net-operator/controllers/networkinterface_controller.go b/test/infrastructure/net-operator/controllers/networkinterface_controller.go new file mode 100644 index 0000000000..d1b5556765 --- /dev/null +++ b/test/infrastructure/net-operator/controllers/networkinterface_controller.go @@ -0,0 +1,123 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "time" + + "github.com/pkg/errors" + netopv1alpha1 "github.com/vmware-tanzu/net-operator-api/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + kerrors "k8s.io/apimachinery/pkg/util/errors" + "sigs.k8s.io/cluster-api/util/patch" + "sigs.k8s.io/cluster-api/util/predicates" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "sigs.k8s.io/cluster-api-provider-vsphere/test/framework/vmoperator" +) + +type NetworkInterfaceReconciler struct { + Client client.Client + EnableKeepAlive bool + KeepAliveDuration time.Duration + + // WatchFilterValue is the label value used to filter events prior to reconciliation. + WatchFilterValue string +} + +// +kubebuilder:rbac:groups=netoperator.vmware.com,resources=networkinterfaces,verbs=get;list;watch;patch +// +kubebuilder:rbac:groups=netoperator.vmware.com,resources=networkinterfaces/status,verbs=get;update;patch +// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create +// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create + +func (r *NetworkInterfaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) { + log := ctrl.LoggerFrom(ctx) + log.Info("Reconciling NetworkInterface") + + // Fetch the NetworkInterface instance + networkInterface := &netopv1alpha1.NetworkInterface{} + if err := r.Client.Get(ctx, req.NamespacedName, networkInterface); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + // Initialize the patch helper + patchHelper, err := patch.NewHelper(networkInterface, r.Client) + if err != nil { + return ctrl.Result{}, err + } + + // Always attempt to Patch the NetworkInterface object and status after each reconciliation. + defer func() { + if err := patchHelper.Patch(ctx, networkInterface); err != nil { + reterr = kerrors.NewAggregate([]error{reterr, err}) + } + }() + + if networkInterface.Status.NetworkID == "" { + s, err := vmoperator.GetVCenterSession(ctx, r.Client, r.EnableKeepAlive, r.KeepAliveDuration) + if err != nil { + return reconcile.Result{}, errors.Wrapf(err, "failed to get vcenter session") + } + + distributedPortGroupName, err := vmoperator.GetDistributedPortGroup(ctx, r.Client) + if err != nil { + return reconcile.Result{}, err + } + + distributedPortGroup, err := s.Finder.Network(ctx, distributedPortGroupName) + if err != nil { + return ctrl.Result{}, errors.Wrapf(err, "failed to get DistributedPortGroup %s", distributedPortGroupName) + } + + networkInterface.Status.NetworkID = distributedPortGroup.Reference().Value + networkInterface.Status.Conditions = []netopv1alpha1.NetworkInterfaceCondition{ + { + Type: netopv1alpha1.NetworkInterfaceReady, + Status: corev1.ConditionTrue, + }, + } + + // NOTE: we are not setting networkInterface.Status.IPConfigs because we are using dhcp to assign ip in supervisor tests (or the vmIP reconciler with vcsim). + + log.Info("Reconciling NetworkInterface status simulating successful net-operator reconcile") + } + + return ctrl.Result{}, nil +} + +// SetupWithManager will add watches for this controller. +func (r *NetworkInterfaceReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { + err := ctrl.NewControllerManagedBy(mgr). + For(&netopv1alpha1.NetworkInterface{}). + WithOptions(options). + WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(ctrl.LoggerFrom(ctx), r.WatchFilterValue)). + Complete(r) + + if err != nil { + return errors.Wrap(err, "failed setting up with a controller manager") + } + + return nil +} diff --git a/test/infrastructure/net-operator/main.go b/test/infrastructure/net-operator/main.go new file mode 100644 index 0000000000..66c0da82f2 --- /dev/null +++ b/test/infrastructure/net-operator/main.go @@ -0,0 +1,296 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package main define main for the net-operator controller. +package main + +import ( + "context" + "flag" + "fmt" + "os" + "reflect" + goruntime "runtime" + "time" + + "github.com/pkg/errors" + "github.com/spf13/pflag" + netopv1alpha1 "github.com/vmware-tanzu/net-operator-api/api/v1alpha1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/tools/leaderelection/resourcelock" + cliflag "k8s.io/component-base/cli/flag" + "k8s.io/component-base/logs" + logsv1 "k8s.io/component-base/logs/api/v1" + "k8s.io/klog/v2" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/controllers/remote" + "sigs.k8s.io/cluster-api/feature" + "sigs.k8s.io/cluster-api/util/flags" + "sigs.k8s.io/cluster-api/version" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" + + vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1" + "sigs.k8s.io/cluster-api-provider-vsphere/pkg/constants" + "sigs.k8s.io/cluster-api-provider-vsphere/test/infrastructure/net-operator/controllers" +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + controllerName = "net-operator-controller-manager" + + // common flags flags. + enableLeaderElection bool + leaderElectionLeaseDuration time.Duration + leaderElectionRenewDeadline time.Duration + leaderElectionRetryPeriod time.Duration + watchFilterValue string + watchNamespace string + profilerAddress string + enableContentionProfiling bool + syncPeriod time.Duration + restConfigQPS float32 + restConfigBurst int + healthAddr string + diagnosticsOptions = flags.DiagnosticsOptions{} + logOptions = logs.NewOptions() + // net operator specific flags. + networkInterfaceConcurrency int + // vsphere session specific flags. + enableKeepAlive bool + keepAliveDuration time.Duration +) + +func init() { + // scheme used for operating on the management cluster. + _ = corev1.AddToScheme(scheme) + _ = netopv1alpha1.AddToScheme(scheme) +} + +// InitFlags initializes the flags. +func InitFlags(fs *pflag.FlagSet) { + logsv1.AddFlags(logOptions, fs) + + fs.BoolVar(&enableLeaderElection, "leader-elect", false, + "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") + + fs.DurationVar(&leaderElectionLeaseDuration, "leader-elect-lease-duration", 15*time.Second, + "Interval at which non-leader candidates will wait to force acquire leadership (duration string)") + + fs.DurationVar(&leaderElectionRenewDeadline, "leader-elect-renew-deadline", 10*time.Second, + "Duration that the leading controller manager will retry refreshing leadership before giving up (duration string)") + + fs.DurationVar(&leaderElectionRetryPeriod, "leader-elect-retry-period", 2*time.Second, + "Duration the LeaderElector clients should wait between tries of actions (duration string)") + + fs.StringVar(&watchNamespace, "namespace", "", + "Namespace that the controller watches to reconcile cluster-api objects. If unspecified, the controller watches for cluster-api objects across all namespaces.") + + fs.StringVar(&watchFilterValue, "watch-filter", "", + fmt.Sprintf("Label value that the controller watches to reconcile cluster-api objects. Label key is always %s. If unspecified, the controller watches for all cluster-api objects.", clusterv1.WatchLabel)) + + fs.StringVar(&profilerAddress, "profiler-address", "", + "Bind address to expose the pprof profiler (e.g. localhost:6060)") + + fs.BoolVar(&enableContentionProfiling, "contention-profiling", false, + "Enable block profiling") + + fs.IntVar(&networkInterfaceConcurrency, "network-interface-concurrency", 10, + "Number of NetworkInterface to process simultaneously") + + fs.DurationVar(&syncPeriod, "sync-period", 10*time.Minute, + "The minimum interval at which watched resources are reconciled (e.g. 15m)") + + fs.Float32Var(&restConfigQPS, "kube-api-qps", 20, + "Maximum queries per second from the controller client to the Kubernetes API server. Defaults to 20") + + fs.IntVar(&restConfigBurst, "kube-api-burst", 30, + "Maximum number of queries that should be allowed in one burst from the controller client to the Kubernetes API server. Default 30") + + fs.StringVar(&healthAddr, "health-addr", ":9440", + "The address the health endpoint binds to.") + + fs.BoolVar(&enableKeepAlive, "enable-keep-alive", constants.DefaultEnableKeepAlive, + "feature to enable keep alive handler in vsphere sessions. This functionality is enabled by default.") + + fs.DurationVar(&keepAliveDuration, "keep-alive-duration", constants.DefaultKeepAliveDuration, + "idle time interval(minutes) in between send() requests in keepalive handler") + + flags.AddDiagnosticsOptions(fs, &diagnosticsOptions) + + feature.MutableGates.AddFlag(fs) +} + +// Add RBAC for the authorized diagnostics endpoint. +// +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create +// +kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create + +func main() { + InitFlags(pflag.CommandLine) + pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + // Set log level 2 as default. + if err := pflag.CommandLine.Set("v", "2"); err != nil { + setupLog.Error(err, "failed to set default log level") + os.Exit(1) + } + pflag.Parse() + + if err := logsv1.ValidateAndApply(logOptions, nil); err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + + // klog.Background will automatically use the right logger. + ctrl.SetLogger(klog.Background()) + + restConfig := ctrl.GetConfigOrDie() + restConfig.QPS = restConfigQPS + restConfig.Burst = restConfigBurst + restConfig.UserAgent = remote.DefaultClusterAPIUserAgent(controllerName) + + diagnosticsOpts := flags.GetDiagnosticsOptions(diagnosticsOptions) + + var watchNamespaces map[string]cache.Config + if watchNamespace != "" { + watchNamespaces = map[string]cache.Config{ + watchNamespace: {}, + } + } + + if enableContentionProfiling { + goruntime.SetBlockProfileRate(1) + } + + ctrlOptions := ctrl.Options{ + Scheme: scheme, + LeaderElection: enableLeaderElection, + LeaderElectionID: "net-operator-leader-election-capi", + LeaseDuration: &leaderElectionLeaseDuration, + RenewDeadline: &leaderElectionRenewDeadline, + RetryPeriod: &leaderElectionRetryPeriod, + LeaderElectionResourceLock: resourcelock.LeasesResourceLock, + HealthProbeBindAddress: healthAddr, + PprofBindAddress: profilerAddress, + Metrics: diagnosticsOpts, + Cache: cache.Options{ + DefaultNamespaces: watchNamespaces, + SyncPeriod: &syncPeriod, + }, + Client: client.Options{ + Cache: &client.CacheOptions{ + DisableFor: []client.Object{ + &corev1.ConfigMap{}, + &corev1.Secret{}, + }, + }, + }, + // WebhookServer: webhook.NewServer( + // webhook.Options{ + // Port: webhookPort, + // CertDir: webhookCertDir, + // TLSOpts: tlsOptionOverrides, + // }, + // ), + } + + mgr, err := ctrl.NewManager(restConfig, ctrlOptions) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + + // Setup the context that's going to be used in controllers and for the manager. + ctx := ctrl.SetupSignalHandler() + + // Check for supervisor VSphereCluster and start controller if found + gvr := vmwarev1.GroupVersion.WithResource(reflect.TypeOf(&vmwarev1.VSphereCluster{}).Elem().Name()) + supervisorMode, err := isCRDDeployed(mgr, gvr) + if err != nil { + setupLog.Error(err, "unable to detect supervisor mode") + os.Exit(1) + } + + // Continuing startup does not make sense without having managers added. + if !supervisorMode { + err := errors.New("supervisor CRDs not detected") + setupLog.Error(err, "CAPV CRDs are not deployed yet, restarting") + os.Exit(1) + } + + setupChecks(mgr, supervisorMode) + setupIndexes(ctx, mgr, supervisorMode) + setupReconcilers(ctx, mgr, supervisorMode) + setupWebhooks(mgr, supervisorMode) + + setupLog.Info("starting manager", "version", version.Get().String()) + if err := mgr.Start(ctx); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +} + +func setupChecks(mgr ctrl.Manager, _ bool) { + if err := mgr.AddReadyzCheck("webhook", mgr.GetWebhookServer().StartedChecker()); err != nil { + setupLog.Error(err, "unable to create ready check") + os.Exit(1) + } + + if err := mgr.AddHealthzCheck("webhook", mgr.GetWebhookServer().StartedChecker()); err != nil { + setupLog.Error(err, "unable to create health check") + os.Exit(1) + } +} + +func setupIndexes(_ context.Context, _ ctrl.Manager, _ bool) { +} + +func setupReconcilers(ctx context.Context, mgr ctrl.Manager, _ bool) { + if err := (&controllers.NetworkInterfaceReconciler{ + Client: mgr.GetClient(), + EnableKeepAlive: enableKeepAlive, + KeepAliveDuration: keepAliveDuration, + WatchFilterValue: watchFilterValue, + }).SetupWithManager(ctx, mgr, concurrency(networkInterfaceConcurrency)); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "NetworkInterfaceReconciler") + os.Exit(1) + } +} + +func setupWebhooks(_ ctrl.Manager, _ bool) { +} + +func concurrency(c int) controller.Options { + return controller.Options{MaxConcurrentReconciles: c} +} + +func isCRDDeployed(mgr ctrlmgr.Manager, gvr schema.GroupVersionResource) (bool, error) { + _, err := mgr.GetRESTMapper().KindFor(gvr) + if err != nil { + if meta.IsNoMatchError(err) { + return false, nil + } + return false, err + } + return true, nil +} diff --git a/test/infrastructure/net-operator/tilt-provider.yaml b/test/infrastructure/net-operator/tilt-provider.yaml new file mode 100644 index 0000000000..5a72dea94c --- /dev/null +++ b/test/infrastructure/net-operator/tilt-provider.yaml @@ -0,0 +1,9 @@ +--- +- name: net-operator + config: + version: v1.10.99 + image: gcr.io/k8s-staging-capi-vsphere/cluster-api-net-operator + live_reload_deps: + - main.go + - controllers + label: NETOP diff --git a/test/infrastructure/vcsim/Dockerfile b/test/infrastructure/vcsim/Dockerfile index c4cd873767..e425bce913 100644 --- a/test/infrastructure/vcsim/Dockerfile +++ b/test/infrastructure/vcsim/Dockerfile @@ -36,8 +36,9 @@ ENV GOPROXY=$goproxy COPY go.mod go.mod COPY go.sum go.sum -# Essentially, change directories into the test go module +# Change directories into the test go module WORKDIR /workspace/test + # Copy the Go Modules manifests COPY test/go.mod go.mod COPY test/go.sum go.sum @@ -47,12 +48,12 @@ COPY test/go.sum go.sum RUN --mount=type=cache,target=/go/pkg/mod \ go mod download -# This needs to build with the entire Cluster API context +# This needs to build with the entire CAPV context WORKDIR /workspace # Copy the sources (which includes the test/infrastructure/vcsim subdirectory) COPY ./ ./ -# Essentially, change directories into vcsim +# Change directories into vcsim WORKDIR /workspace/test/infrastructure/vcsim # Cache the go build into the Go’s compiler cache folder so we take benefits of compiler caching across docker build calls diff --git a/test/infrastructure/vcsim/README.md b/test/infrastructure/vcsim/README.md index 072aa629db..f49ac67395 100644 --- a/test/infrastructure/vcsim/README.md +++ b/test/infrastructure/vcsim/README.md @@ -168,6 +168,7 @@ provide extra args or enable debugging for this provider e.g. ... provider_repos: - ../cluster-api-provider-vsphere + - ../cluster-api-provider-vsphere/test/infrastructure/vcsim enable_providers: - kubeadm-bootstrap - kubeadm-control-plane diff --git a/test/infrastructure/vcsim/api/v1alpha1/vmoperatordependencies_types.go b/test/infrastructure/vcsim/api/v1alpha1/vmoperatordependencies_types.go index 7cd337e424..87bbf929db 100644 --- a/test/infrastructure/vcsim/api/v1alpha1/vmoperatordependencies_types.go +++ b/test/infrastructure/vcsim/api/v1alpha1/vmoperatordependencies_types.go @@ -61,12 +61,12 @@ type VCenterSpec struct { Thumbprint string `json:"thumbprint,omitempty"` // supervisor is based on a single vCenter cluster - Datacenter string `json:"datacenter,omitempty"` - Cluster string `json:"cluster,omitempty"` - Folder string `json:"folder,omitempty"` - ResourcePool string `json:"resourcePool,omitempty"` - ContentLibrary ContentLibraryConfig `json:"contentLibrary,omitempty"` - NetworkName string `json:"networkName,omitempty"` + Datacenter string `json:"datacenter,omitempty"` + Cluster string `json:"cluster,omitempty"` + Folder string `json:"folder,omitempty"` + ResourcePool string `json:"resourcePool,omitempty"` + ContentLibrary ContentLibraryConfig `json:"contentLibrary,omitempty"` + DistributedPortGroupName string `json:"distributedPortGroupName,omitempty"` } type StorageClass struct { @@ -139,16 +139,18 @@ func (d *VMOperatorDependencies) SetVCenterFromVCenterSimulator(vCenterSimulator datacenter := 0 cluster := 0 datastore := 0 + distributedPortGroup := 0 d.Spec.VCenter = &VCenterSpec{ - ServerURL: vCenterSimulator.Status.Host, - Username: vCenterSimulator.Status.Username, - Password: vCenterSimulator.Status.Password, - Thumbprint: vCenterSimulator.Status.Thumbprint, - Datacenter: vcsimhelpers.DatacenterName(datacenter), - Cluster: vcsimhelpers.ClusterPath(datacenter, cluster), - Folder: vcsimhelpers.VMFolderName(datacenter), - ResourcePool: vcsimhelpers.ResourcePoolPath(datacenter, cluster), + ServerURL: vCenterSimulator.Status.Host, + Username: vCenterSimulator.Status.Username, + Password: vCenterSimulator.Status.Password, + Thumbprint: vCenterSimulator.Status.Thumbprint, + Datacenter: vcsimhelpers.DatacenterName(datacenter), + Cluster: vcsimhelpers.ClusterPath(datacenter, cluster), + Folder: vcsimhelpers.VMFolderName(datacenter), + DistributedPortGroupName: vcsimhelpers.DistributedPortGroupName(datacenter, distributedPortGroup), + ResourcePool: vcsimhelpers.ResourcePoolPath(datacenter, cluster), ContentLibrary: ContentLibraryConfig{ Name: "vcsim", Datastore: vcsimhelpers.DatastorePath(datacenter, datastore), diff --git a/test/infrastructure/vcsim/config/crd/bases/vcsim.infrastructure.cluster.x-k8s.io_vmoperatordependencies.yaml b/test/infrastructure/vcsim/config/crd/bases/vcsim.infrastructure.cluster.x-k8s.io_vmoperatordependencies.yaml index 7a4b3b736c..5ac81b092c 100644 --- a/test/infrastructure/vcsim/config/crd/bases/vcsim.infrastructure.cluster.x-k8s.io_vmoperatordependencies.yaml +++ b/test/infrastructure/vcsim/config/crd/bases/vcsim.infrastructure.cluster.x-k8s.io_vmoperatordependencies.yaml @@ -96,9 +96,9 @@ spec: datacenter: description: supervisor is based on a single vCenter cluster type: string - folder: + distributedPortGroupName: type: string - networkName: + folder: type: string password: type: string diff --git a/test/infrastructure/vcsim/config/rbac/role.yaml b/test/infrastructure/vcsim/config/rbac/role.yaml index 0cc43e16a7..2dd13d4428 100644 --- a/test/infrastructure/vcsim/config/rbac/role.yaml +++ b/test/infrastructure/vcsim/config/rbac/role.yaml @@ -235,6 +235,14 @@ rules: - get - list - watch +- apiGroups: + - vmoperator.vmware.com + resources: + - virtualmachineimages/status + verbs: + - get + - patch + - update - apiGroups: - vmoperator.vmware.com resources: diff --git a/test/infrastructure/vcsim/controllers/vcsim_controller.go b/test/infrastructure/vcsim/controllers/vcsim_controller.go index 5f5e3b0209..5632267c7a 100644 --- a/test/infrastructure/vcsim/controllers/vcsim_controller.go +++ b/test/infrastructure/vcsim/controllers/vcsim_controller.go @@ -54,15 +54,6 @@ import ( const ( vcsimMinVersionForCAPV = "7.0.0" - - // vmIP reconciler secret. - - netConfigMapName = "vsphere.provider.config.netoperator.vmware.com" - netConfigServerURLKey = "server" - netConfigDatacenterKey = "datacenter" - netConfigUsernameKey = "username" - netConfigPasswordKey = "password" - netConfigThumbprintKey = "thumbprint" ) type VCenterSimulatorReconciler struct { @@ -85,6 +76,7 @@ type VCenterSimulatorReconciler struct { // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=contentsources,verbs=get;list;watch;create // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=contentsourcebindings,verbs=get;list;watch;create // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachineimages,verbs=get;list;watch;create +// +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachineimages/status,verbs=get;update;patch // +kubebuilder:rbac:groups=storage.k8s.io,resources=storageclasses,verbs=get;list;watch;create // +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create @@ -231,15 +223,6 @@ func (r *VCenterSimulatorReconciler) reconcileNormal(ctx context.Context, vCente if err := vmoperator.ReconcileDependencies(ctx, r.Client, dependenciesConfig); err != nil { return err } - - // The vm-operator doesn't take care of the networking part of the VM, which is usually - // managed by other components in the supervisor cluster. - // In order to make things to work in vcsim, there is the vmIP reconciler, which requires - // some info about the vcsim instance; in order to do so, we are creating a Secret. - - if err := addPreRequisitesForVMIPReconciler(ctx, r.Client, dependenciesConfig); err != nil { - return err - } } return nil @@ -276,42 +259,6 @@ func createVMTemplates(ctx context.Context, vCenterSimulator *vcsimv1.VCenterSim return nil } -func addPreRequisitesForVMIPReconciler(ctx context.Context, c client.Client, config *vcsimv1.VMOperatorDependencies) error { - log := ctrl.LoggerFrom(ctx) - log.Info("Reconciling requirements for the Fake net-operator Deployment") - - // default the OperatorRef if not specified. - if config.Spec.OperatorRef == nil { - config.Spec.OperatorRef = &vcsimv1.VMOperatorRef{Namespace: vmoperator.DefaultNamespace} - } - - netOperatorSecret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: netConfigMapName, - Namespace: config.Spec.OperatorRef.Namespace, - }, - StringData: map[string]string{ - netConfigServerURLKey: config.Spec.VCenter.ServerURL, - netConfigDatacenterKey: config.Spec.VCenter.Datacenter, - netConfigUsernameKey: config.Spec.VCenter.Username, - netConfigPasswordKey: config.Spec.VCenter.Password, - netConfigThumbprintKey: config.Spec.VCenter.Thumbprint, - }, - Type: corev1.SecretTypeOpaque, - } - if err := c.Get(ctx, client.ObjectKeyFromObject(netOperatorSecret), netOperatorSecret); err != nil { - if !apierrors.IsNotFound(err) { - return errors.Wrapf(err, "failed to get net-operator Secret %s", netOperatorSecret.Name) - } - if err := c.Create(ctx, netOperatorSecret); err != nil { - return errors.Wrapf(err, "failed to create net-operator Secret %s", netOperatorSecret.Name) - } - log.Info("Created net-operator Secret", "Secret", klog.KObj(netOperatorSecret)) - } - - return nil -} - func (r *VCenterSimulatorReconciler) reconcileDelete(ctx context.Context, vCenterSimulator *vcsimv1.VCenterSimulator) { log := ctrl.LoggerFrom(ctx) log.Info("Reconciling delete VCenter server") diff --git a/test/infrastructure/vcsim/controllers/virtualmachine_controller.go b/test/infrastructure/vcsim/controllers/virtualmachine_controller.go index a1e703b597..c799d8bd25 100644 --- a/test/infrastructure/vcsim/controllers/virtualmachine_controller.go +++ b/test/infrastructure/vcsim/controllers/virtualmachine_controller.go @@ -24,7 +24,6 @@ import ( "github.com/pkg/errors" vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" - corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kerrors "k8s.io/apimachinery/pkg/util/errors" @@ -295,48 +294,7 @@ func (r *VirtualMachineReconciler) getVMBootstrapReconciler(virtualMachine *vmop } func (r *VirtualMachineReconciler) getVCenterSession(ctx context.Context) (*session.Session, error) { - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: netConfigMapName, - Namespace: vmoperator.DefaultNamespace, // This is where tilt deploys the vm-operator - }, - } - if err := r.Client.Get(ctx, client.ObjectKeyFromObject(secret), secret); err != nil { - return nil, errors.Wrapf(err, "failed to get vm-operator Secret %s", secret.Name) - } - - serverURL := string(secret.Data[netConfigServerURLKey]) - if serverURL == "" { - return nil, errors.Errorf("%s value is missing from the vm-operator Secret %s", netConfigServerURLKey, secret.Name) - } - datacenter := string(secret.Data[netConfigDatacenterKey]) - if datacenter == "" { - return nil, errors.Errorf("%s value is missing from the vm-operator Secret %s", netConfigDatacenterKey, secret.Name) - } - username := string(secret.Data[netConfigUsernameKey]) - if username == "" { - return nil, errors.Errorf("%s value is missing from the vm-operator Secret %s", netConfigUsernameKey, secret.Name) - } - password := string(secret.Data[netConfigPasswordKey]) - if password == "" { - return nil, errors.Errorf("%s value is missing from the vm-operator Secret %s", netConfigPasswordKey, secret.Name) - } - thumbprint := string(secret.Data[netConfigThumbprintKey]) - if thumbprint == "" { - return nil, errors.Errorf("%s value is missing from the vm-operator Secret %s", netConfigThumbprintKey, secret.Name) - } - - params := session.NewParams(). - WithServer(serverURL). - WithDatacenter(datacenter). - WithUserInfo(username, password). - WithThumbprint(thumbprint). - WithFeatures(session.Feature{ - EnableKeepAlive: r.EnableKeepAlive, - KeepAliveDuration: r.KeepAliveDuration, - }) - - return session.GetOrCreate(ctx, params) + return vmoperator.GetVCenterSession(ctx, r.Client, r.EnableKeepAlive, r.KeepAliveDuration) } // SetupWithManager will add watches for this controller. diff --git a/test/infrastructure/vm-operator/README.md b/test/infrastructure/vm-operator/README.md index f483895200..a3f50b36a3 100644 --- a/test/infrastructure/vm-operator/README.md +++ b/test/infrastructure/vm-operator/README.md @@ -14,7 +14,7 @@ This project has the requirement to test CAPV in supervisor mode using all the CAPI, CAPV and vCenter supervisor, and also for all the version built from open PRs. In order to achieve this without incurring the cost/complexity of creating multiple, ad-hoc vCenter distributions, -we are using a "limited version of the supervisor", composed by the vm-operator only. +we are using a "limited version of the supervisor", composed by the vm-operator and a minimal version of the net-operator. This "limited version of the supervisor" is considered enough to provide a signal for CAPV development and test; however, due to the necessary trade-offs required to get a simple and cheap test environment, the solution described below @@ -24,6 +24,8 @@ The picture explains how this works in detail: ![Architecture](architecture-part1.drawio.svg) +NOTE: net-operator is not represented for sake of simplicity, it is complementary to the vm-operator. + As you might notice, it is required to have an additional component taking care of setting up the management cluster and vCenter as required by the vm-operator. This component exist in different variants according to the use cases described in following paragraphs. @@ -44,11 +46,14 @@ NOTE: before using `vm-operator` for the first time, you have to run `make vm-op ... provider_repos: - ../cluster-api-provider-vsphere + - ../cluster-api-provider-vsphere/test/infrastructure/net-operator + - ../cluster-api-provider-vsphere/test/infrastructure/vcsim enable_providers: - kubeadm-bootstrap - kubeadm-control-plane - vsphere-supervisor - vm-operator + - net-operator - vcsim extra_args: vcsim: @@ -73,6 +78,8 @@ The following image summarizes all the moving parts involved in this scenario. ![Architecture](architecture-part2.drawio.svg) +NOTE: net-operator is not represented for sake of simplicity, it is complementary to the vm-operator. + ## E2E tests for CAPV in supervisor mode A subset of CAPV E2E tests can be executed using the supervisor mode by setting `GINKGO_FOCUS="\[supervisor\]"`. diff --git a/test/infrastructure/vm-operator/kustomization.yaml b/test/infrastructure/vm-operator/kustomization.yaml index 5aed49c1e7..177f367450 100644 --- a/test/infrastructure/vm-operator/kustomization.yaml +++ b/test/infrastructure/vm-operator/kustomization.yaml @@ -12,6 +12,8 @@ resources: patchesStrategicMerge: - vm-operator-replicas.yaml - vm-operator-pull-policy.yaml +- vm-operator-env-var-patch.yaml +- net-operator-networkinterface-status.yaml patches: - target: diff --git a/test/infrastructure/vm-operator/net-operator-networkinterface-status.yaml b/test/infrastructure/vm-operator/net-operator-networkinterface-status.yaml new file mode 100644 index 0000000000..1d9fe13caf --- /dev/null +++ b/test/infrastructure/vm-operator/net-operator-networkinterface-status.yaml @@ -0,0 +1,140 @@ +# This is a copy of the CRD in vm-operator with the only addition of the status subresource. As soon as this is fixed in the +# vm-operator code base, we can get rid of this patch +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: networkinterfaces.netoperator.vmware.com +spec: + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: NetworkInterface is the Schema for the networkinterfaces API. + A NetworkInterface represents a user's request for network configuration + to use to place a VM/Pod/Container's nic on a specified network. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NetworkInterfaceSpec defines the desired state of NetworkInterface. + properties: + networkName: + description: NetworkName refers to a NetworkObject in the same namespace. + type: string + providerRef: + description: ProviderRef is a reference to a provider specific network + interface object that specifies the network interface configuration. + If unset, default configuration is assumed. + properties: + apiGroup: + description: APIGroup is the group for the resource being referenced. + type: string + apiVersion: + description: API version of the referent. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - apiGroup + - kind + - name + type: object + type: + description: Type is the type of NetworkInterface. Supported values + are vmxnet3. + type: string + type: object + status: + description: NetworkInterfaceStatus defines the observed state of NetworkInterface. + Once NetworkInterfaceReady condition is True, it should contain configuration + to use to place a VM/Pod/Container's nic on the specified network. + properties: + conditions: + description: Conditions is an array of current observed network interface + conditions. + items: + description: NetworkInterfaceCondition describes the state of a + NetworkInterface at a certain point. + properties: + lastTransitionTime: + description: LastTransitionTime is the timestamp corresponding + to the last status change of this condition. + format: date-time + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Machine understandable string that gives the reason + for condition's last transition. + type: string + status: + description: Status is the status of the condition. Can be True, + False, Unknown. + type: string + type: + description: Type is the type of network interface condition. + type: string + required: + - status + - type + type: object + type: array + externalID: + description: ExternalID is a network provider specific identifier + assigned to the network interface. + type: string + ipConfigs: + description: IPConfigs is an array of IP configurations for the network + interface. + items: + description: IPConfig represents an IP configuration. + properties: + gateway: + description: Gateway setting. + type: string + ip: + description: IP setting. + type: string + ipFamily: + description: IPFamily specifies the IP family (IPv4 vs IPv6) + the IP belongs to. + type: string + subnetMask: + description: SubnetMask setting. + type: string + required: + - gateway + - ip + - ipFamily + - subnetMask + type: object + type: array + macAddress: + description: MacAddress setting for the network interface. + type: string + networkID: + description: NetworkID is an network provider specific identifier + for the network backing the network interface. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: { } diff --git a/test/infrastructure/vm-operator/vm-operator-env-var-patch.yaml b/test/infrastructure/vm-operator/vm-operator-env-var-patch.yaml new file mode 100644 index 0000000000..b0131d9f3f --- /dev/null +++ b/test/infrastructure/vm-operator/vm-operator-env-var-patch.yaml @@ -0,0 +1,47 @@ +# This patch derives from vm-operator config/local, but it is now using NETWORK_PROVIDER = VSPHERE_NETWORK +# as suggested by the vm-operator maintainers. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: vmware-system-vmop-controller-manager + namespace: vmware-system-vmop +spec: + template: + spec: + containers: + - name: manager + env: + - name: VSPHERE_NETWORKING + value: "true" + - name: NETWORK_PROVIDER + value: "VSPHERE_NETWORK" + + - name: FSS_WCP_INSTANCE_STORAGE + value: "true" + - name: FSS_WCP_VMSERVICE_BACKUPRESTORE + value: "false" + - name: FSS_PODVMONSTRETCHEDSUPERVISOR + value: "false" + - name: FSS_WCP_TKG_Multiple_CL + value: "false" + # + # Feature state switch flags beneath this line are enabled on main and + # only retained in this file because it is used by internal testing to + # determine the state of the feature. Since this is used by older + # branches as well, the flags must remain, otherwise the absence of the + # flag indicates a feature is not present or disabled. + # + - name: FSS_WCP_Unified_TKG + value: "true" + - name: FSS_WCP_VMSERVICE_V1ALPHA2 + value: "true" + - name: FSS_WCP_VM_CLASS_AS_CONFIG + value: "true" + - name: FSS_WCP_VM_CLASS_AS_CONFIG_DAYNDATE + value: "true" + - name: FSS_WCP_VM_IMAGE_REGISTRY + value: "true" + - name: FSS_WCP_NAMESPACED_VM_CLASS + value: "true" + - name: FSS_WCP_WINDOWS_SYSPREP + value: "true" diff --git a/tilt-provider.yaml b/tilt-provider.yaml index 50718cc6b1..751c9517af 100644 --- a/tilt-provider.yaml +++ b/tilt-provider.yaml @@ -32,7 +32,7 @@ # IMPORTANT: run "make vm-operator-manifest-build" before using this "provider" with tilt - name: vm-operator config: - version: v1.8.1 + version: v1.8.6 label: VMOP kustomize_folder: "/test/infrastructure/vm-operator" kustomize_options: