Skip to content

Commit

Permalink
Merge pull request #1405 from p-strusiewiczsurmacki-mobica/clustercla…
Browse files Browse the repository at this point in the history
…ss-support

🌱 Support for ClusterClass template
  • Loading branch information
metal3-io-bot authored Apr 30, 2024
2 parents 7d37aad + 7e3d56c commit e948801
Show file tree
Hide file tree
Showing 31 changed files with 2,028 additions and 20 deletions.
60 changes: 60 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,10 @@ test: lint unit ## Run tests
test-e2e: ## Run e2e tests with capi e2e testing framework
./scripts/ci-e2e.sh

.PHONY: test-clusterclass-e2e
test-clusterclass-e2e: ## Run e2e tests with capi e2e testing framework
CLUSTER_TOPOLOGY=true GINKGO_FOCUS=basic ./scripts/ci-e2e.sh

GINKGO_NOCOLOR ?= false
ARTIFACTS ?= $(ROOT_DIR)/_artifacts
E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/e2e_conf.yaml
Expand Down Expand Up @@ -167,6 +171,14 @@ cluster-templates: $(KUSTOMIZE) ## Generate cluster templates
$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-ubuntu > $(E2E_OUT_DIR)/cluster-template-ubuntu.yaml
$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-centos > $(E2E_OUT_DIR)/cluster-template-centos.yaml
$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-upgrade-workload > $(E2E_OUT_DIR)/cluster-template-upgrade-workload.yaml
touch $(E2E_OUT_DIR)/clusterclass.yaml

.PHONY: clusterclass-templates
clusterclass-templates: $(KUSTOMIZE) ## Generate cluster templates
$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/clusterclass-template-ubuntu > $(E2E_OUT_DIR)/cluster-template-ubuntu.yaml
$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/clusterclass-template-centos > $(E2E_OUT_DIR)/cluster-template-centos.yaml
$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/clusterclass-template-upgrade-workload > $(E2E_OUT_DIR)/cluster-template-upgrade-workload.yaml
$(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/clusterclass > $(E2E_OUT_DIR)/clusterclass.yaml

## --------------------------------------
## E2E Testing
Expand Down Expand Up @@ -201,6 +213,29 @@ e2e-tests: $(GINKGO) e2e-substitutions cluster-templates # This target should be

rm $(E2E_CONF_FILE_ENVSUBST)

.PHONY: e2e-clusterclass-tests
e2e-clusterclass-tests: CONTAINER_RUNTIME?=docker # Env variable can override this default
export CONTAINER_RUNTIME

e2e-clusterclass-tests: $(GINKGO) e2e-substitutions clusterclass-templates # This target should be called from scripts/ci-e2e.sh
for image in $(E2E_CONTAINERS); do \
$(CONTAINER_RUNTIME) pull $$image; \
done

$(GINKGO) --timeout=$(GINKGO_TIMEOUT) -v --trace --tags=e2e \
--show-node-events --no-color=$(GINKGO_NOCOLOR) \
--fail-fast="$(KEEP_TEST_ENV)" \
--junit-report="junit.e2e_suite.1.xml" \
--focus="$(GINKGO_FOCUS)" $(_SKIP_ARGS) "$(ROOT_DIR)/$(TEST_DIR)/e2e/" -- \
-e2e.artifacts-folder="$(ARTIFACTS)" \
-e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \
-e2e.skip-resource-cleanup=$(SKIP_CLEANUP) \
-e2e.keep-test-environment=$(KEEP_TEST_ENV) \
-e2e.trigger-ephemeral-test=$(EPHEMERAL_TEST) \
-e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER)

rm $(E2E_CONF_FILE_ENVSUBST)

## --------------------------------------
## Build
## --------------------------------------
Expand Down Expand Up @@ -394,6 +429,10 @@ generate-manifests: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc.
generate-examples: $(KUSTOMIZE) clean-examples ## Generate examples configurations to run a cluster.
./examples/generate.sh

.PHONY: generate-examples-clusterclass
generate-examples-clusterclass: $(KUSTOMIZE) clean-examples ## Generate examples configurations to run a cluster.
CLUSTER_TOPOLOGY=true ./examples/generate.sh

## --------------------------------------
## Docker
## --------------------------------------
Expand Down Expand Up @@ -471,18 +510,35 @@ deploy: generate-examples
kubectl apply -f examples/_out/provider-components.yaml
kubectl apply -f examples/_out/metal3crds.yaml

deploy-clusterclass: generate-examples-clusterclass
kubectl apply -f examples/_out/cert-manager.yaml
kubectl wait --for=condition=Available --timeout=300s -n cert-manager deployment cert-manager
kubectl wait --for=condition=Available --timeout=300s -n cert-manager deployment cert-manager-cainjector
kubectl wait --for=condition=Available --timeout=300s -n cert-manager deployment cert-manager-webhook
kubectl apply -f examples/_out/provider-components.yaml
kubectl apply -f examples/_out/metal3crds.yaml

deploy-examples:
kubectl apply -f ./examples/_out/metal3plane.yaml
kubectl apply -f ./examples/_out/cluster.yaml
kubectl apply -f ./examples/_out/machinedeployment.yaml
kubectl apply -f ./examples/_out/controlplane.yaml

deploy-examples-clusterclass:
kubectl apply -f ./examples/_out/metal3plane.yaml
kubectl apply -f ./examples/_out/clusterclass.yaml
kubectl apply -f ./examples/_out/cluster.yaml

delete-examples:
kubectl delete -f ./examples/_out/machinedeployment.yaml || true
kubectl delete -f ./examples/_out/controlplane.yaml || true
kubectl delete -f ./examples/_out/cluster.yaml || true
kubectl delete -f ./examples/_out/metal3plane.yaml || true

delete-examples-clusterclass:
kubectl delete -f ./examples/_out/cluster.yaml || true
kubectl delete -f ./examples/_out/clusterclass.yaml || true
kubectl delete -f ./examples/_out/metal3plane.yaml || true

## --------------------------------------
## Release
Expand Down Expand Up @@ -537,6 +593,10 @@ kind-create: ## create capm3 kind cluster if needed
tilt-settings:
./hack/gen_tilt_settings.sh

.PHONY: tilt-settings-clusterclass
tilt-settings-clusterclass:
CLUSTER_TOPOLOGY=true ./hack/gen_tilt_settings.sh

.PHONY: tilt-up
tilt-up: $(ENVSUBST) $(KUSTOMIZE) kind-create ## start tilt and build kind cluster if needed
$(MAKE) deploy-bmo-crd
Expand Down
56 changes: 49 additions & 7 deletions Tiltfile
Original file line number Diff line number Diff line change
Expand Up @@ -69,27 +69,69 @@ def deploy_capi():
core_extra_args = extra_args.get("core")
if core_extra_args:
for namespace in ["capi-system", "capi-webhook-system"]:
patch_args_with_extra_args(namespace, "capi-controller-manager", core_extra_args)
patch_args_with_extra_args(namespace, "capi-controller-manager", core_extra_args, 1)
patch_capi_manager_role_with_exp_infra_rbac()
if extra_args.get("kubeadm-bootstrap"):
kb_extra_args = extra_args.get("kubeadm-bootstrap")
if kb_extra_args:
patch_args_with_extra_args("capi-kubeadm-bootstrap-system", "capi-kubeadm-bootstrap-controller-manager", kb_extra_args)


def patch_args_with_extra_args(namespace, name, extra_args):
args_str = str(local('kubectl get deployments {} -n {} -o jsonpath={{.spec.template.spec.containers[1].args}}'.format(name, namespace)))
patch_args_with_extra_args("capi-kubeadm-bootstrap-system", "capi-kubeadm-bootstrap-controller-manager", kb_extra_args, 1)
if extra_args.get("feature_gates"):
feature_gates = extra_args.get("feature_gates")
if feature_gates:
for gate in feature_gates:
set_feature_gate("capi-system", "capi-controller-manager", gate, feature_gates[gate], 0)
set_feature_gate("capi-kubeadm-control-plane-system", "capi-kubeadm-control-plane-controller-manager", gate, feature_gates[gate], 0)

def patch_args_with_extra_args(namespace, name, extra_args, container):
args_str = str(local('kubectl get deployments {} -n {} -o jsonpath={{.spec.template.spec.containers[{}].args}}'.format(name, namespace, container)))
args_to_add = [arg for arg in extra_args if arg not in args_str]
if args_to_add:
args = args_str[1:-1].split()
args.extend(args_to_add)
patch = [{
"op": "replace",
"path": "/spec/template/spec/containers/1/args",
"path": "/spec/template/spec/containers/" + str(container) + "/args",
"value": args,
}]
local("kubectl patch deployment {} -n {} --type json -p='{}'".format(name, namespace, str(encode_json(patch)).replace("\n", "")))

def set_feature_gate(namespace, name, feature_gate, value, container):
args_str = str(local('kubectl get deployments {} -n {} -o jsonpath={{.spec.template.spec.containers[{}].args}}'.format(name, namespace, container)))
args = args_str[2:-2].split("\",\"")
print("args")
print(args)

args_to_add = []
for arg in args:
print("arg")
print(arg)
lower_arg = arg.lower()
feature_gate_lower = feature_gate.lower()
start = lower_arg.find(feature_gate_lower)
if start > -1:
end = lower_arg.find(",", start)
if end < -1:
end = len(lower_arg)-1
new_arg = arg[:start] + feature_gate + "=" + value + arg[end:]
print(new_arg)
args_to_add.append(new_arg)
else:
args_to_add.append(arg)

print("args_to_add")
print(args_to_add)
if len(args_to_add) > 0:
patches = []
for i in range(0, len(args_to_add)):
patch = [{
"op": "replace",
"path": "/spec/template/spec/containers/" + str(container) + "/args/" + str(i),
"value": args_to_add[i],
}]
print("patch")
print(patch)
local("kubectl patch deployment {} -n {} --type json -p='{}'".format(name, namespace, str(encode_json(patch)).replace("\n", "")))


# patch the CAPI manager role to also provide access to experimental infrastructure
def patch_capi_manager_role_with_exp_infra_rbac():
Expand Down
2 changes: 1 addition & 1 deletion api/v1beta1/metal3cluster_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ func (s *Metal3ClusterSpec) IsValid() error {
}

if s.ControlPlaneEndpoint.Port == 0 {
missing = append(missing, "ControlPlaneEndpoint.Host")
missing = append(missing, "ControlPlaneEndpoint.Port")
}

if len(missing) > 0 {
Expand Down
53 changes: 53 additions & 0 deletions api/v1beta1/metal3clustertemplate_types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1beta1

import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// Metal3ClusterTemplateSpec defines the desired state of Metal3ClusterTemplate.
type Metal3ClusterTemplateSpec struct {
Template Metal3ClusterTemplateResource `json:"template"`
}

// +kubebuilder:object:root=true
// +kubebuilder:resource:path=metal3clustertemplates,scope=Namespaced,categories=cluster-api,shortName=m3ct
// +kubebuilder:storageversion

// Metal3ClusterTemplate is the Schema for the metal3clustertemplates API.
type Metal3ClusterTemplate struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`

Spec Metal3ClusterTemplateSpec `json:"spec,omitempty"`
}

// +kubebuilder:object:root=true

// Metal3ClusterTemplateList contains a list of Metal3ClusterTemplate.
type Metal3ClusterTemplateList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []Metal3ClusterTemplate `json:"items"`
}

func init() {
objectTypes = append(objectTypes, &Metal3ClusterTemplate{}, &Metal3ClusterTemplateList{})
}

// Metal3ClusterTemplateResource describes the data for creating a Metal3Cluster from a template.
type Metal3ClusterTemplateResource struct {
Spec Metal3ClusterSpec `json:"spec"`
}
67 changes: 67 additions & 0 deletions api/v1beta1/metal3clustertemplate_webhook.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1beta1

import (
"github.com/pkg/errors"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/webhook"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
)

// SetupWebhookWithManager sets up and registers the webhook with the manager.
func (c *Metal3ClusterTemplate) SetupWebhookWithManager(mgr ctrl.Manager) error {
return ctrl.NewWebhookManagedBy(mgr).
For(c).
Complete()
}

// +kubebuilder:webhook:verbs=create;update,path=/validate-infrastructure-cluster-x-k8s-io-v1beta1-metal3clustertemplate,mutating=false,failurePolicy=fail,groups=infrastructure.cluster.x-k8s.io,resources=metal3clustertemplates,versions=v1beta1,name=validation.metal3clustertemplate.infrastructure.cluster.x-k8s.io,matchPolicy=Equivalent,sideEffects=None,admissionReviewVersions=v1;v1beta1
// +kubebuilder:webhook:verbs=create;update,path=/mutate-infrastructure-cluster-x-k8s-io-v1beta1-metal3clustertemplate,mutating=true,failurePolicy=fail,groups=infrastructure.cluster.x-k8s.io,resources=metal3clustertemplates,versions=v1beta1,name=default.metal3clustertemplate.infrastructure.cluster.x-k8s.io,matchPolicy=Equivalent,sideEffects=None,admissionReviewVersions=v1;v1beta1

var _ webhook.Defaulter = &Metal3ClusterTemplate{}
var _ webhook.Validator = &Metal3ClusterTemplate{}

func (c *Metal3ClusterTemplate) Default() {
}

// ValidateCreate implements webhook.Validator so a webhook will be registered for the type.
func (c *Metal3ClusterTemplate) ValidateCreate() (admission.Warnings, error) {
return nil, c.validate()
}

// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type.
func (c *Metal3ClusterTemplate) ValidateUpdate(old runtime.Object) (admission.Warnings, error) {
oldM3ct, ok := old.(*Metal3ClusterTemplate)
if !ok || oldM3ct == nil {
return nil, apierrors.NewInternalError(errors.New("unable to convert existing object"))
}

if err := oldM3ct.Spec.Template.Spec.IsValid(); err != nil {
return nil, err
}

return nil, c.validate()
}

// ValidateDelete implements webhook.Validator so a webhook will be registered for the type.
func (c *Metal3ClusterTemplate) ValidateDelete() (admission.Warnings, error) {
return nil, nil
}

func (c *Metal3ClusterTemplate) validate() error {
return c.Spec.Template.Spec.IsValid()
}
Loading

0 comments on commit e948801

Please sign in to comment.