diff --git a/Makefile b/Makefile index b3e1ca64..8d79dcc5 100644 --- a/Makefile +++ b/Makefile @@ -227,16 +227,21 @@ CALICO_VERSION ?= v3.26.3 crs-calico: ## Generates crs manifests for Calico. curl -o templates/crs/cni/calico.yaml https://raw.githubusercontent.com/projectcalico/calico/$(CALICO_VERSION)/manifests/calico.yaml -METALLB_VERSION ?= 0.14.3 +METALLB_VERSION ?= 0.14.4 FRR_K8S_DIR = metallb/charts/metallb/charts/frr-k8s/templates -METALLB_TOLERATIONS = [{"key": "node-role.kubernetes.io/load-balancer", "operator": "Exists", "effect": "NoSchedule"}] +LB_TOLERATIONS = [{"key": "node-role.kubernetes.io/load-balancer", "operator": "Exists", "effect": "NoSchedule"}] +CP_TOLERATIONS = [{"key": "node-role.kubernetes.io/control-plane", "operator": "Exists", "effect": "NoSchedule"}] +FRR_NODESELECTOR = {"node-role.kubernetes.io/load-balancer": ""} .PHONY: crs-metallb crs-metallb: ## Generates crs manifests for MetalLB. $(HELM) repo add metallb https://metallb.github.io/metallb - $(HELM) template metallb metallb/metallb --version $(METALLB_VERSION) --set frrk8s.enabled=true,speaker.frr.enabled=false --set-json 'controller.tolerations=$(METALLB_TOLERATIONS)' --set-json 'speaker.tolerations=$(METALLB_TOLERATIONS)' --set-json 'frr-k8s.frrk8s.tolerations=$(METALLB_TOLERATIONS)' --namespace=metallb-system > templates/crs/metallb.yaml - - @# fixup namespacing in frr-k8s to work with clusterresourcesets - @sed -e '7bp;48bp;69bp;1682bp;1854bp;1887bp;2253bp;bn' -e ':p i\ namespace: "metallb-system"' -e ':n' -i templates/crs/metallb.yaml + $(HELM) template metallb metallb/metallb --version $(METALLB_VERSION) \ + --set frrk8s.enabled=true,speaker.frr.enabled=false \ + --set-json 'controller.tolerations=$(CP_TOLERATIONS)' \ + --set-json 'speaker.tolerations=$(LB_TOLERATIONS)' \ + --set-json 'frr-k8s.frrk8s.tolerations=$(LB_TOLERATIONS)' \ + --set-json 'frr-k8s.frrk8s.nodeSelector=$(FRR_NODESELECTOR)' \ + --namespace=metallb-system > templates/crs/metallb.yaml ##@ Release diff --git a/api/v1alpha1/proxmoxcluster_types.go b/api/v1alpha1/proxmoxcluster_types.go index 8196bb2d..f2019131 100644 --- a/api/v1alpha1/proxmoxcluster_types.go +++ b/api/v1alpha1/proxmoxcluster_types.go @@ -66,6 +66,26 @@ type ProxmoxClusterSpec struct { // DNSServers contains information about nameservers used by the machines. // +kubebuilder:validation:MinItems=1 DNSServers []string `json:"dnsServers"` + + // NodeCloneSpec is the configuration pertaining to all items configurable + // in the configuration and cloning of a proxmox VM. Multiple types of nodes can be specified. + // +optional + CloneSpec *ProxmoxClusterCloneSpec `json:"cloneSpec,omitempty"` +} + +// ProxmoxClusterCloneSpec is the configuration pertaining to all items configurable +// in the configuration and cloning of a proxmox VM. +type ProxmoxClusterCloneSpec struct { + // +kubebuilder:validation:XValidation:rule="has(self.controlPlane)",message="Cowardly refusing to deploy cluster without control plane" + ProxmoxMachineSpec map[string]ProxmoxMachineSpec `json:"machineSpec"` + + // SshAuthorizedKeys contains the authorized keys deployed to the PROXMOX VMs. + // +optional + SSHAuthorizedKeys []string `json:"sshAuthorizedKeys,omitempty"` + + // VirtualIPNetworkInterface is the interface the k8s control plane binds to. + // +optional + VirtualIPNetworkInterface string `json:"virtualIPNetworkInterface,omitempty"` } // IPConfigSpec contains information about available IP config. diff --git a/api/v1alpha1/proxmoxcluster_types_test.go b/api/v1alpha1/proxmoxcluster_types_test.go index b090a5e5..1715faac 100644 --- a/api/v1alpha1/proxmoxcluster_types_test.go +++ b/api/v1alpha1/proxmoxcluster_types_test.go @@ -1,5 +1,5 @@ /* -Copyright 2023 IONOS Cloud. +Copyright 2023-2024 IONOS Cloud. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -85,6 +85,15 @@ func defaultCluster() *ProxmoxCluster { Prefix: 24, }, DNSServers: []string{"1.2.3.4"}, + CloneSpec: &ProxmoxClusterCloneSpec{ + ProxmoxMachineSpec: map[string]ProxmoxMachineSpec{ + "controlPlane": { + VirtualMachineCloneSpec: VirtualMachineCloneSpec{ + SourceNode: "pve1", + }, + }, + }, + }, }, } } @@ -129,6 +138,15 @@ var _ = Describe("ProxmoxCluster Test", func() { Expect(k8sClient.Create(context.Background(), defaultCluster())).To(Succeed()) }) + Context("CloneSpecs", func() { + It("Should not allow Cluster without ControlPlane nodes", func() { + dc := defaultCluster() + dc.Spec.CloneSpec.ProxmoxMachineSpec = map[string]ProxmoxMachineSpec{} + + Expect(k8sClient.Create(context.Background(), dc)).Should(MatchError(ContainSubstring("control plane"))) + }) + }) + Context("IPV6Config", func() { It("Should not allow empty addresses", func() { dc := defaultCluster() diff --git a/api/v1alpha1/proxmoxclustertemplate_types.go b/api/v1alpha1/proxmoxclustertemplate_types.go new file mode 100644 index 00000000..1c431dfe --- /dev/null +++ b/api/v1alpha1/proxmoxclustertemplate_types.go @@ -0,0 +1,60 @@ +/* +Copyright 2024 IONOS Cloud. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" +) + +// ProxmoxClusterTemplateSpec defines the desired state of ProxmoxClusterTemplate. +type ProxmoxClusterTemplateSpec struct { + Template ProxmoxClusterTemplateResource `json:"template"` +} + +// ProxmoxClusterTemplateResource defines the spec and metadata for ProxmoxClusterTemplate supported by capi. +type ProxmoxClusterTemplateResource struct { + // Standard object's metadata. + // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + // +optional + ObjectMeta clusterv1.ObjectMeta `json:"metadata,omitempty"` + Spec ProxmoxClusterSpec `json:"spec"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// ProxmoxClusterTemplate is the Schema for the proxmoxclustertemplates API. +type ProxmoxClusterTemplate struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ProxmoxClusterTemplateSpec `json:"spec,omitempty"` +} + +//+kubebuilder:object:root=true + +// ProxmoxClusterTemplateList contains a list of ProxmoxClusterTemplate. +type ProxmoxClusterTemplateList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ProxmoxClusterTemplate `json:"items"` +} + +func init() { + objectTypes = append(objectTypes, &ProxmoxClusterTemplate{}, &ProxmoxClusterTemplateList{}) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 9fd4b107..e8985546 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -269,6 +269,33 @@ func (in *ProxmoxCluster) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxmoxClusterCloneSpec) DeepCopyInto(out *ProxmoxClusterCloneSpec) { + *out = *in + if in.ProxmoxMachineSpec != nil { + in, out := &in.ProxmoxMachineSpec, &out.ProxmoxMachineSpec + *out = make(map[string]ProxmoxMachineSpec, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } + if in.SSHAuthorizedKeys != nil { + in, out := &in.SSHAuthorizedKeys, &out.SSHAuthorizedKeys + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxmoxClusterCloneSpec. +func (in *ProxmoxClusterCloneSpec) DeepCopy() *ProxmoxClusterCloneSpec { + if in == nil { + return nil + } + out := new(ProxmoxClusterCloneSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProxmoxClusterList) DeepCopyInto(out *ProxmoxClusterList) { *out = *in @@ -330,6 +357,11 @@ func (in *ProxmoxClusterSpec) DeepCopyInto(out *ProxmoxClusterSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.CloneSpec != nil { + in, out := &in.CloneSpec, &out.CloneSpec + *out = new(ProxmoxClusterCloneSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxmoxClusterSpec. @@ -374,6 +406,97 @@ func (in *ProxmoxClusterStatus) DeepCopy() *ProxmoxClusterStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxmoxClusterTemplate) DeepCopyInto(out *ProxmoxClusterTemplate) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxmoxClusterTemplate. +func (in *ProxmoxClusterTemplate) DeepCopy() *ProxmoxClusterTemplate { + if in == nil { + return nil + } + out := new(ProxmoxClusterTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ProxmoxClusterTemplate) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxmoxClusterTemplateList) DeepCopyInto(out *ProxmoxClusterTemplateList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ProxmoxClusterTemplate, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxmoxClusterTemplateList. +func (in *ProxmoxClusterTemplateList) DeepCopy() *ProxmoxClusterTemplateList { + if in == nil { + return nil + } + out := new(ProxmoxClusterTemplateList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ProxmoxClusterTemplateList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxmoxClusterTemplateResource) DeepCopyInto(out *ProxmoxClusterTemplateResource) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxmoxClusterTemplateResource. +func (in *ProxmoxClusterTemplateResource) DeepCopy() *ProxmoxClusterTemplateResource { + if in == nil { + return nil + } + out := new(ProxmoxClusterTemplateResource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxmoxClusterTemplateSpec) DeepCopyInto(out *ProxmoxClusterTemplateSpec) { + *out = *in + in.Template.DeepCopyInto(&out.Template) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxmoxClusterTemplateSpec. +func (in *ProxmoxClusterTemplateSpec) DeepCopy() *ProxmoxClusterTemplateSpec { + if in == nil { + return nil + } + out := new(ProxmoxClusterTemplateSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProxmoxMachine) DeepCopyInto(out *ProxmoxMachine) { *out = *in diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_proxmoxclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_proxmoxclusters.yaml index 17a41e81..5bdd3018 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_proxmoxclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_proxmoxclusters.yaml @@ -57,6 +57,498 @@ spec: items: type: string type: array + cloneSpec: + description: NodeCloneSpec is the configuration pertaining to all + items configurable in the configuration and cloning of a proxmox + VM. Multiple types of nodes can be specified. + properties: + machineSpec: + additionalProperties: + description: ProxmoxMachineSpec defines the desired state of + a ProxmoxMachine. + properties: + description: + description: Description for the new VM. + type: string + disks: + description: Disks contains a set of disk configuration + options, which will be applied before the first startup. + properties: + bootVolume: + description: BootVolume defines the storage size for + the boot volume. This field is optional, and should + only be set if you want to change the size of the + boot volume. + properties: + disk: + description: 'Disk is the name of the disk device, + that should be resized. Example values are: ide[0-3], + scsi[0-30], sata[0-5].' + type: string + sizeGb: + description: "Size defines the size in gigabyte. + \n As Proxmox does not support shrinking, the + size must be bigger than the already configured + size in the template." + format: int32 + minimum: 5 + type: integer + required: + - disk + - sizeGb + type: object + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + type: object + format: + default: raw + description: Format for file storage. Only valid for full + clone. + enum: + - raw + - qcow2 + - vmdk + type: string + full: + default: true + description: Full Create a full copy of all disks. This + is always done when you clone a normal VM. Create a Full + clone by default. + type: boolean + memoryMiB: + description: MemoryMiB is the size of a virtual machine's + memory, in MiB. Defaults to the property value in the + template from which the virtual machine is cloned. + format: int32 + multipleOf: 8 + type: integer + network: + description: Network is the network configuration for this + machine's VM. + properties: + additionalDevices: + description: AdditionalDevices defines additional network + devices bound to the virtual machine. + items: + description: AdditionalNetworkDevice the definition + of a Proxmox network device. + properties: + bridge: + description: Bridge is the network bridge to attach + to the machine. + minLength: 1 + type: string + dnsServers: + description: DNSServers contains information about + nameservers to be used for this interface. If + this field is not set, it will use the default + dns servers from the ProxmoxCluster. + items: + type: string + minItems: 1 + type: array + ipv4PoolRef: + description: IPv4PoolRef is a reference to an + IPAM Pool resource, which exposes IPv4 addresses. + The network device will use an available IP + address from the referenced pool. This can be + combined with `IPv6PoolRef` in order to enable + dual stack. + properties: + apiGroup: + description: APIGroup is the group for the + resource being referenced. If APIGroup is + not specified, the specified Kind must be + in the core API group. For any other third-party + types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: ipv4PoolRef allows only IPAM apiGroup + ipam.cluster.x-k8s.io + rule: self.apiGroup == 'ipam.cluster.x-k8s.io' + - message: ipv4PoolRef allows either InClusterIPPool + or GlobalInClusterIPPool + rule: self.kind == 'InClusterIPPool' || self.kind + == 'GlobalInClusterIPPool' + ipv6PoolRef: + description: IPv6PoolRef is a reference to an + IPAM pool resource, which exposes IPv6 addresses. + The network device will use an available IP + address from the referenced pool. this can be + combined with `IPv4PoolRef` in order to enable + dual stack. + properties: + apiGroup: + description: APIGroup is the group for the + resource being referenced. If APIGroup is + not specified, the specified Kind must be + in the core API group. For any other third-party + types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: ipv6PoolRef allows only IPAM apiGroup + ipam.cluster.x-k8s.io + rule: self.apiGroup == 'ipam.cluster.x-k8s.io' + - message: ipv6PoolRef allows either InClusterIPPool + or GlobalInClusterIPPool + rule: self.kind == 'InClusterIPPool' || self.kind + == 'GlobalInClusterIPPool' + model: + default: virtio + description: Model is the network device model. + enum: + - e1000 + - virtio + - rtl8139 + - vmxnet3 + type: string + mtu: + description: MTU is the network device Maximum + Transmission Unit. Only works with virtio Model. + Set to 1 to inherit the MTU value from the underlying + bridge. + maximum: 65520 + minimum: 1 + type: integer + name: + description: Name is the network device name. + Must be unique within the virtual machine and + different from the primary device 'net0'. + minLength: 1 + type: string + x-kubernetes-validations: + - message: additional network devices doesn't + allow net0 + rule: self != 'net0' + vlan: + description: VLAN is the network L2 VLAN. + maximum: 4094 + minimum: 1 + type: integer + required: + - bridge + - name + type: object + x-kubernetes-validations: + - message: at least one pool reference must be set, + either ipv4PoolRef or ipv6PoolRef + rule: self.ipv4PoolRef != null || self.ipv6PoolRef + != null + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + default: + description: Default is the default network device, + which will be used for the primary network interface. + net0 is always the default network device. + properties: + bridge: + description: Bridge is the network bridge to attach + to the machine. + minLength: 1 + type: string + model: + default: virtio + description: Model is the network device model. + enum: + - e1000 + - virtio + - rtl8139 + - vmxnet3 + type: string + mtu: + description: MTU is the network device Maximum Transmission + Unit. Only works with virtio Model. Set to 1 to + inherit the MTU value from the underlying bridge. + maximum: 65520 + minimum: 1 + type: integer + vlan: + description: VLAN is the network L2 VLAN. + maximum: 4094 + minimum: 1 + type: integer + required: + - bridge + type: object + vrfs: + description: Definition of a VRF Device. + items: + description: VRFDevice defines Virtual Routing Flow + devices. + properties: + dnsServers: + description: DNSServers contains information about + nameservers to be used for this interface. If + this field is not set, it will use the default + dns servers from the ProxmoxCluster. + items: + type: string + minItems: 1 + type: array + interfaces: + description: Interfaces is the list of proxmox + network devices managed by this virtual device. + items: + type: string + type: array + ipv4PoolRef: + description: IPv4PoolRef is a reference to an + IPAM Pool resource, which exposes IPv4 addresses. + The network device will use an available IP + address from the referenced pool. This can be + combined with `IPv6PoolRef` in order to enable + dual stack. + properties: + apiGroup: + description: APIGroup is the group for the + resource being referenced. If APIGroup is + not specified, the specified Kind must be + in the core API group. For any other third-party + types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: ipv4PoolRef allows only IPAM apiGroup + ipam.cluster.x-k8s.io + rule: self.apiGroup == 'ipam.cluster.x-k8s.io' + - message: ipv4PoolRef allows either InClusterIPPool + or GlobalInClusterIPPool + rule: self.kind == 'InClusterIPPool' || self.kind + == 'GlobalInClusterIPPool' + ipv6PoolRef: + description: IPv6PoolRef is a reference to an + IPAM pool resource, which exposes IPv6 addresses. + The network device will use an available IP + address from the referenced pool. this can be + combined with `IPv4PoolRef` in order to enable + dual stack. + properties: + apiGroup: + description: APIGroup is the group for the + resource being referenced. If APIGroup is + not specified, the specified Kind must be + in the core API group. For any other third-party + types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: ipv6PoolRef allows only IPAM apiGroup + ipam.cluster.x-k8s.io + rule: self.apiGroup == 'ipam.cluster.x-k8s.io' + - message: ipv6PoolRef allows either InClusterIPPool + or GlobalInClusterIPPool + rule: self.kind == 'InClusterIPPool' || self.kind + == 'GlobalInClusterIPPool' + name: + description: Name is the virtual network device + name. Must be unique within the virtual machine. + minLength: 3 + type: string + routes: + description: Routes are the routes associated + with the l3mdev policy. + items: + description: RouteSpec describes an IPv4/IPv6 + Route. + properties: + metric: + description: Metric is the priority of the + route in the routing table. + format: int32 + type: integer + table: + description: Table is the routing table + used for this route. + format: int32 + type: integer + to: + description: To is the subnet to be routed. + type: string + via: + description: Via is the gateway to the subnet. + type: string + type: object + minItems: 1 + type: array + routingPolicy: + description: RoutingPolicy is the l3mdev policy + inserted into FIB. + items: + description: RoutingPolicySpec is a Linux FIB + rule. + properties: + from: + description: From is the subnet of the source. + type: string + priority: + description: Priority is the position in + the ip rule FIB table. + format: int32 + maximum: 4294967295 + type: integer + x-kubernetes-validations: + - message: Cowardly refusing to insert fib + rule matching kernel rules + rule: (self > 0 && self < 32765) || (self + > 32766) + table: + description: Table is the routing table + ID. + format: int32 + type: integer + to: + description: To is the subnet of the target. + type: string + type: object + minItems: 1 + type: array + table: + description: Table is the ID of the routing table + used for the l3mdev vrf device. + format: int32 + maximum: 4294967295 + type: integer + x-kubernetes-validations: + - message: Cowardly refusing to insert l3mdev + rules into kernel tables + rule: (self > 0 && self < 254) || (self > 255) + required: + - name + - table + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + numCores: + description: NumCores is the number of cores per CPU socket + in a virtual machine. Defaults to the property value in + the template from which the virtual machine is cloned. + format: int32 + minimum: 1 + type: integer + numSockets: + description: NumSockets is the number of CPU sockets in + a virtual machine. Defaults to the property value in the + template from which the virtual machine is cloned. + format: int32 + minimum: 1 + type: integer + pool: + description: Pool Add the new VM to the specified pool. + type: string + providerID: + description: ProviderID is the virtual machine BIOS UUID + formatted as proxmox://6c3fa683-bef9-4425-b413-eaa45a9d6191 + type: string + snapName: + description: SnapName The name of the snapshot. + type: string + sourceNode: + description: "SourceNode is the initially selected proxmox + node. This node will be used to locate the template VM, + which will be used for cloning operations. \n Cloning + will be performed according to the configuration. Setting + the `Target` field will tell Proxmox to clone the VM on + that target node. \n When Target is not set and the ProxmoxCluster + contains a set of `AllowedNodes`, the algorithm will instead + evenly distribute the VMs across the nodes from that list. + \n If neither a `Target` nor `AllowedNodes` was set, the + VM will be cloned onto the same node as SourceNode." + minLength: 1 + type: string + storage: + description: Storage for full clone. + type: string + target: + description: Target node. Only allowed if the original VM + is on shared storage. + type: string + templateID: + description: TemplateID the vm_template vmid used for cloning + a new VM. + format: int32 + type: integer + virtualMachineID: + description: VirtualMachineID is the Proxmox identifier + for the ProxmoxMachine VM. + format: int64 + type: integer + required: + - sourceNode + type: object + type: object + x-kubernetes-validations: + - message: Cowardly refusing to deploy cluster without control + plane + rule: has(self.controlPlane) + sshAuthorizedKeys: + description: SshAuthorizedKeys contains the authorized keys deployed + to the PROXMOX VMs. + items: + type: string + type: array + virtualIPNetworkInterface: + description: VirtualIPNetworkInterface is the interface the k8s + control plane binds to. + type: string + required: + - machineSpec + type: object controlPlaneEndpoint: description: ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_proxmoxclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_proxmoxclustertemplates.yaml new file mode 100644 index 00000000..f53dd2f9 --- /dev/null +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_proxmoxclustertemplates.yaml @@ -0,0 +1,702 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.3 + creationTimestamp: null + name: proxmoxclustertemplates.infrastructure.cluster.x-k8s.io +spec: + group: infrastructure.cluster.x-k8s.io + names: + kind: ProxmoxClusterTemplate + listKind: ProxmoxClusterTemplateList + plural: proxmoxclustertemplates + singular: proxmoxclustertemplate + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: ProxmoxClusterTemplate is the Schema for the proxmoxclustertemplates + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ProxmoxClusterTemplateSpec defines the desired state of ProxmoxClusterTemplate. + properties: + template: + description: ProxmoxClusterTemplateResource defines the spec and metadata + for ProxmoxClusterTemplate supported by capi. + properties: + metadata: + description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata' + properties: + annotations: + additionalProperties: + type: string + description: 'Annotations is an unstructured key value map + stored with a resource that may be set by external tools + to store and retrieve arbitrary metadata. They are not queryable + and should be preserved when modifying objects. More info: + http://kubernetes.io/docs/user-guide/annotations' + type: object + labels: + additionalProperties: + type: string + description: 'Map of string keys and values that can be used + to organize and categorize (scope and select) objects. May + match selectors of replication controllers and services. + More info: http://kubernetes.io/docs/user-guide/labels' + type: object + type: object + spec: + description: ProxmoxClusterSpec defines the desired state of a + ProxmoxCluster. + properties: + allowedNodes: + description: AllowedNodes specifies all Proxmox nodes which + will be considered for operations. This implies that VMs + can be cloned on different nodes from the node which holds + the VM template. + items: + type: string + type: array + cloneSpec: + description: NodeCloneSpec is the configuration pertaining + to all items configurable in the configuration and cloning + of a proxmox VM. Multiple types of nodes can be specified. + properties: + machineSpec: + additionalProperties: + description: ProxmoxMachineSpec defines the desired + state of a ProxmoxMachine. + properties: + description: + description: Description for the new VM. + type: string + disks: + description: Disks contains a set of disk configuration + options, which will be applied before the first + startup. + properties: + bootVolume: + description: BootVolume defines the storage + size for the boot volume. This field is optional, + and should only be set if you want to change + the size of the boot volume. + properties: + disk: + description: 'Disk is the name of the disk + device, that should be resized. Example + values are: ide[0-3], scsi[0-30], sata[0-5].' + type: string + sizeGb: + description: "Size defines the size in gigabyte. + \n As Proxmox does not support shrinking, + the size must be bigger than the already + configured size in the template." + format: int32 + minimum: 5 + type: integer + required: + - disk + - sizeGb + type: object + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + type: object + format: + default: raw + description: Format for file storage. Only valid + for full clone. + enum: + - raw + - qcow2 + - vmdk + type: string + full: + default: true + description: Full Create a full copy of all disks. + This is always done when you clone a normal VM. + Create a Full clone by default. + type: boolean + memoryMiB: + description: MemoryMiB is the size of a virtual + machine's memory, in MiB. Defaults to the property + value in the template from which the virtual machine + is cloned. + format: int32 + multipleOf: 8 + type: integer + network: + description: Network is the network configuration + for this machine's VM. + properties: + additionalDevices: + description: AdditionalDevices defines additional + network devices bound to the virtual machine. + items: + description: AdditionalNetworkDevice the definition + of a Proxmox network device. + properties: + bridge: + description: Bridge is the network bridge + to attach to the machine. + minLength: 1 + type: string + dnsServers: + description: DNSServers contains information + about nameservers to be used for this + interface. If this field is not set, + it will use the default dns servers + from the ProxmoxCluster. + items: + type: string + minItems: 1 + type: array + ipv4PoolRef: + description: IPv4PoolRef is a reference + to an IPAM Pool resource, which exposes + IPv4 addresses. The network device will + use an available IP address from the + referenced pool. This can be combined + with `IPv6PoolRef` in order to enable + dual stack. + properties: + apiGroup: + description: APIGroup is the group + for the resource being referenced. + If APIGroup is not specified, the + specified Kind must be in the core + API group. For any other third-party + types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: ipv4PoolRef allows only IPAM + apiGroup ipam.cluster.x-k8s.io + rule: self.apiGroup == 'ipam.cluster.x-k8s.io' + - message: ipv4PoolRef allows either InClusterIPPool + or GlobalInClusterIPPool + rule: self.kind == 'InClusterIPPool' + || self.kind == 'GlobalInClusterIPPool' + ipv6PoolRef: + description: IPv6PoolRef is a reference + to an IPAM pool resource, which exposes + IPv6 addresses. The network device will + use an available IP address from the + referenced pool. this can be combined + with `IPv4PoolRef` in order to enable + dual stack. + properties: + apiGroup: + description: APIGroup is the group + for the resource being referenced. + If APIGroup is not specified, the + specified Kind must be in the core + API group. For any other third-party + types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: ipv6PoolRef allows only IPAM + apiGroup ipam.cluster.x-k8s.io + rule: self.apiGroup == 'ipam.cluster.x-k8s.io' + - message: ipv6PoolRef allows either InClusterIPPool + or GlobalInClusterIPPool + rule: self.kind == 'InClusterIPPool' + || self.kind == 'GlobalInClusterIPPool' + model: + default: virtio + description: Model is the network device + model. + enum: + - e1000 + - virtio + - rtl8139 + - vmxnet3 + type: string + mtu: + description: MTU is the network device + Maximum Transmission Unit. Only works + with virtio Model. Set to 1 to inherit + the MTU value from the underlying bridge. + maximum: 65520 + minimum: 1 + type: integer + name: + description: Name is the network device + name. Must be unique within the virtual + machine and different from the primary + device 'net0'. + minLength: 1 + type: string + x-kubernetes-validations: + - message: additional network devices + doesn't allow net0 + rule: self != 'net0' + vlan: + description: VLAN is the network L2 VLAN. + maximum: 4094 + minimum: 1 + type: integer + required: + - bridge + - name + type: object + x-kubernetes-validations: + - message: at least one pool reference must + be set, either ipv4PoolRef or ipv6PoolRef + rule: self.ipv4PoolRef != null || self.ipv6PoolRef + != null + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + default: + description: Default is the default network + device, which will be used for the primary + network interface. net0 is always the default + network device. + properties: + bridge: + description: Bridge is the network bridge + to attach to the machine. + minLength: 1 + type: string + model: + default: virtio + description: Model is the network device + model. + enum: + - e1000 + - virtio + - rtl8139 + - vmxnet3 + type: string + mtu: + description: MTU is the network device Maximum + Transmission Unit. Only works with virtio + Model. Set to 1 to inherit the MTU value + from the underlying bridge. + maximum: 65520 + minimum: 1 + type: integer + vlan: + description: VLAN is the network L2 VLAN. + maximum: 4094 + minimum: 1 + type: integer + required: + - bridge + type: object + vrfs: + description: Definition of a VRF Device. + items: + description: VRFDevice defines Virtual Routing + Flow devices. + properties: + dnsServers: + description: DNSServers contains information + about nameservers to be used for this + interface. If this field is not set, + it will use the default dns servers + from the ProxmoxCluster. + items: + type: string + minItems: 1 + type: array + interfaces: + description: Interfaces is the list of + proxmox network devices managed by this + virtual device. + items: + type: string + type: array + ipv4PoolRef: + description: IPv4PoolRef is a reference + to an IPAM Pool resource, which exposes + IPv4 addresses. The network device will + use an available IP address from the + referenced pool. This can be combined + with `IPv6PoolRef` in order to enable + dual stack. + properties: + apiGroup: + description: APIGroup is the group + for the resource being referenced. + If APIGroup is not specified, the + specified Kind must be in the core + API group. For any other third-party + types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: ipv4PoolRef allows only IPAM + apiGroup ipam.cluster.x-k8s.io + rule: self.apiGroup == 'ipam.cluster.x-k8s.io' + - message: ipv4PoolRef allows either InClusterIPPool + or GlobalInClusterIPPool + rule: self.kind == 'InClusterIPPool' + || self.kind == 'GlobalInClusterIPPool' + ipv6PoolRef: + description: IPv6PoolRef is a reference + to an IPAM pool resource, which exposes + IPv6 addresses. The network device will + use an available IP address from the + referenced pool. this can be combined + with `IPv4PoolRef` in order to enable + dual stack. + properties: + apiGroup: + description: APIGroup is the group + for the resource being referenced. + If APIGroup is not specified, the + specified Kind must be in the core + API group. For any other third-party + types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: ipv6PoolRef allows only IPAM + apiGroup ipam.cluster.x-k8s.io + rule: self.apiGroup == 'ipam.cluster.x-k8s.io' + - message: ipv6PoolRef allows either InClusterIPPool + or GlobalInClusterIPPool + rule: self.kind == 'InClusterIPPool' + || self.kind == 'GlobalInClusterIPPool' + name: + description: Name is the virtual network + device name. Must be unique within the + virtual machine. + minLength: 3 + type: string + routes: + description: Routes are the routes associated + with the l3mdev policy. + items: + description: RouteSpec describes an + IPv4/IPv6 Route. + properties: + metric: + description: Metric is the priority + of the route in the routing table. + format: int32 + type: integer + table: + description: Table is the routing + table used for this route. + format: int32 + type: integer + to: + description: To is the subnet to + be routed. + type: string + via: + description: Via is the gateway + to the subnet. + type: string + type: object + minItems: 1 + type: array + routingPolicy: + description: RoutingPolicy is the l3mdev + policy inserted into FIB. + items: + description: RoutingPolicySpec is a + Linux FIB rule. + properties: + from: + description: From is the subnet + of the source. + type: string + priority: + description: Priority is the position + in the ip rule FIB table. + format: int32 + maximum: 4294967295 + type: integer + x-kubernetes-validations: + - message: Cowardly refusing to + insert fib rule matching kernel + rules + rule: (self > 0 && self < 32765) + || (self > 32766) + table: + description: Table is the routing + table ID. + format: int32 + type: integer + to: + description: To is the subnet of + the target. + type: string + type: object + minItems: 1 + type: array + table: + description: Table is the ID of the routing + table used for the l3mdev vrf device. + format: int32 + maximum: 4294967295 + type: integer + x-kubernetes-validations: + - message: Cowardly refusing to insert + l3mdev rules into kernel tables + rule: (self > 0 && self < 254) || (self + > 255) + required: + - name + - table + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + numCores: + description: NumCores is the number of cores per + CPU socket in a virtual machine. Defaults to the + property value in the template from which the + virtual machine is cloned. + format: int32 + minimum: 1 + type: integer + numSockets: + description: NumSockets is the number of CPU sockets + in a virtual machine. Defaults to the property + value in the template from which the virtual machine + is cloned. + format: int32 + minimum: 1 + type: integer + pool: + description: Pool Add the new VM to the specified + pool. + type: string + providerID: + description: ProviderID is the virtual machine BIOS + UUID formatted as proxmox://6c3fa683-bef9-4425-b413-eaa45a9d6191 + type: string + snapName: + description: SnapName The name of the snapshot. + type: string + sourceNode: + description: "SourceNode is the initially selected + proxmox node. This node will be used to locate + the template VM, which will be used for cloning + operations. \n Cloning will be performed according + to the configuration. Setting the `Target` field + will tell Proxmox to clone the VM on that target + node. \n When Target is not set and the ProxmoxCluster + contains a set of `AllowedNodes`, the algorithm + will instead evenly distribute the VMs across + the nodes from that list. \n If neither a `Target` + nor `AllowedNodes` was set, the VM will be cloned + onto the same node as SourceNode." + minLength: 1 + type: string + storage: + description: Storage for full clone. + type: string + target: + description: Target node. Only allowed if the original + VM is on shared storage. + type: string + templateID: + description: TemplateID the vm_template vmid used + for cloning a new VM. + format: int32 + type: integer + virtualMachineID: + description: VirtualMachineID is the Proxmox identifier + for the ProxmoxMachine VM. + format: int64 + type: integer + required: + - sourceNode + type: object + type: object + x-kubernetes-validations: + - message: Cowardly refusing to deploy cluster without + control plane + rule: has(self.controlPlane) + sshAuthorizedKeys: + description: SshAuthorizedKeys contains the authorized + keys deployed to the PROXMOX VMs. + items: + type: string + type: array + virtualIPNetworkInterface: + description: VirtualIPNetworkInterface is the interface + the k8s control plane binds to. + type: string + required: + - machineSpec + type: object + controlPlaneEndpoint: + description: ControlPlaneEndpoint represents the endpoint + used to communicate with the control plane. + properties: + host: + description: The hostname on which the API server is serving. + type: string + port: + description: The port on which the API server is serving. + format: int32 + type: integer + required: + - host + - port + type: object + dnsServers: + description: DNSServers contains information about nameservers + used by the machines. + items: + type: string + minItems: 1 + type: array + ipv4Config: + description: IPv4Config contains information about available + IPV4 address pools and the gateway. This can be combined + with ipv6Config in order to enable dual stack. Either IPv4Config + or IPv6Config must be provided. + properties: + addresses: + description: Addresses is a list of IP addresses that + can be assigned. This set of addresses can be non-contiguous. + items: + type: string + type: array + gateway: + description: Gateway + type: string + prefix: + description: Prefix is the network prefix to use. + maximum: 128 + type: integer + required: + - addresses + - prefix + type: object + x-kubernetes-validations: + - message: IPv4Config addresses must be provided + rule: self.addresses.size() > 0 + ipv6Config: + description: IPv6Config contains information about available + IPV6 address pools and the gateway. This can be combined + with ipv4Config in order to enable dual stack. Either IPv4Config + or IPv6Config must be provided. + properties: + addresses: + description: Addresses is a list of IP addresses that + can be assigned. This set of addresses can be non-contiguous. + items: + type: string + type: array + gateway: + description: Gateway + type: string + prefix: + description: Prefix is the network prefix to use. + maximum: 128 + type: integer + required: + - addresses + - prefix + type: object + x-kubernetes-validations: + - message: IPv6Config addresses must be provided + rule: self.addresses.size() > 0 + schedulerHints: + description: SchedulerHints allows to influence the decision + on where a VM will be scheduled. For example by applying + a multiplicator to a node's resources, to allow for overprovisioning + or to ensure a node will always have a safety buffer. + properties: + memoryAdjustment: + description: MemoryAdjustment allows to adjust a node's + memory by a given percentage. For example, setting it + to 300 allows to allocate 300% of a host's memory for + VMs, and setting it to 95 limits memory allocation to + 95% of a host's memory. Setting it to 0 entirely disables + scheduling memory constraints. By default 100% of a + node's memory will be used for allocation. + format: int64 + type: integer + type: object + required: + - dnsServers + type: object + required: + - spec + type: object + required: + - template + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 0cfa06fc..6107a743 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -3,6 +3,7 @@ # It should be run by config/default resources: - bases/infrastructure.cluster.x-k8s.io_proxmoxclusters.yaml +- bases/infrastructure.cluster.x-k8s.io_proxmoxclustertemplates.yaml - bases/infrastructure.cluster.x-k8s.io_proxmoxmachines.yaml - bases/infrastructure.cluster.x-k8s.io_proxmoxmachinetemplates.yaml #+kubebuilder:scaffold:crdkustomizeresource diff --git a/config/samples/infrastructure_v1alpha1_proxmoxclustertemplate.yaml b/config/samples/infrastructure_v1alpha1_proxmoxclustertemplate.yaml new file mode 100644 index 00000000..c2bf065c --- /dev/null +++ b/config/samples/infrastructure_v1alpha1_proxmoxclustertemplate.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: ProxmoxClusterTemplate +metadata: + labels: + app.kubernetes.io/name: proxmoxclustertemplate + app.kubernetes.io/instance: proxmoxclustertemplate-sample + app.kubernetes.io/part-of: cluster-api-provider-proxmox + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: cluster-api-provider-proxmox + name: proxmoxclustertemplate-sample +spec: diff --git a/docs/Usage.md b/docs/Usage.md index 03b6ff22..7fa8a65a 100644 --- a/docs/Usage.md +++ b/docs/Usage.md @@ -19,6 +19,7 @@ Table of contents * [Additional flavors](#additional-flavors) * [Cleaning a cluster](#cleaning-a-cluster) * [Custom cluster templates](#custom-cluster-templates) + * [Using Cluster Classes](#using-cluster-classes) @@ -105,6 +106,7 @@ NUM_CORES: "4" # The number of co MEMORY_MIB: "8048" # The memory size for the VMs. EXP_CLUSTER_RESOURCE_SET: "true" # This enables the ClusterResourceSet feature that we are using to deploy CNI +CLUSTER_TOPOLOGY: "true" # This enables experimental ClusterClass templating ``` the `CONTROL_PLANE_ENDPOINT_IP` is an IP that must be on the same subnet as the control plane machines @@ -243,3 +245,60 @@ $ clusterctl generate custom-cluster proxmox-quickstart \ --worker-machine-count 3 \ --from ~/workspace/custom-cluster-template.yaml > custom-cluster.yaml ``` + +## Using Cluster Classes +[ClusterClass](https://cluster-api.sigs.k8s.io/tasks/experimental-features/cluster-class/) +is an experimental feature to manage clusters without templating. In this case, you only +need to write the cluster definition (referring to the cluster class), and all required resources +are automatically created for you. + +This feature requires [CLUSTER_TOPOLOGY](https://cluster-api.sigs.k8s.io/tasks/experimental-features/experimental-features#enabling-experimental-features-on-tilt) +to be set in your capi controller and in the environment of clusterctl. + +We provide the following ClusterClasses: + +| Flavor | Template File | CRS File | Example Cluster Manifest | +|----------------| ----------------------------------------------- |-------------------------------|------------------------------- +| cilium | templates/cluster-class-cilium.yaml | templates/crs/cni/cilium.yaml | examples/cluster-cilium.yaml | +| calico | templates/cluster-class-calico.yaml | templates/crs/cni/calico.yaml | examples/cluster-calico.yaml | +| default | templates/cluster-class.yaml | - | examples/cluster.yaml | + +### Creating a cluster from a ClusterClass +1. Choose a ClusterClass +All ClusterClasses provide the same features except for the CNI they refer to. The base ClusterClass +also does not provide MachineHealthChecks as those can not be successful until a CNI is deployed. + +We recommend that you start with a ClusterClass which defines a CNI. Please +refer to [CNI Cilium](#flavor-with-cilium-cni) for details on how to get started. + +Apply the ClusterClass custom resource definition so you can create cluster manifests: + +```bash +kubectl apply -f templates/cluster-class-cilium.yaml +``` + +2. Write the cluster manifest +An example can be found in [examples/cluster-cilum.yaml](../examples/cluster-cilium.yaml). + +Important fields: +- `.metadata.name: cluster-name` the name of the cluster to be generated. +- `.spec.topology.class: `proxmox-clusterclass-cilium-v0.1.0` the clusterClass used for generating resources. +- `.spec.topology.version: 1.25.10` The k8s version used by kubeadm. + +All possible fields refer to [CAPMOX environment variables](#capmox-environment-variables). + +3. Preview the cluster topology + +```bash +clusterctl alpha topology plan -f examples/cluster-cilium.yaml -o out/ +``` + +The to-be-created resources will be located in `out/created`. + +4. Apply the Cluster Manifest + +```bash +kubectl apply -f mycluster.yaml +``` + +If you run into issues, refer to [Cluster Health and deployment status](#cluster-health-and-deployment-status). diff --git a/examples/cluster-calico.yaml b/examples/cluster-calico.yaml new file mode 100644 index 00000000..0cebab88 --- /dev/null +++ b/examples/cluster-calico.yaml @@ -0,0 +1,146 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cluster.x-k8s.io/proxmox-cluster-cni: calico + name: capmox-cluster +spec: + topology: + class: proxmox-clusterclass-calico-v0.1.0 + version: 1.27.8 + controlPlane: + replicas: 1 + workers: + machineDeployments: + - class: proxmox-worker + name: proxmox-worker-pool + replicas: 3 + - class: proxmox-loadbalancer + name: proxmox-loadbalancer-pool + replicas: 0 + variables: + - name: allowedNodes + value: + - pve1 + - pve2 + - pve3 + - name: controlPlaneEndpoint + value: + host: 10.10.10.9 + port: 6443 + - name: dnsServers + value: [8.8.8.8, 8.8.4.4] + - name: ipv4Config + value: + addresses: [10.10.10.10-10.10.10.20] + gateway: 10.10.10.1 + prefix: 24 +# - name: ipv6Config +# value: +# addresses: [2001:db8:1::10-2001:db8:1::20] +# gateway: 2001:db8:1::1 +# prefix: 64 + - name: kubeProxy + value: + mode: iptables + - name: cloneSpec + value: + machineSpec: + controlPlane: + network: + default: + bridge: vmbr0 +# additionalDevices: +# - name: net1 +# model: virtio +# ipv4PoolRef: +# name: shared-inclusteripv4pool +# apiGroup: ipam.cluster.x-k8s.io +# kind: GlobalInClusterIPPool +# ipv6PoolRef: +# name: shared-inclusteripv6pool +# apiGroup: ipam.cluster.x-k8s.io +# kind: GlobalInClusterIPPool +# bridge: vmbr1 +# numSockets: 1 +# numCores: 4 +# memoryMiB: 4096 + sourceNode: pve1 + templateID: 100 + workerNode: + network: + default: + bridge: vmbr0 +# additionalDevices: +# - name: net1 +# model: virtio +# ipv4PoolRef: +# name: shared-inclusteripv4pool +# apiGroup: ipam.cluster.x-k8s.io +# kind: GlobalInClusterIPPool +# ipv6PoolRef: +# name: shared-inclusteripv6pool +# apiGroup: ipam.cluster.x-k8s.io +# kind: GlobalInClusterIPPool +# bridge: vmbr1 +# numSockets: 1 +# numCores: 4 +# memoryMiB: 4096 + sourceNode: pve1 + templateID: 100 + loadBalancer: + network: + default: + bridge: vmbr0 +# additionalDevices: +# - name: net1 +# model: virtio +# ipv4PoolRef: +# name: shared-inclusteripv4pool +# apiGroup: ipam.cluster.x-k8s.io +# kind: GlobalInClusterIPPool +# ipv6PoolRef: +# name: shared-inclusteripv6pool +# apiGroup: ipam.cluster.x-k8s.io +# kind: GlobalInClusterIPPool +# bridge: vmbr1 +# vrfs: +# - name: vrf-ext +# interfaces: +# - net1 +# table: 500 +# routingPolicy: +# - from: 192.0.2.0/24 +# numSockets: 1 +# numCores: 2 +# memoryMiB: 2048 + sourceNode: pve1 + templateID: 100 + sshAuthorizedKeys: + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJPK5kBd7cxXAHZ6UbeE+ysOlSjOFare3fCCZJ3xtXt1 capmox@k8s.io +# virtualIPNetworkInterface: eno1 +# --- +# apiVersion: ipam.cluster.x-k8s.io/v1alpha2 +# kind: GlobalInClusterIPPool +# metadata: +# name: shared-inclusteripv4pool +# spec: +# addresses: +# - 10.10.11.1 +# - 10.10.11.2 +# - 10.10.11.3 +# prefix: 24 +# gateway: 10.10.11.254 +# --- +# apiVersion: ipam.cluster.x-k8s.io/v1alpha2 +# kind: GlobalInClusterIPPool +# metadata: +# name: shared-inclusteripv6pool +# spec: +# addresses: +# - 2001:db8::10 +# - 2001:db8::11 +# - 2001:db8::12 +# prefix: 64 +# gateway: 2001:db8::1 diff --git a/examples/cluster-cilium.yaml b/examples/cluster-cilium.yaml new file mode 100644 index 00000000..6de39d8c --- /dev/null +++ b/examples/cluster-cilium.yaml @@ -0,0 +1,146 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cluster.x-k8s.io/proxmox-cluster-cni: cilium + name: capmox-cluster +spec: + topology: + class: proxmox-clusterclass-cilium-v0.1.0 + version: 1.27.8 + controlPlane: + replicas: 1 + workers: + machineDeployments: + - class: proxmox-worker + name: proxmox-worker-pool + replicas: 3 + - class: proxmox-loadbalancer + name: proxmox-loadbalancer-pool + replicas: 0 + variables: + - name: allowedNodes + value: + - pve1 + - pve2 + - pve3 + - name: controlPlaneEndpoint + value: + host: 10.10.10.9 + port: 6443 + - name: dnsServers + value: [8.8.8.8, 8.8.4.4] + - name: ipv4Config + value: + addresses: [10.10.10.10-10.10.10.20] + gateway: 10.10.10.1 + prefix: 24 +# - name: ipv6Config +# value: +# addresses: [2001:db8:1::10-2001:db8:1::20] +# gateway: 2001:db8:1::1 +# prefix: 64 + - name: kubeProxy + value: + mode: iptables + - name: cloneSpec + value: + machineSpec: + controlPlane: + network: + default: + bridge: vmbr0 +# additionalDevices: +# - name: net1 +# model: virtio +# ipv4PoolRef: +# name: shared-inclusteripv4pool +# apiGroup: ipam.cluster.x-k8s.io +# kind: GlobalInClusterIPPool +# ipv6PoolRef: +# name: shared-inclusteripv6pool +# apiGroup: ipam.cluster.x-k8s.io +# kind: GlobalInClusterIPPool +# bridge: vmbr1 +# numSockets: 1 +# numCores: 4 +# memoryMiB: 4096 + sourceNode: pve1 + templateID: 100 + workerNode: + network: + default: + bridge: vmbr0 +# additionalDevices: +# - name: net1 +# model: virtio +# ipv4PoolRef: +# name: shared-inclusteripv4pool +# apiGroup: ipam.cluster.x-k8s.io +# kind: GlobalInClusterIPPool +# ipv6PoolRef: +# name: shared-inclusteripv6pool +# apiGroup: ipam.cluster.x-k8s.io +# kind: GlobalInClusterIPPool +# bridge: vmbr1 +# numSockets: 1 +# numCores: 4 +# memoryMiB: 4096 + sourceNode: pve1 + templateID: 100 + loadBalancer: + network: + default: + bridge: vmbr0 +# additionalDevices: +# - name: net1 +# model: virtio +# ipv4PoolRef: +# name: shared-inclusteripv4pool +# apiGroup: ipam.cluster.x-k8s.io +# kind: GlobalInClusterIPPool +# ipv6PoolRef: +# name: shared-inclusteripv6pool +# apiGroup: ipam.cluster.x-k8s.io +# kind: GlobalInClusterIPPool +# bridge: vmbr1 +# vrfs: +# - name: vrf-ext +# interfaces: +# - net1 +# table: 500 +# routingPolicy: +# - from: 192.0.2.0/24 +# numSockets: 1 +# numCores: 2 +# memoryMiB: 2048 + sourceNode: pve1 + templateID: 100 + sshAuthorizedKeys: + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJPK5kBd7cxXAHZ6UbeE+ysOlSjOFare3fCCZJ3xtXt1 capmox@k8s.io +# virtualIPNetworkInterface: eno1 +# --- +# apiVersion: ipam.cluster.x-k8s.io/v1alpha2 +# kind: GlobalInClusterIPPool +# metadata: +# name: shared-inclusteripv4pool +# spec: +# addresses: +# - 10.10.11.1 +# - 10.10.11.2 +# - 10.10.11.3 +# prefix: 24 +# gateway: 10.10.11.254 +# --- +# apiVersion: ipam.cluster.x-k8s.io/v1alpha2 +# kind: GlobalInClusterIPPool +# metadata: +# name: shared-inclusteripv6pool +# spec: +# addresses: +# - 2001:db8::10 +# - 2001:db8::11 +# - 2001:db8::12 +# prefix: 64 +# gateway: 2001:db8::1 diff --git a/examples/cluster.yaml b/examples/cluster.yaml new file mode 100644 index 00000000..aed92e45 --- /dev/null +++ b/examples/cluster.yaml @@ -0,0 +1,143 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cluster.x-k8s.io/proxmox-cluster-cni: + name: capmox-cluster +spec: + topology: + class: proxmox-clusterclass-v0.1.0 + version: 1.27.8 + controlPlane: + replicas: 1 + workers: + machineDeployments: + - class: proxmox-worker + name: proxmox-worker-pool + replicas: 3 + - class: proxmox-loadbalancer + name: proxmox-loadbalancer-pool + replicas: 0 + variables: + - name: allowedNodes + value: + - pve1 + - pve2 + - pve3 + - name: controlPlaneEndpoint + value: + host: 10.10.10.9 + port: 6443 + - name: dnsServers + value: [8.8.8.8, 8.8.4.4] + - name: ipv4Config + value: + addresses: [10.10.10.10-10.10.10.20] + gateway: 10.10.10.1 + prefix: 24 +# - name: ipv6Config +# value: +# addresses: [2001:db8:1::10-2001:db8:1::20] +# gateway: 2001:db8:1::1 +# prefix: 64 + - name: cloneSpec + value: + machineSpec: + controlPlane: + network: + default: + bridge: vmbr0 +# additionalDevices: +# - name: net1 +# model: virtio +# ipv4PoolRef: +# name: shared-inclusteripv4pool +# apiGroup: ipam.cluster.x-k8s.io +# kind: GlobalInClusterIPPool +# ipv6PoolRef: +# name: shared-inclusteripv6pool +# apiGroup: ipam.cluster.x-k8s.io +# kind: GlobalInClusterIPPool +# bridge: vmbr1 +# numSockets: 1 +# numCores: 4 +# memoryMiB: 4096 + sourceNode: pve1 + templateID: 100 + workerNode: + network: + default: + bridge: vmbr0 +# additionalDevices: +# - name: net1 +# model: virtio +# ipv4PoolRef: +# name: shared-inclusteripv4pool +# apiGroup: ipam.cluster.x-k8s.io +# kind: GlobalInClusterIPPool +# ipv6PoolRef: +# name: shared-inclusteripv6pool +# apiGroup: ipam.cluster.x-k8s.io +# kind: GlobalInClusterIPPool +# bridge: vmbr1 +# numSockets: 1 +# numCores: 4 +# memoryMiB: 4096 + sourceNode: pve1 + templateID: 100 + loadBalancer: + network: + default: + bridge: vmbr0 +# additionalDevices: +# - name: net1 +# model: virtio +# ipv4PoolRef: +# name: shared-inclusteripv4pool +# apiGroup: ipam.cluster.x-k8s.io +# kind: GlobalInClusterIPPool +# ipv6PoolRef: +# name: shared-inclusteripv6pool +# apiGroup: ipam.cluster.x-k8s.io +# kind: GlobalInClusterIPPool +# bridge: vmbr1 +# vrfs: +# - name: vrf-ext +# interfaces: +# - net1 +# table: 500 +# routingPolicy: +# - from: 192.0.2.0/24 +# numSockets: 1 +# numCores: 2 +# memoryMiB: 2048 + sourceNode: pve1 + templateID: 100 + sshAuthorizedKeys: + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJPK5kBd7cxXAHZ6UbeE+ysOlSjOFare3fCCZJ3xtXt1 capmox@k8s.io +# virtualIPNetworkInterface: eno1 +# --- +# apiVersion: ipam.cluster.x-k8s.io/v1alpha2 +# kind: GlobalInClusterIPPool +# metadata: +# name: shared-inclusteripv4pool +# spec: +# addresses: +# - 10.10.11.1 +# - 10.10.11.2 +# - 10.10.11.3 +# prefix: 24 +# gateway: 10.10.11.254 +# --- +# apiVersion: ipam.cluster.x-k8s.io/v1alpha2 +# kind: GlobalInClusterIPPool +# metadata: +# name: shared-inclusteripv6pool +# spec: +# addresses: +# - 2001:db8::10 +# - 2001:db8::11 +# - 2001:db8::12 +# prefix: 64 +# gateway: 2001:db8::1 diff --git a/templates/cluster-class-calico.yaml b/templates/cluster-class-calico.yaml new file mode 100644 index 00000000..157710cf --- /dev/null +++ b/templates/cluster-class-calico.yaml @@ -0,0 +1,1033 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: proxmox-clusterclass-calico-v0.1.0 +spec: + controlPlane: + namingStrategy: + template: "{{ .cluster.name }}-control-plane-{{ .random }}" + ref: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + name: proxmox-clusterclass-v0.1.0-control-plane + machineInfrastructure: + ref: + kind: ProxmoxMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + name: proxmox-clusterclass-v0.1.0-control-plane-template + machineHealthCheck: + maxUnhealthy: 100% + nodeStartupTimeout: 15m + unhealthyConditions: + - type: Ready + status: Unknown + timeout: 300s + - type: Ready + status: "False" + timeout: 300s + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: ProxmoxClusterTemplate + name: proxmox-clusterclass-calico-v0.1.0-clustertemplate + workers: + machineDeployments: + - class: proxmox-worker + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: proxmox-clusterclass-v0.1.0-workertemplate + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: ProxmoxMachineTemplate + name: proxmox-clusterclass-v0.1.0-workertemplate + metadata: + labels: + node-role.kubernetes.io/node: "" + machineHealthCheck: + maxUnhealthy: 33% + nodeStartupTimeout: 15m + unhealthyConditions: + - type: Ready + status: Unknown + timeout: 300s + - type: Ready + status: "False" + timeout: 300s + namingStrategy: + template: "{{ .cluster.name }}-worker-{{ .random }}" + - class: proxmox-loadbalancer + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: proxmox-clusterclass-v0.1.0-loadbalancer-template + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: ProxmoxMachineTemplate + name: proxmox-clusterclass-v0.1.0-loadbalancer-template + metadata: + labels: + node-role.kubernetes.io/load-balancer: "" + node-role.kubernetes.io/node: "" + machineHealthCheck: + maxUnhealthy: 33% + nodeStartupTimeout: 15m + unhealthyConditions: + - type: Ready + status: Unknown + timeout: 300s + - type: Ready + status: "False" + timeout: 300s + namingStrategy: + template: "{{ .cluster.name }}-loadbalancer-{{ .random }}" + variables: + - name: controlPlaneEndpoint + required: true + schema: + openAPIV3Schema: + type: object + properties: + host: + example: 10.10.10.9 + type: string + port: + type: integer + default: 6443 + - name: ipv4Config + required: false + schema: + openAPIV3Schema: + type: object + properties: + addresses: + minItems: 1 + type: array + items: + type: string + default: + - "10.10.10.10-10.10.10.15" + prefix: + type: integer + default: 24 + gateway: + type: string + default: "10.10.10.1" + - name: ipv6Config + required: false + schema: + openAPIV3Schema: + type: object + properties: + addresses: + minItems: 1 + type: array + items: + type: string + default: + - "2001:db8::0002-2001:db8::ffff" + prefix: + type: integer + default: 64 + gateway: + type: string + default: "2001:db8::0001" + - name: dnsServers + required: true + schema: + openAPIV3Schema: + type: array + minItems: 1 + items: + type: string + default: [8.8.8.8, 8.8.4.4] + example: [8.8.8.8, 8.8.4.4] + - name: allowedNodes + required: false + schema: + openAPIV3Schema: + type: array + items: + type: string + example: ["pve1", "pve2", "pve3"] + - name: kubeProxy + required: false + schema: + openAPIV3Schema: + type: object + properties: + mode: + type: string + enum: ["ipvs","iptables"] + - name: cloneSpec + required: true + schema: + openAPIV3Schema: + type: object + properties: + sshAuthorizedKeys: + type: array + items: + type: string + default: [] + example: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJPK5kBd7cxXAHZ6UbeE+ysOlSjOFare3fCCZJ3xtXt1 example@capmox","ssh-rsa ..."] + virtualIPNetworkInterface: + type: string + default: "" + example: "vmbr0" + machineSpec: + type: object + properties: + controlPlane: &machineSpec + type: object + required: + - sourceNode + properties: + disks: + type: object + properties: + bootVolume: + type: object + properties: + disk: + type: string + sizeGb: + type: integer + minimum: 5 + format: int32 + required: + - disk + - sizeGb + format: + type: string + enum: [raw, qcow2, vmdk] + network: + type: object + properties: + additionalDevices: + type: array + items: + type: object + properties: + bridge: + type: string + dnsServers: + type: array + items: + type: string + ipv4PoolRef: + type: object + properties: + apiGroup: + default: ipam.cluster.x-k8s.io + type: string + kind: + default: GlobalInClusterIPPool + type: string + name: + default: shared-inclusterippool + type: string + required: + - kind + - name + ipv6PoolRef: + type: object + properties: + apiGroup: + default: ipam.cluster.x-k8s.io + type: string + kind: + default: GlobalInClusterIPPool + type: string + name: + default: shared-inclusteripv4pool + type: string + required: + - kind + - name + model: + type: string + default: virtio + enum: [e1000, virtio, rtl8139, vmxnet3] + name: + minLength: 1 + type: string + required: + - bridge + - name + default: + type: object + properties: + bridge: + type: string + model: + type: string + default: virtio + enum: [e1000, virtio, rtl8139, vmxnet3] + required: + - bridge + vrfs: + type: array + items: + type: object + properties: + dnsServers: + type: array + items: + type: string + interfaces: + type: array + items: + type: string + description: parent interfaces of a vrf device + ipv4PoolRef: + type: object + properties: + apiGroup: + default: ipam.cluster.x-k8s.io + type: string + kind: + default: GlobalInClusterIPPool + type: string + name: + default: shared-inclusterippool + type: string + required: + - kind + - name + ipv6PoolRef: + type: object + properties: + apiGroup: + default: ipam.cluster.x-k8s.io + type: string + kind: + default: GlobalInClusterIPPool + type: string + name: + default: shared-inclusteripv4pool + type: string + required: + - kind + - name + name: + minLength: 1 + type: string + routes: + type: array + minItems: 1 + items: + type: object + properties: + metric: + format: int32 + type: integer + table: + format: int32 + type: integer + to: + type: string + via: + type: string + routingPolicy: + type: array + minItems: 1 + items: + type: object + properties: + from: + type: string + priority: + type: integer + format: int32 + table: + format: int32 + type: integer + to: + type: string + table: + format: int32 + type: integer + required: + - name + - table + required: + - default + memoryMiB: + type: integer + example: 2048 + numCores: + type: integer + example: 1 + numSockets: + type: integer + example: 1 + sourceNode: + type: string + example: pve1 + templateID: + type: integer + example: 100 + workerNode: *machineSpec + loadBalancer: *machineSpec + patches: + - name: ProxmoxClusterTemplateGeneral + description: "Configure Cluster" + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: ProxmoxClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: /spec/template/spec/allowedNodes + valueFrom: + variable: allowedNodes + - op: add + path: /spec/template/spec/controlPlaneEndpoint/host + valueFrom: + variable: controlPlaneEndpoint.host + - op: add + path: /spec/template/spec/controlPlaneEndpoint/port + valueFrom: + variable: controlPlaneEndpoint.port + - op: replace + path: /spec/template/spec/dnsServers + valueFrom: + variable: dnsServers + - op: replace + path: /spec/template/spec/cloneSpec/sshAuthorizedKeys + valueFrom: + variable: cloneSpec.sshAuthorizedKeys + - op: replace + path: /spec/template/spec/cloneSpec/virtualIPNetworkInterface + valueFrom: + variable: cloneSpec.virtualIPNetworkInterface + - name: ClusterIPv4Config + description: "Configure Cluster IPv4 config" + enabledIf: "{{ if .ipv4Config }}true{{ end }}" + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: ProxmoxClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: /spec/template/spec/ipv4Config + valueFrom: + variable: ipv4Config + - name: ClusterIPv6Config + description: "Configure Cluster IPv6 config" + enabledIf: "{{ if .ipv6Config }}true{{ end }}" + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: ProxmoxClusterTemplate + matchResources: + infrastructureCluster: true + jsonPatches: + - op: add + path: /spec/template/spec/ipv6Config + valueFrom: + variable: ipv6Config + - name: ControlPlaneSetup + description: "How to bind the Control Plane and what K8S version" + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: /spec/template/spec/kubeadmConfigSpec/users + valueFrom: + template: | + - name: root + sshAuthorizedKeys: {{ .cloneSpec.sshAuthorizedKeys }} + - op: add + path: /spec/template/spec/kubeadmConfigSpec/files/- + valueFrom: + template: | + owner: root:root + path: /etc/kubernetes/manifests/kube-vip.yaml + content: | + apiVersion: v1 + kind: Pod + metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system + spec: + containers: + - args: + - manager + env: + - name: cp_enable + value: "true" + - name: vip_interface + value: "{{ .cloneSpec.virtualIPNetworkInterface }}" + - name: address + value: "{{ .controlPlaneEndpoint.host }}" + - name: port + value: "6443" + - name: vip_arp + value: "true" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "15" + - name: vip_renewdeadline + value: "10" + - name: vip_retryperiod + value: "2" + image: ghcr.io/kube-vip/kube-vip:v0.5.11 + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + type: FileOrCreate + name: kubeconfig + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: ProxmoxMachineTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: replace + path: /spec/template/spec/sourceNode + valueFrom: + variable: cloneSpec.machineSpec.controlPlane.sourceNode + - op: replace + path: /spec/template/spec/templateID + valueFrom: + variable: cloneSpec.machineSpec.controlPlane.templateID + - name: kube-proxy-setup + description: "kube-proxy configuration" + enabledIf: "{{ if eq .kubeProxy.mode \"ipvs\" }}true{{ end }}" + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: /spec/template/spec/kubeadmConfigSpec/files/- + valueFrom: + template: | + content: | + #/bin/sh + cat >> /run/kubeadm/kubeadm.yaml <> /run/kubeadm/kubeadm.yaml <> /run/kubeadm/kubeadm.yaml <