🔥 Add external credentials (#215)

api/v1alpha1/proxmoxcluster_types.go

@@ -21,6 +21,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/ptr"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/errors"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -71,6 +72,11 @@ type ProxmoxClusterSpec struct {
 	// in the configuration and cloning of a proxmox VM. Multiple types of nodes can be specified.
 	// +optional
 	CloneSpec *ProxmoxClusterCloneSpec `json:"cloneSpec,omitempty"`
+
+	// CredentialsRef is a reference to a Secret that contains the credentials to use for provisioning this cluster. If not
+	// supplied then the credentials of the controller will be used.
+	// +optional
+	CredentialsRef *corev1.SecretReference `json:"credentialsRef,omitempty"`
 }
 
 // ProxmoxClusterCloneSpec is the configuration pertaining to all items configurable
@@ -141,6 +147,44 @@ type ProxmoxClusterStatus struct {
 	// +optional
 	NodeLocations *NodeLocations `json:"nodeLocations,omitempty"`
 
+	// FailureReason will be set in the event that there is a terminal problem
+	// reconciling the Machine and will contain a succinct value suitable
+	// for machine interpretation.
+	//
+	// This field should not be set for transitive errors that a controller
+	// faces that are expected to be fixed automatically over
+	// time (like service outages), but instead indicate that something is
+	// fundamentally wrong with the Machine's spec or the configuration of
+	// the controller, and that manual intervention is required. Examples
+	// of terminal errors would be invalid combinations of settings in the
+	// spec, values that are unsupported by the controller, or the
+	// responsible controller itself being critically misconfigured.
+	//
+	// Any transient errors that occur during the reconciliation of ProxmoxCluster
+	// can be added as events to the ProxmoxCluster object and/or logged in the
+	// controller's output.
+	// +optional
+	FailureReason *errors.ClusterStatusError `json:"failureReason,omitempty"`
+
+	// FailureMessage will be set in the event that there is a terminal problem
+	// reconciling the Machine and will contain a more verbose string suitable
+	// for logging and human consumption.
+	//
+	// This field should not be set for transitive errors that a controller
+	// faces that are expected to be fixed automatically over
+	// time (like service outages), but instead indicate that something is
+	// fundamentally wrong with the Machine's spec or the configuration of
+	// the controller, and that manual intervention is required. Examples
+	// of terminal errors would be invalid combinations of settings in the
+	// spec, values that are unsupported by the controller, or the
+	// responsible controller itself being critically misconfigured.
+	//
+	// Any transient errors that occur during the reconciliation of ProxmoxMachines
+	// can be added as events to the ProxmoxCluster object and/or logged in the
+	// controller's output.
+	// +optional
+	FailureMessage *string `json:"failureMessage,omitempty"`
+
 	// Conditions defines current service state of the ProxmoxCluster.
 	// +optional
 	Conditions clusterv1.Conditions `json:"conditions,omitempty"`

cmd/main.go

@@ -27,7 +27,6 @@ import (
 
 	"github.com/go-logr/logr"
 	"github.com/luthermonson/go-proxmox"
-	"github.com/pkg/errors"
 	"github.com/spf13/pflag"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
@@ -185,8 +184,13 @@ func setupReconcilers(ctx context.Context, mgr ctrl.Manager, client capmox.Clien
 }
 
 func setupProxmoxClient(ctx context.Context, logger logr.Logger) (capmox.Client, error) {
+	// we return nil if the env variables are not set
+	// so the proxmoxcontroller can create the client later from spec.credentialsRef
+	// or fail the cluster if no credentials found
+	if ProxmoxURL == "" || ProxmoxTokenID == "" || ProxmoxSecret == "" {
+		return nil, nil
+	}
 	// TODO, check if we need to delete tls config
-	// You can disable security check for a client:
 	tr := &http.Transport{
 		TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec
 	}
@@ -214,23 +218,4 @@ func initFlagsAndEnv(fs *pflag.FlagSet) {
 		"If true, run webhook server alongside manager")
 
 	feature.MutableGates.AddFlag(fs)
-
-	err := validate()
-	if err != nil {
-		setupLog.Error(err, "validate fails")
-		os.Exit(1)
-	}
-}
-
-func validate() error {
-	if ProxmoxURL == "" {
-		return errors.New("required variable `PROXMOX_URL` is not set")
-	}
-	if ProxmoxTokenID == "" {
-		return errors.New("required variable `PROXMOX_TOKEN` is not set")
-	}
-	if ProxmoxSecret == "" {
-		return errors.New("required variable `PROXMOX_SECRET` is not set")
-	}
-	return nil
 }

config/crd/bases/infrastructure.cluster.x-k8s.io_proxmoxclusters.yaml

@@ -561,6 +561,21 @@ spec:
                 - host
                 - port
                 type: object
+              credentialsRef:
+                description: CredentialsRef is a reference to a Secret that contains
+                  the credentials to use for provisioning this cluster. If not supplied
+                  then the credentials of the controller will be used.
+                properties:
+                  name:
+                    description: name is unique within a namespace to reference a
+                      secret resource.
+                    type: string
+                  namespace:
+                    description: namespace defines the space within which the secret
+                      name must be unique.
+                    type: string
+                type: object
+                x-kubernetes-map-type: atomic
               dnsServers:
                 description: DNSServers contains information about nameservers used
                   by the machines.
@@ -690,6 +705,37 @@ spec:
                   - type
                   type: object
                 type: array
+              failureMessage:
+                description: "FailureMessage will be set in the event that there is
+                  a terminal problem reconciling the Machine and will contain a more
+                  verbose string suitable for logging and human consumption. \n This
+                  field should not be set for transitive errors that a controller
+                  faces that are expected to be fixed automatically over time (like
+                  service outages), but instead indicate that something is fundamentally
+                  wrong with the Machine's spec or the configuration of the controller,
+                  and that manual intervention is required. Examples of terminal errors
+                  would be invalid combinations of settings in the spec, values that
+                  are unsupported by the controller, or the responsible controller
+                  itself being critically misconfigured. \n Any transient errors that
+                  occur during the reconciliation of ProxmoxMachines can be added
+                  as events to the ProxmoxCluster object and/or logged in the controller's
+                  output."
+                type: string
+              failureReason:
+                description: "FailureReason will be set in the event that there is
+                  a terminal problem reconciling the Machine and will contain a succinct
+                  value suitable for machine interpretation. \n This field should
+                  not be set for transitive errors that a controller faces that are
+                  expected to be fixed automatically over time (like service outages),
+                  but instead indicate that something is fundamentally wrong with
+                  the Machine's spec or the configuration of the controller, and that
+                  manual intervention is required. Examples of terminal errors would
+                  be invalid combinations of settings in the spec, values that are
+                  unsupported by the controller, or the responsible controller itself
+                  being critically misconfigured. \n Any transient errors that occur
+                  during the reconciliation of ProxmoxCluster can be added as events
+                  to the ProxmoxCluster object and/or logged in the controller's output."
+                type: string
               inClusterIpPoolRef:
                 description: InClusterIPPoolRef is the reference to the created in-cluster
                   IP pool.

config/crd/bases/infrastructure.cluster.x-k8s.io_proxmoxclustertemplates.yaml

@@ -610,6 +610,22 @@ spec:
                         - host
                         - port
                         type: object
+                      credentialsRef:
+                        description: CredentialsRef is a reference to a Secret that
+                          contains the credentials to use for provisioning this cluster.
+                          If not supplied then the credentials of the controller will
+                          be used.
+                        properties:
+                          name:
+                            description: name is unique within a namespace to reference
+                              a secret resource.
+                            type: string
+                          namespace:
+                            description: namespace defines the space within which
+                              the secret name must be unique.
+                            type: string
+                        type: object
+                        x-kubernetes-map-type: atomic
                       dnsServers:
                         description: DNSServers contains information about nameservers
                           used by the machines.

config/default/proxmox-credentials-secret.yaml

@@ -1,9 +1,11 @@
 ---
 apiVersion: v1
 stringData:
-  secret: ${PROXMOX_SECRET}
-  token: ${PROXMOX_TOKEN}
-  url: ${PROXMOX_URL}
+  secret: ${PROXMOX_SECRET=""}
+  token: ${PROXMOX_TOKEN=""}
+  url: ${PROXMOX_URL=""}
 kind: Secret
 metadata:
   name: manager-credentials
+  labels:
+    platform.ionos.com/secret-type: "proxmox-credentials"

docs/Usage.md

@@ -120,8 +120,11 @@ Once you have access to a management cluster, you can initialize Cluster API wit
 clusterctl init --infrastructure proxmox --ipam in-cluster --core cluster-api:v1.6.1
 ```
 
+**Note:** The Proxmox credentials are optional when installing the provider,
+but they are required when creating a cluster.
+
 ### Create a Workload Cluster
-In order to create a new cluster, you need to generate a cluster manifest.
+To create a new cluster, you need to generate a cluster manifest.
 
 ```bash
 $ clusterctl generate cluster proxmox-quickstart \
@@ -167,15 +170,57 @@ For templates using `CNI`s you're required to create `ConfigMaps` to make `Clust
 We provide the following templates:
 
 | Flavor              | Tepmlate File                                        | CRS File                                                  |
-|---------------------| -----------------------------------------------------|-----------------------------------------------------------|
+|---------------------|------------------------------------------------------|-----------------------------------------------------------|
 | cilium              | templates/cluster-template-cilium.yaml               | templates/crs/cni/cilium.yaml                             |
 | calico              | templates/cluster-template-calico.yaml               | templates/crs/cni/calico.yaml                             |
 | multiple-vlans      | templates/cluster-template-multiple-vlans.yaml       | -                                                         |
 | default             | templates/cluster-template.yaml                      | -                                                         |
 | cilium loadbalancer | templates/cluster-template-cilium-load-balancer.yaml | templates/crs/cni/cilium.yaml, templates/crs/metallb.yaml |
+| external-creds      | templates/cluster-template-external-creds.yaml       |                                                           |
 
 For more information about advanced clusters please check our [advanced setups docs](advanced-setups.md).
 
+#### External Credentials
+
+The `external-creds` flavor is used to create a cluster with external credentials.
+This is useful when you want to use different Proxmox Datacenters.
+
+you will need these environment variables to generate a cluster with external credentials:
+
+```env
+PROXMOX_URL: "https://pve.example:8006"                       # The Proxmox VE host
+PROXMOX_TOKEN: "root@pam!capi"                                # The Proxmox VE TokenID for authentication
+PROXMOX_SECRET: "REDACTED"                                    # The secret associated with the TokenID
+```
+
+However, to use external-credentials in your own Cluster manifests, you need to create a secret
+and reference it in the cluster manifest.
+```yaml
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: ProxmoxCluster
+metadata:
+  name: "my-cluster"
+spec:
+  controlPlaneEndpoint:
+    host: ${CONTROL_PLANE_ENDPOINT_IP}
+    port: 6443
+  # ...  
+  credentialsRef:
+    name: "my-cluster-proxmox-credentials"
+---
+apiVersion: v1
+stringData:
+  secret: ${PROXMOX_SECRET}
+  token: ${PROXMOX_TOKEN}
+  url: ${PROXMOX_URL}
+kind: Secret
+metadata:
+  name: my-cluster-proxmox-credentials
+  labels:
+    # Custom IONOS Label
+    platform.ionos.com/secret-type: "proxmox-credentials"
+```
+
 #### Flavor with Cilium CNI
 Before this cluster can be deployed, `cilium` needs to be configured. As a first step we
 need to generate a manifest. Simply use our makefile: