diff --git a/docs/src/snap/reference/bootstrap-config-reference.md b/docs/src/snap/reference/bootstrap-config-reference.md
index 7109d5aa64..047622c5d9 100644
--- a/docs/src/snap/reference/bootstrap-config-reference.md
+++ b/docs/src/snap/reference/bootstrap-config-reference.md
@@ -37,6 +37,7 @@ Determines if the feature should be enabled.
If omitted defaults to `true`
#### cluster-config.dns.cluster-domain
+
**Type:** `string`
**Required:** `No`
@@ -59,7 +60,8 @@ Can be used to point to an external dns server when feature is disabled.
**Type:** `list[string]`
**Required:** `No`
-Sets the upstream nameservers used to forward queries for out-of-cluster endpoints.
+Sets the upstream nameservers used to forward queries for out-of-cluster
+endpoints.
If omitted defaults to `/etc/resolv.conf` and uses the nameservers of the node.
@@ -83,7 +85,8 @@ If omitted defaults to `false`
**Type:** `string`
**Required:** `No`
-Sets the name of the secret to be used for providing default encryption to ingresses.
+Sets the name of the secret to be used for providing default encryption to
+ingresses.
Ingresses can specify another TLS secret in their resource definitions,
in which case the default secret won't be used.
@@ -117,7 +120,8 @@ If omitted defaults to `false`
**Type:** `list[string]`
**Required:** `No`
-Sets the CIDRs used for assigning IP addresses to Kubernetes services with type `LoadBalancer`.
+Sets the CIDRs used for assigning IP addresses to Kubernetes services with type
+`LoadBalancer`.
#### cluster-config.load-balancer.l2-mode
@@ -316,7 +320,8 @@ The CA certificate to be used when communicating with the external datastore.
**Type:** `string`
**Required:** `No`
-The client certificate to be used when communicating with the external datastore.
+The client certificate to be used when communicating with the external
+datastore.
### datastore-client-key
@@ -386,7 +391,8 @@ If omitted defaults to an auto generated key.
**Type:** `string`
**Required:** `No`
-The client certificate to be used by kubelet for communicating with the kube-apiserver.
+The client certificate to be used by kubelet for communicating with the
+kube-apiserver.
If omitted defaults to an auto generated certificate.
### apiserver-kubelet-client-key
@@ -437,6 +443,79 @@ If omitted defaults to an auto generated certificate.
The key to be used for the kubelet.
If omitted defaults to an auto generated key.
+### extra-node-config-files
+
+**Type:** `map[string]string`
+**Required:** `No`
+
+Additional files that are uploaded `/var/snap/k8s/common/args/conf.d/`
+to a node on bootstrap. These files can them be references by Kubernetes
+service arguments.
+The format is `map[]`.
+
+### extra-node-kube-apiserver-args
+
+**Type:** `map[string]string`
+**Required:** `No`
+
+Additional arguments that are passed to the `kube-apiserver` only for that
+specific node. Overwrites default configuration. A parameter that is explicitly
+set to `null` is deleted. The format is `map[<--flag-name>]`.
+
+### extra-node-kube-controller-manager-args
+
+**Type:** `map[string]string`
+**Required:** `No`
+
+Additional arguments that are passed to the `kube-controller-manager` only for
+that specific node. Overwrites default configuration. A parameter that is
+explicitly set to `null` is deleted. The format is `map[<--flag-name>]`.
+
+### extra-node-kube-scheduler-args
+
+**Type:** `map[string]string`
+**Required:** `No`
+
+Additional arguments that are passed to the `kube-scheduler` only for that
+specific node. Overwrites default configuration. A parameter that is explicitly
+set to `null` is deleted. The format is `map[<--flag-name>]`.
+
+### extra-node-kube-proxy-args
+
+**Type:** `map[string]string`
+**Required:** `No`
+
+Additional arguments that are passed to the `kube-proxy` only for that
+specific node. Overwrites default configuration. A parameter that is explicitly
+set to `null` is deleted. The format is `map[<--flag-name>]`.
+
+### extra-node-kubelet-args
+
+**Type:** `map[string]string`
+**Required:** `No`
+
+Additional arguments that are passed to the `kubelet` only for that
+specific node. Overwrites default configuration. A parameter that is explicitly
+set to `null` is deleted. The format is `map[<--flag-name>]`.
+
+### extra-node-containerd-args
+
+**Type:** `map[string]string`
+**Required:** `No`
+
+Additional arguments that are passed to `containerd` only for that
+specific node. Overwrites default configuration. A parameter that is explicitly
+set to `null` is deleted. The format is `map[<--flag-name>]`.
+
+### extra-node-k8s-dqlite-args
+
+**Type:** `map[string]string`
+**Required:** `No`
+
+Additional arguments that are passed to `k8s-dqlite` only for that
+specific node. Overwrites default configuration. A parameter that is explicitly
+set to `null` is deleted. The format is `map[<--flag-name>]`.
+
## Example
The following example configures and enables certain features, sets an external
@@ -478,4 +557,20 @@ k8s-dqlite-port: 9090
datastore-type: k8s-dqlite
extra-sans:
- custom.kubernetes
+extra-node-config-files:
+ bootstrap-extra-file.yaml: extra-args-test-file-content
+extra-node-kube-apiserver-args:
+ --request-timeout: 2m
+extra-node-kube-controller-manager-args:
+ --leader-elect-retry-period: 3s
+extra-node-kube-scheduler-args:
+ --authorization-webhook-cache-authorized-ttl: 11s
+extra-node-kube-proxy-args:
+ --config-sync-period: 14m
+extra-node-kubelet-args:
+ --authentication-token-webhook-cache-ttl: 3m
+extra-node-containerd-args:
+ --log-level: debug
+extra-node-k8s-dqlite-args:
+ --watch-storage-available-size-interval: 6s
```
diff --git a/src/k8s/api/v1/bootstrap_config.go b/src/k8s/api/v1/bootstrap_config.go
index c1c661957f..8efa4797c6 100644
--- a/src/k8s/api/v1/bootstrap_config.go
+++ b/src/k8s/api/v1/bootstrap_config.go
@@ -54,6 +54,18 @@ type BootstrapConfig struct {
KubeletKey *string `json:"kubelet-key,omitempty" yaml:"kubelet-key,omitempty"`
KubeletClientCert *string `json:"kubelet-client-crt,omitempty" yaml:"kubelet-client-crt,omitempty"`
KubeletClientKey *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"`
+
+ // ExtraNodeConfigFiles will be written to /var/snap/k8s/common/args/conf.d
+ ExtraNodeConfigFiles map[string]string `json:"extra-node-config-files,omitempty" yaml:"extra-node-config-files,omitempty"`
+
+ // Extra args to add to individual services (set any arg to null to delete)
+ ExtraNodeKubeAPIServerArgs map[string]*string `json:"extra-node-kube-apiserver-args,omitempty" yaml:"extra-node-kube-apiserver-args,omitempty"`
+ ExtraNodeKubeControllerManagerArgs map[string]*string `json:"extra-node-kube-controller-manager-args,omitempty" yaml:"extra-node-kube-controller-manager-args,omitempty"`
+ ExtraNodeKubeSchedulerArgs map[string]*string `json:"extra-node-kube-scheduler-args,omitempty" yaml:"extra-node-kube-scheduler-args,omitempty"`
+ ExtraNodeKubeProxyArgs map[string]*string `json:"extra-node-kube-proxy-args,omitempty" yaml:"extra-node-kube-proxy-args,omitempty"`
+ ExtraNodeKubeletArgs map[string]*string `json:"extra-node-kubelet-args,omitempty" yaml:"extra-node-kubelet-args,omitempty"`
+ ExtraNodeContainerdArgs map[string]*string `json:"extra-node-containerd-args,omitempty" yaml:"extra-node-containerd-args,omitempty"`
+ ExtraNodeK8sDqliteArgs map[string]*string `json:"extra-node-k8s-dqlite-args,omitempty" yaml:"extra-node-k8s-dqlite-args,omitempty"`
}
func (b *BootstrapConfig) GetDatastoreType() string { return getField(b.DatastoreType) }
diff --git a/src/k8s/api/v1/bootstrap_config_test.go b/src/k8s/api/v1/bootstrap_config_test.go
index 097f12574a..400be832f1 100644
--- a/src/k8s/api/v1/bootstrap_config_test.go
+++ b/src/k8s/api/v1/bootstrap_config_test.go
@@ -41,13 +41,21 @@ func TestBootstrapConfigToMicrocluster(t *testing.T) {
},
CloudProvider: utils.Pointer("external"),
},
- PodCIDR: utils.Pointer("10.100.0.0/16"),
- ServiceCIDR: utils.Pointer("10.200.0.0/16"),
- DisableRBAC: utils.Pointer(false),
- SecurePort: utils.Pointer(6443),
- K8sDqlitePort: utils.Pointer(9090),
- DatastoreType: utils.Pointer("k8s-dqlite"),
- ExtraSANs: []string{"custom.kubernetes"},
+ PodCIDR: utils.Pointer("10.100.0.0/16"),
+ ServiceCIDR: utils.Pointer("10.200.0.0/16"),
+ DisableRBAC: utils.Pointer(false),
+ SecurePort: utils.Pointer(6443),
+ K8sDqlitePort: utils.Pointer(9090),
+ DatastoreType: utils.Pointer("k8s-dqlite"),
+ ExtraSANs: []string{"custom.kubernetes"},
+ ExtraNodeConfigFiles: map[string]string{"extra-node-config-file": "file-content"},
+ ExtraNodeKubeAPIServerArgs: map[string]*string{"--extra-kube-apiserver-arg": utils.Pointer("extra-kube-apiserver-value")},
+ ExtraNodeKubeControllerManagerArgs: map[string]*string{"--extra-kube-controller-manager-arg": utils.Pointer("extra-kube-controller-manager-value")},
+ ExtraNodeKubeSchedulerArgs: map[string]*string{"--extra-kube-scheduler-arg": utils.Pointer("extra-kube-scheduler-value")},
+ ExtraNodeKubeProxyArgs: map[string]*string{"--extra-kube-proxy-arg": utils.Pointer("extra-kube-proxy-value")},
+ ExtraNodeKubeletArgs: map[string]*string{"--extra-kubelet-arg": utils.Pointer("extra-kubelet-value")},
+ ExtraNodeContainerdArgs: map[string]*string{"--extra-containerd-arg": utils.Pointer("extra-containerd-value")},
+ ExtraNodeK8sDqliteArgs: map[string]*string{"--extra-k8s-dqlite-arg": utils.Pointer("extra-k8s-dqlite-value")},
}
microclusterConfig, err := cfg.ToMicrocluster()
diff --git a/src/k8s/api/v1/join_config.go b/src/k8s/api/v1/join_config.go
index 8324dfdf96..7a69aaa529 100644
--- a/src/k8s/api/v1/join_config.go
+++ b/src/k8s/api/v1/join_config.go
@@ -25,6 +25,18 @@ type ControlPlaneNodeJoinConfig struct {
KubeletKey *string `json:"kubelet-key,omitempty" yaml:"kubelet-key,omitempty"`
KubeletClientCert *string `json:"kubelet-client-crt,omitempty" yaml:"kubelet-client-crt,omitempty"`
KubeletClientKey *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"`
+
+ // ExtraNodeConfigFiles will be written to /var/snap/k8s/common/args/conf.d
+ ExtraNodeConfigFiles map[string]string `json:"extra-node-config-files,omitempty" yaml:"extra-node-config-files,omitempty"`
+
+ // Extra args to add to individual services (set any arg to null to delete)
+ ExtraNodeKubeAPIServerArgs map[string]*string `json:"extra-node-kube-apiserver-args,omitempty" yaml:"extra-node-kube-apiserver-args,omitempty"`
+ ExtraNodeKubeControllerManagerArgs map[string]*string `json:"extra-node-kube-controller-manager-args,omitempty" yaml:"extra-node-kube-controller-manager-args,omitempty"`
+ ExtraNodeKubeSchedulerArgs map[string]*string `json:"extra-node-kube-scheduler-args,omitempty" yaml:"extra-node-kube-scheduler-args,omitempty"`
+ ExtraNodeKubeProxyArgs map[string]*string `json:"extra-node-kube-proxy-args,omitempty" yaml:"extra-node-kube-proxy-args,omitempty"`
+ ExtraNodeKubeletArgs map[string]*string `json:"extra-node-kubelet-args,omitempty" yaml:"extra-node-kubelet-args,omitempty"`
+ ExtraNodeContainerdArgs map[string]*string `json:"extra-node-containerd-args,omitempty" yaml:"extra-node-containerd-args,omitempty"`
+ ExtraNodeK8sDqliteArgs map[string]*string `json:"extra-node-k8s-dqlite-args,omitempty" yaml:"extra-node-k8s-dqlite-args,omitempty"`
}
type WorkerNodeJoinConfig struct {
@@ -34,6 +46,15 @@ type WorkerNodeJoinConfig struct {
KubeletClientKey *string `json:"kubelet-client-key,omitempty" yaml:"kubelet-client-key,omitempty"`
KubeProxyClientCert *string `json:"kube-proxy-client-crt,omitempty" yaml:"kube-proxy-client-crt,omitempty"`
KubeProxyClientKey *string `json:"kube-proxy-client-key,omitempty" yaml:"kube-proxy-client-key,omitempty"`
+
+ // ExtraNodeConfigFiles will be written to /var/snap/k8s/common/args/conf.d
+ ExtraNodeConfigFiles map[string]string `json:"extra-node-config-files,omitempty" yaml:"extra-node-config-files,omitempty"`
+
+ // Extra args to add to individual services (set any arg to null to delete)
+ ExtraNodeKubeProxyArgs map[string]*string `json:"extra-node-kube-proxy-args,omitempty" yaml:"extra-node-kube-proxy-args,omitempty"`
+ ExtraNodeKubeletArgs map[string]*string `json:"extra-node-kubelet-args,omitempty" yaml:"extra-node-kubelet-args,omitempty"`
+ ExtraNodeContainerdArgs map[string]*string `json:"extra-node-containerd-args,omitempty" yaml:"extra-node-containerd-args,omitempty"`
+ ExtraNodeK8sAPIServerProxyArgs map[string]*string `json:"extra-node-k8s-apiserver-proxy-args,omitempty" yaml:"extra-node-k8s-apiserver-proxy-args,omitempty"`
}
func (c *ControlPlaneNodeJoinConfig) GetFrontProxyClientCert() string {
diff --git a/src/k8s/cmd/k8s/k8s_bootstrap_test.go b/src/k8s/cmd/k8s/k8s_bootstrap_test.go
index 24783ccbb7..a10fc922d2 100644
--- a/src/k8s/cmd/k8s/k8s_bootstrap_test.go
+++ b/src/k8s/cmd/k8s/k8s_bootstrap_test.go
@@ -64,14 +64,22 @@ var testCases = []testCase{
},
CloudProvider: utils.Pointer("external"),
},
- ControlPlaneTaints: []string{"node-role.kubernetes.io/control-plane:NoSchedule"},
- PodCIDR: utils.Pointer("10.100.0.0/16"),
- ServiceCIDR: utils.Pointer("10.200.0.0/16"),
- DisableRBAC: utils.Pointer(false),
- SecurePort: utils.Pointer(6443),
- K8sDqlitePort: utils.Pointer(9090),
- DatastoreType: utils.Pointer("k8s-dqlite"),
- ExtraSANs: []string{"custom.kubernetes"},
+ ControlPlaneTaints: []string{"node-role.kubernetes.io/control-plane:NoSchedule"},
+ PodCIDR: utils.Pointer("10.100.0.0/16"),
+ ServiceCIDR: utils.Pointer("10.200.0.0/16"),
+ DisableRBAC: utils.Pointer(false),
+ SecurePort: utils.Pointer(6443),
+ K8sDqlitePort: utils.Pointer(9090),
+ DatastoreType: utils.Pointer("k8s-dqlite"),
+ ExtraSANs: []string{"custom.kubernetes"},
+ ExtraNodeConfigFiles: map[string]string{"extra-node-config-file.yaml": "test-file-content"},
+ ExtraNodeKubeAPIServerArgs: map[string]*string{"--extra-kube-apiserver-arg": utils.Pointer("extra-kube-apiserver-value")},
+ ExtraNodeKubeControllerManagerArgs: map[string]*string{"--extra-kube-controller-manager-arg": utils.Pointer("extra-kube-controller-manager-value")},
+ ExtraNodeKubeSchedulerArgs: map[string]*string{"--extra-kube-scheduler-arg": utils.Pointer("extra-kube-scheduler-value")},
+ ExtraNodeKubeProxyArgs: map[string]*string{"--extra-kube-proxy-arg": utils.Pointer("extra-kube-proxy-value")},
+ ExtraNodeKubeletArgs: map[string]*string{"--extra-kubelet-arg": utils.Pointer("extra-kubelet-value")},
+ ExtraNodeContainerdArgs: map[string]*string{"--extra-containerd-arg": utils.Pointer("extra-containerd-value")},
+ ExtraNodeK8sDqliteArgs: map[string]*string{"--extra-k8s-dqlite-arg": utils.Pointer("extra-k8s-dqlite-value")},
},
},
{
diff --git a/src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml b/src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml
index 71d7ed973c..8431de74d8 100644
--- a/src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml
+++ b/src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml
@@ -31,3 +31,19 @@ k8s-dqlite-port: 9090
datastore-type: k8s-dqlite
extra-sans:
- custom.kubernetes
+extra-node-config-files:
+ extra-node-config-file.yaml: test-file-content
+extra-node-kube-apiserver-args:
+ --extra-kube-apiserver-arg: extra-kube-apiserver-value
+extra-node-kube-controller-manager-args:
+ --extra-kube-controller-manager-arg: extra-kube-controller-manager-value
+extra-node-kube-scheduler-args:
+ --extra-kube-scheduler-arg: extra-kube-scheduler-value
+extra-node-kube-proxy-args:
+ --extra-kube-proxy-arg: extra-kube-proxy-value
+extra-node-kubelet-args:
+ --extra-kubelet-arg: extra-kubelet-value
+extra-node-containerd-args:
+ --extra-containerd-arg: extra-containerd-value
+extra-node-k8s-dqlite-args:
+ --extra-k8s-dqlite-arg: extra-k8s-dqlite-value
diff --git a/src/k8s/pkg/k8sd/app/cluster_util.go b/src/k8s/pkg/k8sd/app/cluster_util.go
index 2d5f55b5dc..adb7842ad0 100644
--- a/src/k8s/pkg/k8sd/app/cluster_util.go
+++ b/src/k8s/pkg/k8sd/app/cluster_util.go
@@ -3,12 +3,10 @@ package app
import (
"context"
"fmt"
- "net"
"path"
"github.com/canonical/k8s/pkg/k8sd/pki"
"github.com/canonical/k8s/pkg/k8sd/setup"
- "github.com/canonical/k8s/pkg/k8sd/types"
"github.com/canonical/k8s/pkg/snap"
snaputil "github.com/canonical/k8s/pkg/snap/util"
"github.com/canonical/microcluster/state"
@@ -35,29 +33,6 @@ func setupKubeconfigs(s *state.State, kubeConfigDir string, securePort int, pki
}
-func setupControlPlaneServices(snap snap.Snap, s *state.State, cfg types.ClusterConfig, nodeIP net.IP) error {
- // Configure services
- if err := setup.Containerd(snap, nil); err != nil {
- return fmt.Errorf("failed to configure containerd: %w", err)
- }
- if err := setup.KubeletControlPlane(snap, s.Name(), nodeIP, cfg.Kubelet.GetClusterDNS(), cfg.Kubelet.GetClusterDomain(), cfg.Kubelet.GetCloudProvider(), cfg.Kubelet.GetControlPlaneTaints()); err != nil {
- return fmt.Errorf("failed to configure kubelet: %w", err)
- }
- if err := setup.KubeProxy(s.Context, snap, s.Name(), cfg.Network.GetPodCIDR()); err != nil {
- return fmt.Errorf("failed to configure kube-proxy: %w", err)
- }
- if err := setup.KubeControllerManager(snap); err != nil {
- return fmt.Errorf("failed to configure kube-controller-manager: %w", err)
- }
- if err := setup.KubeScheduler(snap); err != nil {
- return fmt.Errorf("failed to configure kube-scheduler: %w", err)
- }
- if err := setup.KubeAPIServer(snap, cfg.Network.GetServiceCIDR(), s.Address().Path("1.0", "kubernetes", "auth", "webhook").String(), true, cfg.Datastore, cfg.APIServer.GetAuthorizationMode()); err != nil {
- return fmt.Errorf("failed to configure kube-apiserver: %w", err)
- }
- return nil
-}
-
func startControlPlaneServices(ctx context.Context, snap snap.Snap, datastore string) error {
// Start services
switch datastore {
diff --git a/src/k8s/pkg/k8sd/app/hooks_bootstrap.go b/src/k8s/pkg/k8sd/app/hooks_bootstrap.go
index 34033075c5..9599b87041 100644
--- a/src/k8s/pkg/k8sd/app/hooks_bootstrap.go
+++ b/src/k8s/pkg/k8sd/app/hooks_bootstrap.go
@@ -189,17 +189,20 @@ func (a *App) onBootstrapWorkerNode(s *state.State, encodedToken string, joinCon
}
// Worker node services
- if err := setup.Containerd(snap, nil); err != nil {
+ if err := setup.Containerd(snap, nil, joinConfig.ExtraNodeContainerdArgs); err != nil {
return fmt.Errorf("failed to configure containerd: %w", err)
}
- if err := setup.KubeletWorker(snap, s.Name(), nodeIP, response.ClusterDNS, response.ClusterDomain, response.CloudProvider); err != nil {
+ if err := setup.KubeletWorker(snap, s.Name(), nodeIP, response.ClusterDNS, response.ClusterDomain, response.CloudProvider, joinConfig.ExtraNodeKubeletArgs); err != nil {
return fmt.Errorf("failed to configure kubelet: %w", err)
}
- if err := setup.KubeProxy(s.Context, snap, s.Name(), response.PodCIDR); err != nil {
+ if err := setup.KubeProxy(s.Context, snap, s.Name(), response.PodCIDR, joinConfig.ExtraNodeKubeProxyArgs); err != nil {
return fmt.Errorf("failed to configure kube-proxy: %w", err)
}
- if err := setup.K8sAPIServerProxy(snap, response.APIServers); err != nil {
- return fmt.Errorf("failed to configure kube-proxy: %w", err)
+ if err := setup.K8sAPIServerProxy(snap, response.APIServers, joinConfig.ExtraNodeK8sAPIServerProxyArgs); err != nil {
+ return fmt.Errorf("failed to configure k8s-apiserver-proxy: %w", err)
+ }
+ if err := setup.ExtraNodeConfigFiles(snap, joinConfig.ExtraNodeConfigFiles); err != nil {
+ return fmt.Errorf("failed to write extra node config files: %w", err)
}
// TODO(berkayoz): remove the lock on cleanup
@@ -344,7 +347,7 @@ func (a *App) onBootstrapControlPlane(s *state.State, bootstrapConfig apiv1.Boot
// Configure datastore
switch cfg.Datastore.GetType() {
case "k8s-dqlite":
- if err := setup.K8sDqlite(snap, fmt.Sprintf("%s:%d", nodeIP.String(), cfg.Datastore.GetK8sDqlitePort()), nil); err != nil {
+ if err := setup.K8sDqlite(snap, fmt.Sprintf("%s:%d", nodeIP.String(), cfg.Datastore.GetK8sDqlitePort()), nil, bootstrapConfig.ExtraNodeK8sDqliteArgs); err != nil {
return fmt.Errorf("failed to configure k8s-dqlite: %w", err)
}
case "external":
@@ -353,8 +356,27 @@ func (a *App) onBootstrapControlPlane(s *state.State, bootstrapConfig apiv1.Boot
}
// Configure services
- if err := setupControlPlaneServices(snap, s, cfg, nodeIP); err != nil {
- return fmt.Errorf("failed to configure services: %w", err)
+ if err := setup.Containerd(snap, nil, bootstrapConfig.ExtraNodeContainerdArgs); err != nil {
+ return fmt.Errorf("failed to configure containerd: %w", err)
+ }
+ if err := setup.KubeletControlPlane(snap, s.Name(), nodeIP, cfg.Kubelet.GetClusterDNS(), cfg.Kubelet.GetClusterDomain(), cfg.Kubelet.GetCloudProvider(), cfg.Kubelet.GetControlPlaneTaints(), bootstrapConfig.ExtraNodeKubeletArgs); err != nil {
+ return fmt.Errorf("failed to configure kubelet: %w", err)
+ }
+ if err := setup.KubeProxy(s.Context, snap, s.Name(), cfg.Network.GetPodCIDR(), bootstrapConfig.ExtraNodeKubeProxyArgs); err != nil {
+ return fmt.Errorf("failed to configure kube-proxy: %w", err)
+ }
+ if err := setup.KubeControllerManager(snap, bootstrapConfig.ExtraNodeKubeControllerManagerArgs); err != nil {
+ return fmt.Errorf("failed to configure kube-controller-manager: %w", err)
+ }
+ if err := setup.KubeScheduler(snap, bootstrapConfig.ExtraNodeKubeSchedulerArgs); err != nil {
+ return fmt.Errorf("failed to configure kube-scheduler: %w", err)
+ }
+ if err := setup.KubeAPIServer(snap, cfg.Network.GetServiceCIDR(), s.Address().Path("1.0", "kubernetes", "auth", "webhook").String(), true, cfg.Datastore, cfg.APIServer.GetAuthorizationMode(), bootstrapConfig.ExtraNodeKubeAPIServerArgs); err != nil {
+ return fmt.Errorf("failed to configure kube-apiserver: %w", err)
+ }
+
+ if err := setup.ExtraNodeConfigFiles(snap, bootstrapConfig.ExtraNodeConfigFiles); err != nil {
+ return fmt.Errorf("failed to write extra node config files: %w", err)
}
// Write cluster configuration to dqlite
diff --git a/src/k8s/pkg/k8sd/app/hooks_join.go b/src/k8s/pkg/k8sd/app/hooks_join.go
index 2382525f0c..35d4d8e9af 100644
--- a/src/k8s/pkg/k8sd/app/hooks_join.go
+++ b/src/k8s/pkg/k8sd/app/hooks_join.go
@@ -134,7 +134,7 @@ func (a *App) onPostJoin(s *state.State, initConfig map[string]string) error {
}
address := fmt.Sprintf("%s:%d", nodeIP.String(), cfg.Datastore.GetK8sDqlitePort())
- if err := setup.K8sDqlite(snap, address, cluster); err != nil {
+ if err := setup.K8sDqlite(snap, address, cluster, joinConfig.ExtraNodeK8sDqliteArgs); err != nil {
return fmt.Errorf("failed to configure k8s-dqlite with address=%s cluster=%v: %w", address, cluster, err)
}
case "external":
@@ -143,8 +143,27 @@ func (a *App) onPostJoin(s *state.State, initConfig map[string]string) error {
}
// Configure services
- if err := setupControlPlaneServices(snap, s, cfg, nodeIP); err != nil {
- return fmt.Errorf("failed to configure services: %w", err)
+ if err := setup.Containerd(snap, nil, joinConfig.ExtraNodeContainerdArgs); err != nil {
+ return fmt.Errorf("failed to configure containerd: %w", err)
+ }
+ if err := setup.KubeletControlPlane(snap, s.Name(), nodeIP, cfg.Kubelet.GetClusterDNS(), cfg.Kubelet.GetClusterDomain(), cfg.Kubelet.GetCloudProvider(), cfg.Kubelet.GetControlPlaneTaints(), joinConfig.ExtraNodeKubeletArgs); err != nil {
+ return fmt.Errorf("failed to configure kubelet: %w", err)
+ }
+ if err := setup.KubeProxy(s.Context, snap, s.Name(), cfg.Network.GetPodCIDR(), joinConfig.ExtraNodeKubeProxyArgs); err != nil {
+ return fmt.Errorf("failed to configure kube-proxy: %w", err)
+ }
+ if err := setup.KubeControllerManager(snap, joinConfig.ExtraNodeKubeControllerManagerArgs); err != nil {
+ return fmt.Errorf("failed to configure kube-controller-manager: %w", err)
+ }
+ if err := setup.KubeScheduler(snap, joinConfig.ExtraNodeKubeSchedulerArgs); err != nil {
+ return fmt.Errorf("failed to configure kube-scheduler: %w", err)
+ }
+ if err := setup.KubeAPIServer(snap, cfg.Network.GetServiceCIDR(), s.Address().Path("1.0", "kubernetes", "auth", "webhook").String(), true, cfg.Datastore, cfg.APIServer.GetAuthorizationMode(), joinConfig.ExtraNodeKubeAPIServerArgs); err != nil {
+ return fmt.Errorf("failed to configure kube-apiserver: %w", err)
+ }
+
+ if err := setup.ExtraNodeConfigFiles(snap, joinConfig.ExtraNodeConfigFiles); err != nil {
+ return fmt.Errorf("failed to write extra node config files: %w", err)
}
if err := snapdconfig.SetSnapdFromK8sd(s.Context, cfg.ToUserFacing(), snap); err != nil {
diff --git a/src/k8s/pkg/k8sd/setup/containerd.go b/src/k8s/pkg/k8sd/setup/containerd.go
index 1e7abab334..4b61a9fa4d 100644
--- a/src/k8s/pkg/k8sd/setup/containerd.go
+++ b/src/k8s/pkg/k8sd/setup/containerd.go
@@ -113,7 +113,7 @@ func containerdHostConfig(registry types.ContainerdRegistry) containerdHostsConf
// Containerd configures configuration and arguments for containerd on the local node.
// Optionally, a number of registry mirrors and auths can be configured.
-func Containerd(snap snap.Snap, registries []types.ContainerdRegistry) error {
+func Containerd(snap snap.Snap, registries []types.ContainerdRegistry, extraArgs map[string]*string) error {
configToml, err := os.OpenFile(path.Join(snap.ContainerdConfigDir(), "config.toml"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return fmt.Errorf("failed to open config.toml: %w", err)
@@ -138,6 +138,12 @@ func Containerd(snap snap.Snap, registries []types.ContainerdRegistry) error {
return fmt.Errorf("failed to write arguments file: %w", err)
}
+ // Apply extra arguments after the defaults, so they can override them.
+ updateArgs, deleteArgs := utils.ServiceArgsFromMap(extraArgs)
+ if _, err := snaputil.UpdateServiceArguments(snap, "containerd", updateArgs, deleteArgs); err != nil {
+ return fmt.Errorf("failed to write arguments file: %w", err)
+ }
+
cniBinary := path.Join(snap.CNIBinDir(), "cni")
if err := utils.CopyFile(snap.CNIPluginsBinary(), cniBinary); err != nil {
return fmt.Errorf("failed to copy cni plugin binary: %w", err)
diff --git a/src/k8s/pkg/k8sd/setup/containerd_test.go b/src/k8s/pkg/k8sd/setup/containerd_test.go
index 50b4c55563..9a898f1c79 100644
--- a/src/k8s/pkg/k8sd/setup/containerd_test.go
+++ b/src/k8s/pkg/k8sd/setup/containerd_test.go
@@ -12,6 +12,7 @@ import (
"github.com/canonical/k8s/pkg/k8sd/types"
"github.com/canonical/k8s/pkg/snap/mock"
snaputil "github.com/canonical/k8s/pkg/snap/util"
+ "github.com/canonical/k8s/pkg/utils"
. "github.com/onsi/gomega"
)
@@ -40,7 +41,7 @@ func TestContainerd(t *testing.T) {
},
}
- g.Expect(setup.EnsureAllDirectories(s)).To(BeNil())
+ g.Expect(setup.EnsureAllDirectories(s)).To(Succeed())
g.Expect(setup.Containerd(s, []types.ContainerdRegistry{
{
Host: "docker.io",
@@ -53,6 +54,11 @@ func TestContainerd(t *testing.T) {
URLs: []string{"https://ghcr.mirror.internal"},
Token: "token",
},
+ }, map[string]*string{
+ "--log-level": utils.Pointer("debug"),
+ "--metrics": utils.Pointer("true"),
+ "--address": nil, // This should trigger a delete
+ "--my-extra-arg": utils.Pointer("my-extra-val"),
})).To(Succeed())
t.Run("Config", func(t *testing.T) {
@@ -102,10 +108,12 @@ func TestContainerd(t *testing.T) {
t.Run("Args", func(t *testing.T) {
for key, expectedVal := range map[string]string{
- "--address": path.Join(dir, "containerd-run", "containerd.sock"),
- "--config": path.Join(dir, "containerd", "config.toml"),
- "--root": path.Join(dir, "containerd-root"),
- "--state": path.Join(dir, "containerd-state"),
+ "--config": path.Join(dir, "containerd", "config.toml"),
+ "--root": path.Join(dir, "containerd-root"),
+ "--state": path.Join(dir, "containerd-state"),
+ "--log-level": "debug",
+ "--metrics": "true",
+ "--my-extra-arg": "my-extra-val",
} {
t.Run(key, func(t *testing.T) {
g := NewWithT(t)
@@ -114,6 +122,13 @@ func TestContainerd(t *testing.T) {
g.Expect(val).To(Equal(expectedVal))
})
}
+ // --address was deleted by extraArgs
+ t.Run("--address", func(t *testing.T) {
+ g := NewWithT(t)
+ val, err := snaputil.GetServiceArgument(s, "containerd", "--address")
+ g.Expect(err).To(BeNil())
+ g.Expect(val).To(BeZero())
+ })
})
t.Run("Registries", func(t *testing.T) {
diff --git a/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go b/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go
index a4c9f4f16b..f5683a6a72 100644
--- a/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go
+++ b/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy.go
@@ -7,10 +7,11 @@ import (
"github.com/canonical/k8s/pkg/proxy"
"github.com/canonical/k8s/pkg/snap"
snaputil "github.com/canonical/k8s/pkg/snap/util"
+ "github.com/canonical/k8s/pkg/utils"
)
// K8sAPIServerProxy prepares configuration for k8s-apiserver-proxy.
-func K8sAPIServerProxy(snap snap.Snap, servers []string) error {
+func K8sAPIServerProxy(snap snap.Snap, servers []string, extraArgs map[string]*string) error {
configFile := path.Join(snap.ServiceExtraConfigDir(), "k8s-apiserver-proxy.json")
if err := proxy.WriteEndpointsConfig(servers, configFile); err != nil {
return fmt.Errorf("failed to write proxy configuration file: %w", err)
@@ -24,5 +25,10 @@ func K8sAPIServerProxy(snap snap.Snap, servers []string) error {
return fmt.Errorf("failed to write arguments file: %w", err)
}
+ // Apply extra arguments after the defaults, so they can override them.
+ updateArgs, deleteArgs := utils.ServiceArgsFromMap(extraArgs)
+ if _, err := snaputil.UpdateServiceArguments(snap, "k8s-apiserver-proxy", updateArgs, deleteArgs); err != nil {
+ return fmt.Errorf("failed to write arguments file: %w", err)
+ }
return nil
}
diff --git a/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy_test.go b/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy_test.go
index 4bca4ddf5d..3a9bf54b6d 100644
--- a/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy_test.go
+++ b/src/k8s/pkg/k8sd/setup/k8s_apiserver_proxy_test.go
@@ -27,7 +27,7 @@ func TestK8sApiServerProxy(t *testing.T) {
s := mustSetupSnapAndDirectories(t, setK8sApiServerMock)
- g.Expect(setup.K8sAPIServerProxy(s, nil)).To(Succeed())
+ g.Expect(setup.K8sAPIServerProxy(s, nil, nil)).To(Succeed())
tests := []struct {
key string
@@ -45,6 +45,48 @@ func TestK8sApiServerProxy(t *testing.T) {
g.Expect(tc.expectedVal).To(Equal(val))
})
}
+
+ args, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "k8s-apiserver-proxy"))
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(len(args)).To(Equal(len(tests)))
+ })
+
+ t.Run("WithExtraArgs", func(t *testing.T) {
+ g := NewWithT(t)
+
+ s := mustSetupSnapAndDirectories(t, setK8sApiServerMock)
+
+ extraArgs := map[string]*string{
+ "--kubeconfig": utils.Pointer("overridden-kubelet.conf"),
+ "--listen": nil, // This should trigger a delete
+ "--my-extra-arg": utils.Pointer("my-extra-val"),
+ }
+ g.Expect(setup.K8sAPIServerProxy(s, nil, extraArgs)).To(Succeed())
+
+ tests := []struct {
+ key string
+ expectedVal string
+ }{
+ {key: "--endpoints", expectedVal: path.Join(s.Mock.ServiceExtraConfigDir, "k8s-apiserver-proxy.json")},
+ {key: "--kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "overridden-kubelet.conf")},
+ {key: "--my-extra-arg", expectedVal: "my-extra-val"},
+ }
+ for _, tc := range tests {
+ t.Run(tc.key, func(t *testing.T) {
+ g := NewWithT(t)
+ val, err := snaputil.GetServiceArgument(s, "k8s-apiserver-proxy", tc.key)
+ g.Expect(err).To(BeNil())
+ g.Expect(tc.expectedVal).To(Equal(val))
+ })
+ }
+ // --listen was deleted by extraArgs
+ t.Run("--listen", func(t *testing.T) {
+ g := NewWithT(t)
+ val, err := snaputil.GetServiceArgument(s, "k8s-apiserver-proxy", "--listen")
+ g.Expect(err).To(BeNil())
+ g.Expect(val).To(BeZero())
+ })
+
args, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "k8s-apiserver-proxy"))
g.Expect(err).ToNot(HaveOccurred())
g.Expect(len(args)).To(Equal(len(tests)))
@@ -56,7 +98,7 @@ func TestK8sApiServerProxy(t *testing.T) {
s := mustSetupSnapAndDirectories(t, setK8sApiServerMock)
s.Mock.ServiceExtraConfigDir = "nonexistent"
- g.Expect(setup.K8sAPIServerProxy(s, nil)).ToNot(Succeed())
+ g.Expect(setup.K8sAPIServerProxy(s, nil, nil)).ToNot(Succeed())
})
t.Run("MissingServiceArgumentsDir", func(t *testing.T) {
@@ -65,7 +107,7 @@ func TestK8sApiServerProxy(t *testing.T) {
s := mustSetupSnapAndDirectories(t, setK8sApiServerMock)
s.Mock.ServiceArgumentsDir = "nonexistent"
- g.Expect(setup.K8sAPIServerProxy(s, nil)).ToNot(Succeed())
+ g.Expect(setup.K8sAPIServerProxy(s, nil, nil)).ToNot(Succeed())
})
t.Run("JSONFileContent", func(t *testing.T) {
@@ -76,7 +118,7 @@ func TestK8sApiServerProxy(t *testing.T) {
endpoints := []string{"192.168.0.1", "192.168.0.2", "192.168.0.3"}
fileName := path.Join(s.Mock.ServiceExtraConfigDir, "k8s-apiserver-proxy.json")
- g.Expect(setup.K8sAPIServerProxy(s, endpoints)).To(Succeed())
+ g.Expect(setup.K8sAPIServerProxy(s, endpoints, nil)).To(Succeed())
b, err := os.ReadFile(fileName)
g.Expect(err).NotTo(HaveOccurred())
diff --git a/src/k8s/pkg/k8sd/setup/k8s_dqlite.go b/src/k8s/pkg/k8sd/setup/k8s_dqlite.go
index 5e7ca6f3bc..f9a95d6e64 100644
--- a/src/k8s/pkg/k8sd/setup/k8s_dqlite.go
+++ b/src/k8s/pkg/k8sd/setup/k8s_dqlite.go
@@ -7,6 +7,7 @@ import (
"github.com/canonical/k8s/pkg/snap"
snaputil "github.com/canonical/k8s/pkg/snap/util"
+ "github.com/canonical/k8s/pkg/utils"
"gopkg.in/yaml.v2"
)
@@ -15,7 +16,7 @@ type k8sDqliteInit struct {
Cluster []string `yaml:"Cluster,omitempty"`
}
-func K8sDqlite(snap snap.Snap, address string, cluster []string) error {
+func K8sDqlite(snap snap.Snap, address string, cluster []string, extraArgs map[string]*string) error {
b, err := yaml.Marshal(&k8sDqliteInit{Address: address, Cluster: cluster})
if err != nil {
return fmt.Errorf("failed to create init.yaml file for address=%s cluster=%v: %w", address, cluster, err)
@@ -31,5 +32,11 @@ func K8sDqlite(snap snap.Snap, address string, cluster []string) error {
}, nil); err != nil {
return fmt.Errorf("failed to write arguments file: %w", err)
}
+
+ // Apply extra arguments after the defaults, so they can override them.
+ updateArgs, deleteArgs := utils.ServiceArgsFromMap(extraArgs)
+ if _, err := snaputil.UpdateServiceArguments(snap, "k8s-dqlite", updateArgs, deleteArgs); err != nil {
+ return fmt.Errorf("failed to write arguments file: %w", err)
+ }
return nil
}
diff --git a/src/k8s/pkg/k8sd/setup/k8s_dqlite_test.go b/src/k8s/pkg/k8sd/setup/k8s_dqlite_test.go
index 7f2dc733c9..077f94e6e7 100644
--- a/src/k8s/pkg/k8sd/setup/k8s_dqlite_test.go
+++ b/src/k8s/pkg/k8sd/setup/k8s_dqlite_test.go
@@ -28,7 +28,7 @@ func TestK8sDqlite(t *testing.T) {
s := mustSetupSnapAndDirectories(t, setK8sDqliteMock)
// Call the K8sDqlite setup function with mock arguments
- g.Expect(setup.K8sDqlite(s, "192.168.0.1:1234", []string{"192.168.0.1:1234"})).To(BeNil())
+ g.Expect(setup.K8sDqlite(s, "192.168.0.1:1234", []string{"192.168.0.1:1234"}, nil)).To(BeNil())
// Ensure the K8sDqlite arguments file has the expected arguments and values
tests := []struct {
@@ -53,6 +53,51 @@ func TestK8sDqlite(t *testing.T) {
g.Expect(len(args)).To(Equal(len(tests)))
})
+ t.Run("WithExtraArgs", func(t *testing.T) {
+ g := NewWithT(t)
+
+ // Create a mock snap
+ s := mustSetupSnapAndDirectories(t, setK8sDqliteMock)
+
+ extraArgs := map[string]*string{
+ "--my-extra-arg": utils.Pointer("my-extra-val"),
+ "--listen": nil,
+ "--storage-dir": utils.Pointer("overridden-storage-dir"),
+ }
+ // Call the K8sDqlite setup function with mock arguments
+ g.Expect(setup.K8sDqlite(s, "192.168.0.1:1234", []string{"192.168.0.1:1234"}, extraArgs)).To(BeNil())
+
+ // Ensure the K8sDqlite arguments file has the expected arguments and values
+ tests := []struct {
+ key string
+ expectedVal string
+ }{
+ {key: "--storage-dir", expectedVal: "overridden-storage-dir"},
+ {key: "--my-extra-arg", expectedVal: "my-extra-val"},
+ }
+ for _, tc := range tests {
+ t.Run(tc.key, func(t *testing.T) {
+ g := NewWithT(t)
+ val, err := snaputil.GetServiceArgument(s, "k8s-dqlite", tc.key)
+ g.Expect(err).To(BeNil())
+ g.Expect(val).To(Equal(tc.expectedVal))
+ })
+ }
+
+ // --listen was deleted by extraArgs
+ t.Run("--listen", func(t *testing.T) {
+ g := NewWithT(t)
+ val, err := snaputil.GetServiceArgument(s, "k8s-dqlite", "--listen")
+ g.Expect(err).To(BeNil())
+ g.Expect(val).To(BeZero())
+ })
+
+ // Ensure the K8sDqlite arguments file has exactly the expected number of arguments
+ args, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "k8s-dqlite"))
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(len(args)).To(Equal(len(tests)))
+ })
+
t.Run("YAMLFileContents", func(t *testing.T) {
g := NewWithT(t)
@@ -67,7 +112,7 @@ func TestK8sDqlite(t *testing.T) {
"192.168.0.3:1234",
}
- g.Expect(setup.K8sDqlite(s, "192.168.0.1:1234", cluster)).To(BeNil())
+ g.Expect(setup.K8sDqlite(s, "192.168.0.1:1234", cluster, nil)).To(BeNil())
b, err := os.ReadFile(path.Join(s.Mock.K8sDqliteStateDir, "init.yaml"))
g.Expect(err).To(BeNil())
@@ -81,7 +126,7 @@ func TestK8sDqlite(t *testing.T) {
s.Mock.K8sDqliteStateDir = "nonexistent"
- g.Expect(setup.K8sDqlite(s, "", []string{})).ToNot(Succeed())
+ g.Expect(setup.K8sDqlite(s, "", []string{}, nil)).ToNot(Succeed())
})
t.Run("MissingArgsDir", func(t *testing.T) {
@@ -91,6 +136,6 @@ func TestK8sDqlite(t *testing.T) {
s.Mock.ServiceArgumentsDir = "nonexistent"
- g.Expect(setup.K8sDqlite(s, "", []string{})).ToNot(Succeed())
+ g.Expect(setup.K8sDqlite(s, "", []string{}, nil)).ToNot(Succeed())
})
}
diff --git a/src/k8s/pkg/k8sd/setup/kube_apiserver.go b/src/k8s/pkg/k8sd/setup/kube_apiserver.go
index 6de30ebeaf..be2e9e43bd 100644
--- a/src/k8s/pkg/k8sd/setup/kube_apiserver.go
+++ b/src/k8s/pkg/k8sd/setup/kube_apiserver.go
@@ -9,6 +9,7 @@ import (
"github.com/canonical/k8s/pkg/k8sd/types"
"github.com/canonical/k8s/pkg/snap"
snaputil "github.com/canonical/k8s/pkg/snap/util"
+ "github.com/canonical/k8s/pkg/utils"
)
type apiserverAuthTokenWebhookTemplateConfig struct {
@@ -47,7 +48,7 @@ var (
)
// KubeAPIServer configures kube-apiserver on the local node.
-func KubeAPIServer(snap snap.Snap, serviceCIDR string, authWebhookURL string, enableFrontProxy bool, datastore types.Datastore, authorizationMode string) error {
+func KubeAPIServer(snap snap.Snap, serviceCIDR string, authWebhookURL string, enableFrontProxy bool, datastore types.Datastore, authorizationMode string, extraArgs map[string]*string) error {
authTokenWebhookConfigFile := path.Join(snap.ServiceExtraConfigDir(), "auth-token-webhook.conf")
authTokenWebhookFile, err := os.OpenFile(authTokenWebhookConfigFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
@@ -108,5 +109,11 @@ func KubeAPIServer(snap snap.Snap, serviceCIDR string, authWebhookURL string, en
if _, err := snaputil.UpdateServiceArguments(snap, "kube-apiserver", args, deleteArgs); err != nil {
return fmt.Errorf("failed to render arguments file: %w", err)
}
+
+ // Apply extra arguments after the defaults, so they can override them.
+ updateArgs, deleteArgs := utils.ServiceArgsFromMap(extraArgs)
+ if _, err := snaputil.UpdateServiceArguments(snap, "kube-apiserver", updateArgs, deleteArgs); err != nil {
+ return fmt.Errorf("failed to write arguments file: %w", err)
+ }
return nil
}
diff --git a/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go b/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go
index a9f379a754..6da0cea2da 100644
--- a/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go
+++ b/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go
@@ -36,7 +36,7 @@ func TestKubeAPIServer(t *testing.T) {
s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock)
// Call the KubeAPIServer setup function with mock arguments
- g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", true, types.Datastore{Type: utils.Pointer("k8s-dqlite")}, "Node,RBAC")).To(BeNil())
+ g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", true, types.Datastore{Type: utils.Pointer("k8s-dqlite")}, "Node,RBAC", nil)).To(BeNil())
// Ensure the kube-apiserver arguments file has the expected arguments and values
tests := []struct {
@@ -94,7 +94,7 @@ func TestKubeAPIServer(t *testing.T) {
s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock)
// Call the KubeAPIServer setup function with mock arguments
- g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("k8s-dqlite")}, "Node,RBAC")).To(BeNil())
+ g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("k8s-dqlite")}, "Node,RBAC", nil)).To(BeNil())
// Ensure the kube-apiserver arguments file has the expected arguments and values
tests := []struct {
@@ -138,13 +138,76 @@ func TestKubeAPIServer(t *testing.T) {
g.Expect(len(args)).To(Equal(len(tests)))
})
+ t.Run("WithExtraArgs", func(t *testing.T) {
+ g := NewWithT(t)
+
+ // Create a mock snap
+ s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock)
+
+ extraArgs := map[string]*string{
+ "--allow-privileged": nil,
+ "--secure-port": utils.Pointer("1337"),
+ "--my-extra-arg": utils.Pointer("my-extra-val"),
+ }
+ // Call the KubeAPIServer setup function with mock arguments
+ g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", true, types.Datastore{Type: utils.Pointer("k8s-dqlite")}, "Node,RBAC", extraArgs)).To(BeNil())
+
+ // Ensure the kube-apiserver arguments file has the expected arguments and values
+ tests := []struct {
+ key string
+ expectedVal string
+ }{
+ {key: "--authentication-token-webhook-config-file", expectedVal: path.Join(s.Mock.ServiceExtraConfigDir, "auth-token-webhook.conf")},
+ {key: "--authorization-mode", expectedVal: "Node,RBAC"},
+ {key: "--client-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "client-ca.crt")},
+ {key: "--enable-admission-plugins", expectedVal: "NodeRestriction"},
+ {key: "--kubelet-certificate-authority", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "ca.crt")},
+ {key: "--kubelet-client-certificate", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "apiserver-kubelet-client.crt")},
+ {key: "--kubelet-client-key", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "apiserver-kubelet-client.key")},
+ {key: "--kubelet-preferred-address-types", expectedVal: "InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP"},
+ {key: "--secure-port", expectedVal: "1337"},
+ {key: "--service-account-issuer", expectedVal: "https://kubernetes.default.svc"},
+ {key: "--service-account-key-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "serviceaccount.key")},
+ {key: "--service-account-signing-key-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "serviceaccount.key")},
+ {key: "--service-cluster-ip-range", expectedVal: "10.0.0.0/24"},
+ {key: "--tls-cert-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "apiserver.crt")},
+ {key: "--tls-cipher-suites", expectedVal: apiserverTLSCipherSuites},
+ {key: "--tls-private-key-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "apiserver.key")},
+ {key: "--etcd-servers", expectedVal: fmt.Sprintf("unix://%s", path.Join(s.Mock.K8sDqliteStateDir, "k8s-dqlite.sock"))},
+ {key: "--requestheader-client-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "front-proxy-ca.crt")},
+ {key: "--requestheader-allowed-names", expectedVal: "front-proxy-client"},
+ {key: "--requestheader-extra-headers-prefix", expectedVal: "X-Remote-Extra-"},
+ {key: "--requestheader-group-headers", expectedVal: "X-Remote-Group"},
+ {key: "--requestheader-username-headers", expectedVal: "X-Remote-User"},
+ {key: "--proxy-client-cert-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "front-proxy-client.crt")},
+ {key: "--proxy-client-key-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "front-proxy-client.key")},
+ {key: "--my-extra-arg", expectedVal: "my-extra-val"},
+ }
+ for _, tc := range tests {
+ t.Run(tc.key, func(t *testing.T) {
+ g := NewWithT(t)
+ val, err := snaputil.GetServiceArgument(s, "kube-apiserver", tc.key)
+ g.Expect(err).To(BeNil())
+ g.Expect(val).To(Equal(tc.expectedVal))
+ })
+ }
+ // Ensure that the allow-privileged argument was deleted
+ val, err := snaputil.GetServiceArgument(s, "kube-apiserver", "--allow-privileged")
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(val).To(BeZero())
+
+ // Ensure the kube-apiserver arguments file has exactly the expected number of arguments
+ args, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "kube-apiserver"))
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(len(args)).To(Equal(len(tests)))
+ })
t.Run("ArgsDualstack", func(t *testing.T) {
g := NewWithT(t)
s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock)
// Setup without proxy to simplify argument list
- g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24,fd01::/64", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("external"), ExternalServers: utils.Pointer([]string{"datastoreurl1", "datastoreurl2"})}, "Node,RBAC")).To(BeNil())
+ g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24,fd01::/64", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("external"), ExternalServers: utils.Pointer([]string{"datastoreurl1", "datastoreurl2"})}, "Node,RBAC", nil)).To(BeNil())
g.Expect(snaputil.GetServiceArgument(s, "kube-apiserver", "--service-cluster-ip-range")).To(Equal("10.0.0.0/24,fd01::/64"))
_, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "kube-apiserver"))
@@ -157,7 +220,7 @@ func TestKubeAPIServer(t *testing.T) {
s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock)
// Setup without proxy to simplify argument list
- g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("external"), ExternalServers: utils.Pointer([]string{"datastoreurl1", "datastoreurl2"})}, "Node,RBAC")).To(BeNil())
+ g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("external"), ExternalServers: utils.Pointer([]string{"datastoreurl1", "datastoreurl2"})}, "Node,RBAC", nil)).To(BeNil())
g.Expect(snaputil.GetServiceArgument(s, "kube-apiserver", "--etcd-servers")).To(Equal("datastoreurl1,datastoreurl2"))
_, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "kube-apiserver"))
@@ -171,7 +234,7 @@ func TestKubeAPIServer(t *testing.T) {
s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock)
// Attempt to configure kube-apiserver with an unsupported datastore
- err := setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("unsupported")}, "Node,RBAC")
+ err := setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: utils.Pointer("unsupported")}, "Node,RBAC", nil)
g.Expect(err).To(HaveOccurred())
g.Expect(err).To(MatchError(ContainSubstring("unsupported datastore")))
})
diff --git a/src/k8s/pkg/k8sd/setup/kube_controller_manager.go b/src/k8s/pkg/k8sd/setup/kube_controller_manager.go
index 10b5f6561c..746a44eb2b 100644
--- a/src/k8s/pkg/k8sd/setup/kube_controller_manager.go
+++ b/src/k8s/pkg/k8sd/setup/kube_controller_manager.go
@@ -7,10 +7,11 @@ import (
"github.com/canonical/k8s/pkg/snap"
snaputil "github.com/canonical/k8s/pkg/snap/util"
+ "github.com/canonical/k8s/pkg/utils"
)
// KubeControllerManager configures kube-controller-manager on the local node.
-func KubeControllerManager(snap snap.Snap) error {
+func KubeControllerManager(snap snap.Snap, extraArgs map[string]*string) error {
args := map[string]string{
"--authentication-kubeconfig": path.Join(snap.KubernetesConfigDir(), "controller.conf"),
"--authorization-kubeconfig": path.Join(snap.KubernetesConfigDir(), "controller.conf"),
@@ -31,5 +32,10 @@ func KubeControllerManager(snap snap.Snap) error {
if _, err := snaputil.UpdateServiceArguments(snap, "kube-controller-manager", args, nil); err != nil {
return fmt.Errorf("failed to render arguments file: %w", err)
}
+ // Apply extra arguments after the defaults, so they can override them.
+ updateArgs, deleteArgs := utils.ServiceArgsFromMap(extraArgs)
+ if _, err := snaputil.UpdateServiceArguments(snap, "kube-controller-manager", updateArgs, deleteArgs); err != nil {
+ return fmt.Errorf("failed to write arguments file: %w", err)
+ }
return nil
}
diff --git a/src/k8s/pkg/k8sd/setup/kube_controller_manager_test.go b/src/k8s/pkg/k8sd/setup/kube_controller_manager_test.go
index 91e8686849..625f95fe73 100644
--- a/src/k8s/pkg/k8sd/setup/kube_controller_manager_test.go
+++ b/src/k8s/pkg/k8sd/setup/kube_controller_manager_test.go
@@ -31,7 +31,7 @@ func TestKubeControllerManager(t *testing.T) {
os.Create(path.Join(s.Mock.KubernetesPKIDir, "ca.key"))
// Call the kube controller manager setup function
- g.Expect(setup.KubeControllerManager(s)).To(BeNil())
+ g.Expect(setup.KubeControllerManager(s, nil)).To(BeNil())
// Ensure the kube controller manager arguments file has the expected arguments and values
tests := []struct {
@@ -68,7 +68,7 @@ func TestKubeControllerManager(t *testing.T) {
t.Run("MissingArgsDir", func(t *testing.T) {
g := NewWithT(t)
s.Mock.ServiceArgumentsDir = "nonexistent"
- g.Expect(setup.KubeControllerManager(s)).ToNot(Succeed())
+ g.Expect(setup.KubeControllerManager(s, nil)).ToNot(Succeed())
})
})
@@ -79,7 +79,7 @@ func TestKubeControllerManager(t *testing.T) {
s := mustSetupSnapAndDirectories(t, setKubeControllerManagerMock)
// Call the kube controller manager setup function
- g.Expect(setup.KubeControllerManager(s)).To(BeNil())
+ g.Expect(setup.KubeControllerManager(s, nil)).To(BeNil())
// Ensure the kube controller manager arguments file has the expected arguments and values
tests := []struct {
@@ -114,7 +114,67 @@ func TestKubeControllerManager(t *testing.T) {
t.Run("MissingArgsDir", func(t *testing.T) {
g := NewWithT(t)
s.Mock.ServiceArgumentsDir = "nonexistent"
- g.Expect(setup.KubeControllerManager(s)).ToNot(Succeed())
+ g.Expect(setup.KubeControllerManager(s, nil)).ToNot(Succeed())
+ })
+ })
+
+ t.Run("WithExtraArgs", func(t *testing.T) {
+ g := NewWithT(t)
+
+ // Create a mock snap
+ s := mustSetupSnapAndDirectories(t, setKubeControllerManagerMock)
+
+ // Create ca.key so that cluster-signing-cert-file and cluster-signing-key-file are added to the arguments
+ os.Create(path.Join(s.Mock.KubernetesPKIDir, "ca.key"))
+
+ extraArgs := map[string]*string{
+ "--leader-elect-lease-duration": nil,
+ "--profiling": utils.Pointer("true"),
+ "--my-extra-arg": utils.Pointer("my-extra-val"),
+ }
+ // Call the kube controller manager setup function
+ g.Expect(setup.KubeControllerManager(s, extraArgs)).To(BeNil())
+
+ // Ensure the kube controller manager arguments file has the expected arguments and values
+ tests := []struct {
+ key string
+ expectedVal string
+ }{
+ {key: "--authentication-kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "controller.conf")},
+ {key: "--authorization-kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "controller.conf")},
+ {key: "--kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "controller.conf")},
+ {key: "--leader-elect-renew-deadline", expectedVal: "15s"},
+ {key: "--profiling", expectedVal: "true"},
+ {key: "--root-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "ca.crt")},
+ {key: "--service-account-private-key-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "serviceaccount.key")},
+ {key: "--use-service-account-credentials", expectedVal: "true"},
+ {key: "--cluster-signing-cert-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "ca.crt")},
+ {key: "--cluster-signing-key-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "ca.key")},
+ {key: "--my-extra-arg", expectedVal: "my-extra-val"},
+ }
+ for _, tc := range tests {
+ t.Run(tc.key, func(t *testing.T) {
+ g := NewWithT(t)
+ val, err := snaputil.GetServiceArgument(s, "kube-controller-manager", tc.key)
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(tc.expectedVal).To(Equal(val))
+ })
+ }
+
+ // Ensure that the leader-elect-lease-duration argument was deleted
+ val, err := snaputil.GetServiceArgument(s, "kube-controller-manager", "--leader-elect-lease-duration")
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(val).To(BeZero())
+
+ // Ensure the kube controller manager arguments file has exactly the expected number of arguments
+ args, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "kube-controller-manager"))
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(len(args)).To(Equal(len(tests)))
+
+ t.Run("MissingArgsDir", func(t *testing.T) {
+ g := NewWithT(t)
+ s.Mock.ServiceArgumentsDir = "nonexistent"
+ g.Expect(setup.KubeControllerManager(s, nil)).ToNot(Succeed())
})
})
}
diff --git a/src/k8s/pkg/k8sd/setup/kube_proxy.go b/src/k8s/pkg/k8sd/setup/kube_proxy.go
index a1f7a59f69..64da185d62 100644
--- a/src/k8s/pkg/k8sd/setup/kube_proxy.go
+++ b/src/k8s/pkg/k8sd/setup/kube_proxy.go
@@ -8,10 +8,11 @@ import (
"github.com/canonical/k8s/pkg/snap"
snaputil "github.com/canonical/k8s/pkg/snap/util"
+ "github.com/canonical/k8s/pkg/utils"
)
// KubeProxy configures kube-proxy on the local node.
-func KubeProxy(ctx context.Context, snap snap.Snap, hostname string, podCIDR string) error {
+func KubeProxy(ctx context.Context, snap snap.Snap, hostname string, podCIDR string, extraArgs map[string]*string) error {
serviceArgs := map[string]string{
"--cluster-cidr": podCIDR,
"--healthz-bind-address": "127.0.0.1",
@@ -33,5 +34,11 @@ func KubeProxy(ctx context.Context, snap snap.Snap, hostname string, podCIDR str
if _, err := snaputil.UpdateServiceArguments(snap, "kube-proxy", serviceArgs, nil); err != nil {
return fmt.Errorf("failed to render arguments file: %w", err)
}
+
+ // Apply extra arguments after the defaults, so they can override them.
+ updateArgs, deleteArgs := utils.ServiceArgsFromMap(extraArgs)
+ if _, err := snaputil.UpdateServiceArguments(snap, "kube-proxy", updateArgs, deleteArgs); err != nil {
+ return fmt.Errorf("failed to write arguments file: %w", err)
+ }
return nil
}
diff --git a/src/k8s/pkg/k8sd/setup/kube_proxy_test.go b/src/k8s/pkg/k8sd/setup/kube_proxy_test.go
index 17d8ac3821..69c71d9214 100644
--- a/src/k8s/pkg/k8sd/setup/kube_proxy_test.go
+++ b/src/k8s/pkg/k8sd/setup/kube_proxy_test.go
@@ -9,6 +9,7 @@ import (
"github.com/canonical/k8s/pkg/k8sd/setup"
"github.com/canonical/k8s/pkg/snap/mock"
snaputil "github.com/canonical/k8s/pkg/snap/util"
+ "github.com/canonical/k8s/pkg/utils"
. "github.com/onsi/gomega"
)
@@ -30,11 +31,10 @@ func TestKubeProxy(t *testing.T) {
g.Expect(setup.EnsureAllDirectories(s)).To(BeNil())
t.Run("Args", func(t *testing.T) {
- g.Expect(setup.KubeProxy(context.Background(), s, "myhostname", "10.1.0.0/16")).To(BeNil())
+ g.Expect(setup.KubeProxy(context.Background(), s, "myhostname", "10.1.0.0/16", nil)).To(BeNil())
for key, expectedVal := range map[string]string{
"--cluster-cidr": "10.1.0.0/16",
- "--healthz-bind-address": "127.0.0.1",
"--hostname-override": "myhostname",
"--kubeconfig": path.Join(dir, "kubernetes", "proxy.conf"),
"--profiling": "false",
@@ -49,9 +49,38 @@ func TestKubeProxy(t *testing.T) {
}
})
+ t.Run("WithExtraArgs", func(t *testing.T) {
+ extraArgs := map[string]*string{
+ "--hostname-override": utils.Pointer("myoverriddenhostname"),
+ "--healthz-bind-address": nil,
+ "--my-extra-arg": utils.Pointer("my-extra-val"),
+ }
+ g.Expect(setup.KubeProxy(context.Background(), s, "myhostname", "10.1.0.0/16", extraArgs)).To(BeNil())
+
+ for key, expectedVal := range map[string]string{
+ "--cluster-cidr": "10.1.0.0/16",
+ "--hostname-override": "myoverriddenhostname",
+ "--kubeconfig": path.Join(dir, "kubernetes", "proxy.conf"),
+ "--profiling": "false",
+ "--conntrack-max-per-core": "",
+ "--my-extra-arg": "my-extra-val",
+ } {
+ t.Run(key, func(t *testing.T) {
+ g := NewWithT(t)
+ val, err := snaputil.GetServiceArgument(s, "kube-proxy", key)
+ g.Expect(err).To(BeNil())
+ g.Expect(val).To(Equal(expectedVal))
+ })
+ }
+ // Ensure that the healthz-bind-address argument was deleted
+ val, err := snaputil.GetServiceArgument(s, "kube-proxy", "--healthz-bind-address")
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(val).To(BeZero())
+ })
+
s.Mock.OnLXD = true
t.Run("ArgsOnLXD", func(t *testing.T) {
- g.Expect(setup.KubeProxy(context.Background(), s, "myhostname", "10.1.0.0/16")).To(BeNil())
+ g.Expect(setup.KubeProxy(context.Background(), s, "myhostname", "10.1.0.0/16", nil)).To(BeNil())
for key, expectedVal := range map[string]string{
"--conntrack-max-per-core": "0",
diff --git a/src/k8s/pkg/k8sd/setup/kube_scheduler.go b/src/k8s/pkg/k8sd/setup/kube_scheduler.go
index 1cc6ae5b42..bc687af318 100644
--- a/src/k8s/pkg/k8sd/setup/kube_scheduler.go
+++ b/src/k8s/pkg/k8sd/setup/kube_scheduler.go
@@ -6,10 +6,11 @@ import (
"github.com/canonical/k8s/pkg/snap"
snaputil "github.com/canonical/k8s/pkg/snap/util"
+ "github.com/canonical/k8s/pkg/utils"
)
// KubeScheduler configures kube-scheduler on the local node.
-func KubeScheduler(snap snap.Snap) error {
+func KubeScheduler(snap snap.Snap, extraArgs map[string]*string) error {
if _, err := snaputil.UpdateServiceArguments(snap, "kube-scheduler", map[string]string{
"--authentication-kubeconfig": path.Join(snap.KubernetesConfigDir(), "scheduler.conf"),
"--authorization-kubeconfig": path.Join(snap.KubernetesConfigDir(), "scheduler.conf"),
@@ -20,5 +21,10 @@ func KubeScheduler(snap snap.Snap) error {
}, nil); err != nil {
return fmt.Errorf("failed to render arguments file: %w", err)
}
+ // Apply extra arguments after the defaults, so they can override them.
+ updateArgs, deleteArgs := utils.ServiceArgsFromMap(extraArgs)
+ if _, err := snaputil.UpdateServiceArguments(snap, "kube-scheduler", updateArgs, deleteArgs); err != nil {
+ return fmt.Errorf("failed to write arguments file: %w", err)
+ }
return nil
}
diff --git a/src/k8s/pkg/k8sd/setup/kube_scheduler_test.go b/src/k8s/pkg/k8sd/setup/kube_scheduler_test.go
index f4e9f62b13..d478bd3733 100644
--- a/src/k8s/pkg/k8sd/setup/kube_scheduler_test.go
+++ b/src/k8s/pkg/k8sd/setup/kube_scheduler_test.go
@@ -26,7 +26,7 @@ func TestKubeScheduler(t *testing.T) {
s := mustSetupSnapAndDirectories(t, setKubeSchedulerMock)
// Call the kube scheduler setup function
- g.Expect(setup.KubeScheduler(s)).To(BeNil())
+ g.Expect(setup.KubeScheduler(s, nil)).To(BeNil())
// Ensure the kube scheduler arguments file has the expected arguments and values
tests := []struct {
@@ -56,6 +56,53 @@ func TestKubeScheduler(t *testing.T) {
})
+ t.Run("WithExtraArgs", func(t *testing.T) {
+ g := NewWithT(t)
+
+ // Create a mock snap
+ s := mustSetupSnapAndDirectories(t, setKubeSchedulerMock)
+
+ extraArgs := map[string]*string{
+ "--leader-elect-lease-duration": nil,
+ "--profiling": utils.Pointer("true"),
+ "--my-extra-arg": utils.Pointer("my-extra-val"),
+ }
+ // Call the kube scheduler setup function
+ g.Expect(setup.KubeScheduler(s, extraArgs)).To(BeNil())
+
+ // Ensure the kube scheduler arguments file has the expected arguments and values
+ tests := []struct {
+ key string
+ expectedVal string
+ }{
+ {key: "--authentication-kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "scheduler.conf")},
+ {key: "--authorization-kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "scheduler.conf")},
+ {key: "--kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "scheduler.conf")},
+ {key: "--leader-elect-renew-deadline", expectedVal: "15s"},
+ {key: "--profiling", expectedVal: "true"},
+ {key: "--my-extra-arg", expectedVal: "my-extra-val"},
+ }
+ for _, tc := range tests {
+ t.Run(tc.key, func(t *testing.T) {
+ g := NewWithT(t)
+ val, err := snaputil.GetServiceArgument(s, "kube-scheduler", tc.key)
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(tc.expectedVal).To(Equal(val))
+ })
+ }
+
+ // Ensure that the leader-elect-lease-duration argument was deleted
+ val, err := snaputil.GetServiceArgument(s, "kube-scheduler", "--leader-elect-lease-duration")
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(val).To(BeZero())
+
+ // Ensure the kube scheduler arguments file has exactly the expected number of arguments
+ args, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "kube-scheduler"))
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(len(args)).To(Equal(len(tests)))
+
+ })
+
t.Run("MissingArgsDir", func(t *testing.T) {
g := NewWithT(t)
@@ -63,6 +110,6 @@ func TestKubeScheduler(t *testing.T) {
s.Mock.ServiceArgumentsDir = "nonexistent"
- g.Expect(setup.KubeScheduler(s)).ToNot(Succeed())
+ g.Expect(setup.KubeScheduler(s, nil)).ToNot(Succeed())
})
}
diff --git a/src/k8s/pkg/k8sd/setup/kubelet.go b/src/k8s/pkg/k8sd/setup/kubelet.go
index e4f1db3d14..eec2e36a66 100644
--- a/src/k8s/pkg/k8sd/setup/kubelet.go
+++ b/src/k8s/pkg/k8sd/setup/kubelet.go
@@ -8,6 +8,7 @@ import (
"github.com/canonical/k8s/pkg/snap"
snaputil "github.com/canonical/k8s/pkg/snap/util"
+ "github.com/canonical/k8s/pkg/utils"
)
var kubeletTLSCipherSuites = []string{
@@ -30,17 +31,17 @@ var kubeletWorkerLabels = []string{
}
// KubeletControlPlane configures kubelet on a control plane node.
-func KubeletControlPlane(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string, clusterDomain string, cloudProvider string, registerWithTaints []string) error {
- return kubelet(snap, hostname, nodeIP, clusterDNS, clusterDomain, cloudProvider, registerWithTaints, append(kubeletControlPlaneLabels, kubeletWorkerLabels...))
+func KubeletControlPlane(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string, clusterDomain string, cloudProvider string, registerWithTaints []string, extraArgs map[string]*string) error {
+ return kubelet(snap, hostname, nodeIP, clusterDNS, clusterDomain, cloudProvider, registerWithTaints, append(kubeletControlPlaneLabels, kubeletWorkerLabels...), extraArgs)
}
// KubeletWorker configures kubelet on a worker node.
-func KubeletWorker(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string, clusterDomain string, cloudProvider string) error {
- return kubelet(snap, hostname, nodeIP, clusterDNS, clusterDomain, cloudProvider, nil, kubeletWorkerLabels)
+func KubeletWorker(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string, clusterDomain string, cloudProvider string, extraArgs map[string]*string) error {
+ return kubelet(snap, hostname, nodeIP, clusterDNS, clusterDomain, cloudProvider, nil, kubeletWorkerLabels, extraArgs)
}
// kubelet configures kubelet on the local node.
-func kubelet(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string, clusterDomain string, cloudProvider string, taints []string, labels []string) error {
+func kubelet(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string, clusterDomain string, cloudProvider string, taints []string, labels []string, extraArgs map[string]*string) error {
args := map[string]string{
"--anonymous-auth": "false",
"--authentication-token-webhook": "true",
@@ -75,5 +76,11 @@ func kubelet(snap snap.Snap, hostname string, nodeIP net.IP, clusterDNS string,
if _, err := snaputil.UpdateServiceArguments(snap, "kubelet", args, nil); err != nil {
return fmt.Errorf("failed to render arguments file: %w", err)
}
+
+ // Apply extra arguments after the defaults, so they can override them.
+ updateArgs, deleteArgs := utils.ServiceArgsFromMap(extraArgs)
+ if _, err := snaputil.UpdateServiceArguments(snap, "kubelet", updateArgs, deleteArgs); err != nil {
+ return fmt.Errorf("failed to write arguments file: %w", err)
+ }
return nil
}
diff --git a/src/k8s/pkg/k8sd/setup/kubelet_test.go b/src/k8s/pkg/k8sd/setup/kubelet_test.go
index 57d68f08cc..5145e7a78c 100644
--- a/src/k8s/pkg/k8sd/setup/kubelet_test.go
+++ b/src/k8s/pkg/k8sd/setup/kubelet_test.go
@@ -46,7 +46,7 @@ func TestKubelet(t *testing.T) {
s := mustSetupSnapAndDirectories(t, setKubeletMock)
// Call the kubelet control plane setup function
- g.Expect(setup.KubeletControlPlane(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider", nil)).To(Succeed())
+ g.Expect(setup.KubeletControlPlane(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider", nil, nil)).To(Succeed())
// Ensure the kubelet arguments file has the expected arguments and values
tests := []struct {
@@ -90,6 +90,67 @@ func TestKubelet(t *testing.T) {
g.Expect(len(args)).To(Equal(len(tests)))
})
+ t.Run("ControlPlaneWithExtraArgs", func(t *testing.T) {
+ g := NewWithT(t)
+
+ // Create a mock snap
+ s := mustSetupSnapAndDirectories(t, setKubeletMock)
+
+ extraArgs := map[string]*string{
+ "--cluster-domain": utils.Pointer("override.local"),
+ "--cloud-provider": nil, // This should trigger a delete
+ "--my-extra-arg": utils.Pointer("my-extra-val"),
+ }
+ // Call the kubelet control plane setup function
+ g.Expect(setup.KubeletControlPlane(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider", nil, extraArgs)).To(Succeed())
+
+ // Ensure the kubelet arguments file has the expected arguments and values
+ tests := []struct {
+ key string
+ expectedVal string
+ }{
+ {key: "--anonymous-auth", expectedVal: "false"},
+ {key: "--authentication-token-webhook", expectedVal: "true"},
+ {key: "--cert-dir", expectedVal: s.Mock.KubernetesPKIDir},
+ {key: "--client-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "client-ca.crt")},
+ {key: "--container-runtime-endpoint", expectedVal: path.Join(s.Mock.ContainerdSocketDir, "containerd.sock")},
+ {key: "--containerd", expectedVal: path.Join(s.Mock.ContainerdSocketDir, "containerd.sock")},
+ {key: "--eviction-hard", expectedVal: "'memory.available<100Mi,nodefs.available<1Gi,imagefs.available<1Gi'"},
+ {key: "--fail-swap-on", expectedVal: "false"},
+ {key: "--hostname-override", expectedVal: "dev"},
+ {key: "--kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "kubelet.conf")},
+ {key: "--node-labels", expectedVal: expectedControlPlaneLabels},
+ {key: "--read-only-port", expectedVal: "0"},
+ {key: "--register-with-taints", expectedVal: ""},
+ {key: "--root-dir", expectedVal: s.Mock.KubeletRootDir},
+ {key: "--serialize-image-pulls", expectedVal: "false"},
+ {key: "--tls-cipher-suites", expectedVal: kubeletTLSCipherSuites},
+ {key: "--cluster-dns", expectedVal: "10.152.1.1"},
+ // Overwritten by extraArgs
+ {key: "--cluster-domain", expectedVal: "override.local"},
+ {key: "--node-ip", expectedVal: "192.168.0.1"},
+ {key: "--my-extra-arg", expectedVal: "my-extra-val"},
+ }
+ for _, tc := range tests {
+ t.Run(tc.key, func(t *testing.T) {
+ g := NewWithT(t)
+ val, err := snaputil.GetServiceArgument(s, "kubelet", tc.key)
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(tc.expectedVal).To(Equal(val))
+ })
+ }
+
+ // Ensure that the cloud-provider argument was deleted
+ val, err := snaputil.GetServiceArgument(s, "kubelet", "--cloud-provider")
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(val).To(BeZero())
+
+ // Ensure the kubelet arguments file has exactly the expected number of arguments
+ args, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "kubelet"))
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(len(args)).To(Equal(len(tests)))
+ })
+
t.Run("ControlPlaneArgsNoOptional", func(t *testing.T) {
g := NewWithT(t)
@@ -97,7 +158,7 @@ func TestKubelet(t *testing.T) {
s := mustSetupSnapAndDirectories(t, setKubeletMock)
// Call the kubelet control plane setup function
- g.Expect(setup.KubeletControlPlane(s, "dev", nil, "", "", "", nil)).To(BeNil())
+ g.Expect(setup.KubeletControlPlane(s, "dev", nil, "", "", "", nil, nil)).To(BeNil())
tests := []struct {
key string
@@ -143,7 +204,7 @@ func TestKubelet(t *testing.T) {
s := mustSetupSnapAndDirectories(t, setKubeletMock)
// Call the kubelet worker setup function
- g.Expect(setup.KubeletWorker(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider")).To(BeNil())
+ g.Expect(setup.KubeletWorker(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider", nil)).To(BeNil())
// Ensure the kubelet arguments file has the expected arguments and values
tests := []struct {
@@ -187,6 +248,65 @@ func TestKubelet(t *testing.T) {
g.Expect(len(args)).To(Equal(len(tests)))
})
+ t.Run("WorkerWithExtraArgs", func(t *testing.T) {
+ g := NewWithT(t)
+
+ // Create a mock snap
+ s := mustSetupSnapAndDirectories(t, setKubeletMock)
+
+ extraArgs := map[string]*string{
+ "--cluster-domain": utils.Pointer("override.local"),
+ "--cloud-provider": nil,
+ }
+
+ // Call the kubelet worker setup function
+ g.Expect(setup.KubeletWorker(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider", extraArgs)).To(BeNil())
+
+ // Ensure the kubelet arguments file has the expected arguments and values
+ tests := []struct {
+ key string
+ expectedVal string
+ }{
+ {key: "--anonymous-auth", expectedVal: "false"},
+ {key: "--authentication-token-webhook", expectedVal: "true"},
+ {key: "--cert-dir", expectedVal: s.Mock.KubernetesPKIDir},
+ {key: "--client-ca-file", expectedVal: path.Join(s.Mock.KubernetesPKIDir, "client-ca.crt")},
+ {key: "--container-runtime-endpoint", expectedVal: path.Join(s.Mock.ContainerdSocketDir, "containerd.sock")},
+ {key: "--containerd", expectedVal: path.Join(s.Mock.ContainerdSocketDir, "containerd.sock")},
+ {key: "--eviction-hard", expectedVal: "'memory.available<100Mi,nodefs.available<1Gi,imagefs.available<1Gi'"},
+ {key: "--fail-swap-on", expectedVal: "false"},
+ {key: "--hostname-override", expectedVal: "dev"},
+ {key: "--kubeconfig", expectedVal: path.Join(s.Mock.KubernetesConfigDir, "kubelet.conf")},
+ {key: "--node-labels", expectedVal: expectedWorkerLabels},
+ {key: "--read-only-port", expectedVal: "0"},
+ {key: "--register-with-taints", expectedVal: ""},
+ {key: "--root-dir", expectedVal: s.Mock.KubeletRootDir},
+ {key: "--serialize-image-pulls", expectedVal: "false"},
+ {key: "--tls-cipher-suites", expectedVal: kubeletTLSCipherSuites},
+ {key: "--cluster-dns", expectedVal: "10.152.1.1"},
+ {key: "--cluster-domain", expectedVal: "override.local"},
+ {key: "--node-ip", expectedVal: "192.168.0.1"},
+ }
+ for _, tc := range tests {
+ t.Run(tc.key, func(t *testing.T) {
+ g := NewWithT(t)
+ val, err := snaputil.GetServiceArgument(s, "kubelet", tc.key)
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(tc.expectedVal).To(Equal(val))
+ })
+ }
+
+ // Ensure that the cloud-provider argument was deleted
+ val, err := snaputil.GetServiceArgument(s, "kubelet", "--cloud-provider")
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(val).To(BeZero())
+
+ // Ensure the kubelet arguments file has exactly the expected number of arguments
+ args, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "kubelet"))
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(len(args)).To(Equal(len(tests)))
+ })
+
t.Run("WorkerArgsNoOptional", func(t *testing.T) {
g := NewWithT(t)
@@ -194,7 +314,7 @@ func TestKubelet(t *testing.T) {
s := mustSetupSnapAndDirectories(t, setKubeletMock)
// Call the kubelet worker setup function
- g.Expect(setup.KubeletWorker(s, "dev", nil, "", "", "")).To(BeNil())
+ g.Expect(setup.KubeletWorker(s, "dev", nil, "", "", "", nil)).To(BeNil())
// Ensure the kubelet arguments file has the expected arguments and values
tests := []struct {
@@ -240,7 +360,7 @@ func TestKubelet(t *testing.T) {
s.Mock.ServiceArgumentsDir = "nonexistent"
- g.Expect(setup.KubeletControlPlane(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider", nil)).ToNot(Succeed())
+ g.Expect(setup.KubeletControlPlane(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider", nil, nil)).ToNot(Succeed())
})
t.Run("WorkerNoArgsDir", func(t *testing.T) {
@@ -249,6 +369,6 @@ func TestKubelet(t *testing.T) {
s.Mock.ServiceArgumentsDir = "nonexistent"
- g.Expect(setup.KubeletWorker(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider")).ToNot(Succeed())
+ g.Expect(setup.KubeletWorker(s, "dev", net.ParseIP("192.168.0.1"), "10.152.1.1", "test-cluster.local", "provider", nil)).ToNot(Succeed())
})
}
diff --git a/src/k8s/pkg/k8sd/setup/util_extra_files.go b/src/k8s/pkg/k8sd/setup/util_extra_files.go
new file mode 100644
index 0000000000..e6fef01626
--- /dev/null
+++ b/src/k8s/pkg/k8sd/setup/util_extra_files.go
@@ -0,0 +1,33 @@
+package setup
+
+import (
+ "fmt"
+ "os"
+ "path"
+ "strings"
+
+ "github.com/canonical/k8s/pkg/snap"
+)
+
+// ExtraNodeConfigFiles writes the file contents to the specified filenames in the snap.ExtraFilesDir directory.
+// The files are created with 0400 permissions and owned by root.
+// The filenames must not contain any slashes to prevent path traversal.
+func ExtraNodeConfigFiles(snap snap.Snap, files map[string]string) error {
+ for filename, content := range files {
+ if strings.Contains(filename, "/") {
+ return fmt.Errorf("file name %q must not contain any slashes (possible path-traversal prevented)", filename)
+ }
+
+ filePath := path.Join(snap.ServiceExtraConfigDir(), filename)
+ // Write the content to the file
+ if err := os.WriteFile(filePath, []byte(content), 0400); err != nil {
+ return fmt.Errorf("failed to write to file %s: %w", filePath, err)
+ }
+
+ // Set file owner to root
+ if err := os.Chown(filePath, snap.UID(), snap.GID()); err != nil {
+ return fmt.Errorf("failed to change owner of file %s: %w", filePath, err)
+ }
+ }
+ return nil
+}
diff --git a/src/k8s/pkg/k8sd/setup/util_extra_files_test.go b/src/k8s/pkg/k8sd/setup/util_extra_files_test.go
new file mode 100644
index 0000000000..ad4fa38bb4
--- /dev/null
+++ b/src/k8s/pkg/k8sd/setup/util_extra_files_test.go
@@ -0,0 +1,73 @@
+package setup
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/canonical/k8s/pkg/snap/mock"
+ "github.com/onsi/gomega"
+)
+
+func TestExtraNodeConfigFiles(t *testing.T) {
+ tests := []struct {
+ name string
+ files map[string]string
+ expectErr bool
+ errMessage string
+ }{
+ {
+ name: "ValidFiles",
+ files: map[string]string{
+ "config1": "content1",
+ "config2": "content2",
+ },
+ expectErr: false,
+ },
+ {
+ name: "InvalidFilename",
+ files: map[string]string{
+ "invalid/config": "content",
+ },
+ expectErr: true,
+ errMessage: "file name \"invalid/config\" must not contain any slashes",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := gomega.NewGomegaWithT(t)
+
+ tmpDir := t.TempDir()
+ snap := &mock.Snap{
+ Mock: mock.Mock{
+ ServiceExtraConfigDir: tmpDir,
+ UID: os.Getuid(),
+ GID: os.Getgid(),
+ },
+ }
+
+ err := ExtraNodeConfigFiles(snap, tt.files)
+ if tt.expectErr {
+ g.Expect(err).To(gomega.HaveOccurred())
+ g.Expect(err.Error()).To(gomega.ContainSubstring(tt.errMessage))
+ } else {
+ g.Expect(err).ToNot(gomega.HaveOccurred())
+
+ for filename, content := range tt.files {
+ filePath := filepath.Join(tmpDir, filename)
+
+ // Verify the file exists
+ info, err := os.Stat(filePath)
+ g.Expect(err).ToNot(gomega.HaveOccurred())
+ g.Expect(info.Mode().Perm()).To(gomega.Equal(os.FileMode(0400)))
+
+ // Verify the file content
+ actualContent, err := os.ReadFile(filePath)
+ g.Expect(err).ToNot(gomega.HaveOccurred())
+ g.Expect(string(actualContent)).To(gomega.Equal(content))
+ }
+ }
+ })
+ }
+}
diff --git a/src/k8s/pkg/snap/util/services.go b/src/k8s/pkg/snap/util/services.go
index 56be97bf21..d7587207e0 100644
--- a/src/k8s/pkg/snap/util/services.go
+++ b/src/k8s/pkg/snap/util/services.go
@@ -75,3 +75,20 @@ func StopK8sDqliteServices(ctx context.Context, snap snap.Snap) error {
}
return nil
}
+
+// ServiceArgsFromMap processes a map of string pointers and categorizes them into update and delete lists.
+// - If the value pointer is nil, it adds the argument name to the delete list.
+// - If the value pointer is not nil, it adds the argument and its value to the update map.
+func ServiceArgsFromMap(args map[string]*string) (map[string]string, []string) {
+ updateArgs := make(map[string]string)
+ deleteArgs := make([]string, 0)
+
+ for arg, val := range args {
+ if val == nil {
+ deleteArgs = append(deleteArgs, arg)
+ } else {
+ updateArgs[arg] = *val
+ }
+ }
+ return updateArgs, deleteArgs
+}
diff --git a/src/k8s/pkg/snap/util/services_test.go b/src/k8s/pkg/snap/util/services_test.go
index 203012d819..70ae0bc751 100644
--- a/src/k8s/pkg/snap/util/services_test.go
+++ b/src/k8s/pkg/snap/util/services_test.go
@@ -6,6 +6,7 @@ import (
"testing"
"github.com/canonical/k8s/pkg/snap/mock"
+ "github.com/canonical/k8s/pkg/utils"
. "github.com/onsi/gomega"
)
@@ -108,3 +109,73 @@ func TestStopK8sDqliteServices(t *testing.T) {
g.Expect(StopK8sDqliteServices(context.Background(), mock)).NotTo(Succeed())
})
}
+
+func TestServiceArgsFromMap(t *testing.T) {
+ tests := []struct {
+ name string
+ input map[string]*string
+ expected struct {
+ updateArgs map[string]string
+ deleteArgs []string
+ }
+ }{
+ {
+ name: "NilValue",
+ input: map[string]*string{"arg1": nil},
+ expected: struct {
+ updateArgs map[string]string
+ deleteArgs []string
+ }{
+ updateArgs: map[string]string{},
+ deleteArgs: []string{"arg1"},
+ },
+ },
+ {
+ name: "EmptyString", // Should be threated as normal string
+ input: map[string]*string{"arg1": utils.Pointer("")},
+ expected: struct {
+ updateArgs map[string]string
+ deleteArgs []string
+ }{
+ updateArgs: map[string]string{"arg1": ""},
+ deleteArgs: []string{},
+ },
+ },
+ {
+ name: "NonEmptyString",
+ input: map[string]*string{"arg1": utils.Pointer("value1")},
+ expected: struct {
+ updateArgs map[string]string
+ deleteArgs []string
+ }{
+ updateArgs: map[string]string{"arg1": "value1"},
+ deleteArgs: []string{},
+ },
+ },
+ {
+ name: "MixedValues",
+ input: map[string]*string{
+ "arg1": utils.Pointer("value1"),
+ "arg2": utils.Pointer(""),
+ "arg3": nil,
+ },
+ expected: struct {
+ updateArgs map[string]string
+ deleteArgs []string
+ }{
+ updateArgs: map[string]string{"arg1": "value1", "arg2": ""},
+ deleteArgs: []string{"arg3"},
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ updateArgs, deleteArgs := ServiceArgsFromMap(tt.input)
+ g.Expect(updateArgs).To(Equal(tt.expected.updateArgs))
+ g.Expect(deleteArgs).To(Equal(tt.expected.deleteArgs))
+ })
+ }
+}
diff --git a/src/k8s/pkg/utils/services.go b/src/k8s/pkg/utils/services.go
new file mode 100644
index 0000000000..596ee54254
--- /dev/null
+++ b/src/k8s/pkg/utils/services.go
@@ -0,0 +1,18 @@
+package utils
+
+// ServiceArgsFromMap processes a map of string pointers and categorizes them into update and delete lists.
+// - If the value pointer is nil, it adds the argument name to the delete list.
+// - If the value pointer is not nil, it adds the argument and its value to the update map.
+func ServiceArgsFromMap(args map[string]*string) (map[string]string, []string) {
+ updateArgs := make(map[string]string)
+ deleteArgs := make([]string, 0)
+
+ for arg, val := range args {
+ if val == nil {
+ deleteArgs = append(deleteArgs, arg)
+ } else {
+ updateArgs[arg] = *val
+ }
+ }
+ return updateArgs, deleteArgs
+}
diff --git a/src/k8s/pkg/utils/services_test.go b/src/k8s/pkg/utils/services_test.go
new file mode 100644
index 0000000000..8a18cbfc0c
--- /dev/null
+++ b/src/k8s/pkg/utils/services_test.go
@@ -0,0 +1,77 @@
+package utils
+
+import (
+ "testing"
+
+ . "github.com/onsi/gomega"
+)
+
+func TestServiceArgsFromMap(t *testing.T) {
+ tests := []struct {
+ name string
+ input map[string]*string
+ expected struct {
+ updateArgs map[string]string
+ deleteArgs []string
+ }
+ }{
+ {
+ name: "NilValue",
+ input: map[string]*string{"arg1": nil},
+ expected: struct {
+ updateArgs map[string]string
+ deleteArgs []string
+ }{
+ updateArgs: map[string]string{},
+ deleteArgs: []string{"arg1"},
+ },
+ },
+ {
+ name: "EmptyString", // Should be threated as normal string
+ input: map[string]*string{"arg1": Pointer("")},
+ expected: struct {
+ updateArgs map[string]string
+ deleteArgs []string
+ }{
+ updateArgs: map[string]string{"arg1": ""},
+ deleteArgs: []string{},
+ },
+ },
+ {
+ name: "NonEmptyString",
+ input: map[string]*string{"arg1": Pointer("value1")},
+ expected: struct {
+ updateArgs map[string]string
+ deleteArgs []string
+ }{
+ updateArgs: map[string]string{"arg1": "value1"},
+ deleteArgs: []string{},
+ },
+ },
+ {
+ name: "MixedValues",
+ input: map[string]*string{
+ "arg1": Pointer("value1"),
+ "arg2": Pointer(""),
+ "arg3": nil,
+ },
+ expected: struct {
+ updateArgs map[string]string
+ deleteArgs []string
+ }{
+ updateArgs: map[string]string{"arg1": "value1", "arg2": ""},
+ deleteArgs: []string{"arg3"},
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ g := NewWithT(t)
+
+ updateArgs, deleteArgs := ServiceArgsFromMap(tt.input)
+ g.Expect(updateArgs).To(Equal(tt.expected.updateArgs))
+ g.Expect(deleteArgs).To(Equal(tt.expected.deleteArgs))
+ })
+ }
+}
diff --git a/tests/integration/templates/bootstrap-all.yaml b/tests/integration/templates/bootstrap-session.yaml
similarity index 55%
rename from tests/integration/templates/bootstrap-all.yaml
rename to tests/integration/templates/bootstrap-session.yaml
index 68ad1fbb43..bdeebaecee 100644
--- a/tests/integration/templates/bootstrap-all.yaml
+++ b/tests/integration/templates/bootstrap-session.yaml
@@ -1,3 +1,5 @@
+# Contains the bootstrap configuration for the session instance of the integration tests.
+# The session instance persists over test runs and is used to speed-up the integration tests.
cluster-config:
network:
enabled: true
diff --git a/tests/integration/templates/bootstrap-smoke.yaml b/tests/integration/templates/bootstrap-smoke.yaml
new file mode 100644
index 0000000000..220a43c491
--- /dev/null
+++ b/tests/integration/templates/bootstrap-smoke.yaml
@@ -0,0 +1,32 @@
+# Contains the bootstrap configuration for the smoke test.
+cluster-config:
+ network:
+ enabled: true
+ dns:
+ enabled: true
+ ingress:
+ enabled: true
+ load-balancer:
+ enabled: true
+ local-storage:
+ enabled: true
+ gateway:
+ enabled: true
+ metrics-server:
+ enabled: true
+extra-node-config-files:
+ bootstrap-extra-file.yaml: extra-args-test-file-content
+extra-node-kube-apiserver-args:
+ --request-timeout: 2m
+extra-node-kube-controller-manager-args:
+ --leader-elect-retry-period: 3s
+extra-node-kube-scheduler-args:
+ --authorization-webhook-cache-authorized-ttl: 11s
+extra-node-kube-proxy-args:
+ --config-sync-period: 14m
+extra-node-kubelet-args:
+ --authentication-token-webhook-cache-ttl: 3m
+extra-node-containerd-args:
+ --log-level: debug
+extra-node-k8s-dqlite-args:
+ --watch-storage-available-size-interval: 6s
diff --git a/tests/integration/tests/conftest.py b/tests/integration/tests/conftest.py
index 49926c3cd0..309cf982ba 100644
--- a/tests/integration/tests/conftest.py
+++ b/tests/integration/tests/conftest.py
@@ -126,9 +126,10 @@ def session_instance(
instance = h.new_instance()
util.setup_k8s_snap(instance, snap_path)
- bootstrap_config_path = "/home/ubuntu/bootstrap-all-features.yaml"
+ bootstrap_config_path = "/home/ubuntu/bootstrap-session.yaml"
instance.send_file(
- (config.MANIFESTS_DIR / "bootstrap-all.yaml").as_posix(), bootstrap_config_path
+ (config.MANIFESTS_DIR / "bootstrap-session.yaml").as_posix(),
+ bootstrap_config_path,
)
instance.exec(["k8s", "bootstrap", "--file", bootstrap_config_path])
diff --git a/tests/integration/tests/test_smoke.py b/tests/integration/tests/test_smoke.py
index 2e424904b4..7ff72158c0 100644
--- a/tests/integration/tests/test_smoke.py
+++ b/tests/integration/tests/test_smoke.py
@@ -2,18 +2,55 @@
# Copyright 2024 Canonical, Ltd.
#
import logging
+from typing import List
-from test_util import harness
+import pytest
+from test_util import config, harness, util
LOG = logging.getLogger(__name__)
-def test_smoke(session_instance: harness.Instance):
+@pytest.mark.node_count(1)
+@pytest.mark.disable_k8s_bootstrapping()
+def test_smoke(instances: List[harness.Instance]):
+ instance = instances[0]
+
+ bootstrap_smoke_config_path = "/home/ubuntu/bootstrap-smoke.yaml"
+ instance.send_file(
+ (config.MANIFESTS_DIR / "bootstrap-smoke.yaml").as_posix(),
+ bootstrap_smoke_config_path,
+ )
+
+ instance.exec(["k8s", "bootstrap", "--file", bootstrap_smoke_config_path])
+ util.wait_until_k8s_ready(instance, [instance])
+
# Verify the functionality of the k8s config command during the smoke test.
# It would be excessive to deploy a cluster solely for this purpose.
- result = session_instance.exec(
+ result = instance.exec(
"k8s config --server 192.168.210.41".split(), capture_output=True
)
- config = result.stdout.decode()
- assert len(config) > 0
- assert "server: https://192.168.210.41" in config
+ kubeconfig = result.stdout.decode()
+ assert len(kubeconfig) > 0
+ assert "server: https://192.168.210.41" in kubeconfig
+
+ # Verify extra node configs
+ content = instance.exec(
+ ["cat", "/var/snap/k8s/common/args/conf.d/bootstrap-extra-file.yaml"],
+ capture_output=True,
+ )
+ assert content.stdout.decode() == "extra-args-test-file-content"
+
+ # For each service, verify that the extra arg was written to the args file.
+ for service, value in {
+ "kube-apiserver": "--request-timeout=2m",
+ "kube-controller-manager": "--leader-elect-retry-period=3s",
+ "kube-scheduler": "--authorization-webhook-cache-authorized-ttl=11s",
+ "kube-proxy": "--config-sync-period=14m",
+ "kubelet": "--authentication-token-webhook-cache-ttl=3m",
+ "containerd": "--log-level=debug",
+ "k8s-dqlite": "--watch-storage-available-size-interval=6s",
+ }.items():
+ args = instance.exec(
+ ["cat", f"/var/snap/k8s/common/args/{service}"], capture_output=True
+ )
+ assert value in args.stdout.decode()