diff --git a/src/k8s/pkg/k8sd/app/cluster_util.go b/src/k8s/pkg/k8sd/app/cluster_util.go index 50aa0538a..e8526d953 100644 --- a/src/k8s/pkg/k8sd/app/cluster_util.go +++ b/src/k8s/pkg/k8sd/app/cluster_util.go @@ -57,7 +57,7 @@ func setupControlPlaneServices(snap snap.Snap, s *state.State, cfg types.Cluster if err := setup.KubeScheduler(snap); err != nil { return fmt.Errorf("failed to configure kube-scheduler: %w", err) } - if err := setup.KubeAPIServer(snap, cfg.Network.GetServiceCIDR(), s.Address().Path("1.0", "kubernetes", "auth", "webhook").String(), true, cfg.Datastore.GetType(), cfg.Datastore.GetExternalURL(), cfg.APIServer.GetAuthorizationMode()); err != nil { + if err := setup.KubeAPIServer(snap, cfg.Network.GetServiceCIDR(), s.Address().Path("1.0", "kubernetes", "auth", "webhook").String(), true, cfg.Datastore, cfg.APIServer.GetAuthorizationMode()); err != nil { return fmt.Errorf("failed to configure kube-apiserver: %w", err) } return nil diff --git a/src/k8s/pkg/k8sd/setup/kube_apiserver.go b/src/k8s/pkg/k8sd/setup/kube_apiserver.go index 403106636..9d7fa4c21 100644 --- a/src/k8s/pkg/k8sd/setup/kube_apiserver.go +++ b/src/k8s/pkg/k8sd/setup/kube_apiserver.go @@ -6,6 +6,7 @@ import ( "path" "strings" + "github.com/canonical/k8s/pkg/k8sd/types" "github.com/canonical/k8s/pkg/snap" snaputil "github.com/canonical/k8s/pkg/snap/util" ) @@ -45,7 +46,7 @@ var ( ) // KubeAPIServer configures kube-apiserver on the local node. -func KubeAPIServer(snap snap.Snap, serviceCIDR string, authWebhookURL string, enableFrontProxy bool, datastore string, externalDatastoreURL string, authorizationMode string) error { +func KubeAPIServer(snap snap.Snap, serviceCIDR string, authWebhookURL string, enableFrontProxy bool, datastore types.Datastore, authorizationMode string) error { authTokenWebhookConfigFile := path.Join(snap.ServiceExtraConfigDir(), "auth-token-webhook.conf") authTokenWebhookFile, err := os.OpenFile(authTokenWebhookConfigFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { @@ -78,20 +79,15 @@ func KubeAPIServer(snap snap.Snap, serviceCIDR string, authWebhookURL string, en "--tls-private-key-file": path.Join(snap.KubernetesPKIDir(), "apiserver.key"), } - switch datastore { - case "k8s-dqlite": - args["--etcd-servers"] = fmt.Sprintf("unix://%s", path.Join(snap.K8sDqliteStateDir(), "k8s-dqlite.sock")) - case "external": - args["--etcd-servers"] = externalDatastoreURL - if _, err := os.Stat(path.Join(snap.EtcdPKIDir(), "ca.crt")); err == nil { - args["--etcd-cafile"] = path.Join(snap.EtcdPKIDir(), "ca.crt") - } - if _, err := os.Stat(path.Join(snap.EtcdPKIDir(), "client.key")); err == nil { - args["--etcd-keyfile"] = path.Join(snap.EtcdPKIDir(), "client.key") - args["--etcd-certfile"] = path.Join(snap.EtcdPKIDir(), "client.crt") - } + switch datastore.GetType() { + case "k8s-dqlite", "external": default: - return fmt.Errorf("unsupported datastore %s, must be one of %v", datastore, SupportedDatastores) + return fmt.Errorf("unsupported datastore %s, must be one of %v", datastore.GetType(), SupportedDatastores) + } + + datastoreUpdateArgs, deleteArgs := datastore.ToKubeAPIServerArguments(snap) + for key, val := range datastoreUpdateArgs { + args[key] = val } if enableFrontProxy { @@ -103,7 +99,7 @@ func KubeAPIServer(snap snap.Snap, serviceCIDR string, authWebhookURL string, en args["--proxy-client-cert-file"] = path.Join(snap.KubernetesPKIDir(), "front-proxy-client.crt") args["--proxy-client-key-file"] = path.Join(snap.KubernetesPKIDir(), "front-proxy-client.key") } - if _, err := snaputil.UpdateServiceArguments(snap, "kube-apiserver", args, nil); err != nil { + if _, err := snaputil.UpdateServiceArguments(snap, "kube-apiserver", args, deleteArgs); err != nil { return fmt.Errorf("failed to render arguments file: %w", err) } return nil diff --git a/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go b/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go index b6ec3516e..17ed02a94 100644 --- a/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go +++ b/src/k8s/pkg/k8sd/setup/kube_apiserver_test.go @@ -7,9 +7,11 @@ import ( "testing" "github.com/canonical/k8s/pkg/k8sd/setup" + "github.com/canonical/k8s/pkg/k8sd/types" "github.com/canonical/k8s/pkg/snap/mock" snaputil "github.com/canonical/k8s/pkg/snap/util" "github.com/canonical/k8s/pkg/utils" + "github.com/canonical/k8s/pkg/utils/vals" . "github.com/onsi/gomega" ) @@ -35,7 +37,7 @@ func TestKubeAPIServer(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock) // Call the KubeAPIServer setup function with mock arguments - g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", true, "k8s-dqlite", "datastoreurl", "Node,RBAC")).To(BeNil()) + g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", true, types.Datastore{Type: vals.Pointer("k8s-dqlite")}, "Node,RBAC")).To(BeNil()) // Ensure the kube-apiserver arguments file has the expected arguments and values tests := []struct { @@ -90,7 +92,7 @@ func TestKubeAPIServer(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock) // Call the KubeAPIServer setup function with mock arguments - g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, "k8s-dqlite", "datastoreurl", "Node,RBAC")).To(BeNil()) + g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: vals.Pointer("k8s-dqlite")}, "Node,RBAC")).To(BeNil()) // Ensure the kube-apiserver arguments file has the expected arguments and values tests := []struct { @@ -137,7 +139,7 @@ func TestKubeAPIServer(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock) // Setup without proxy to simplify argument list - g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, "external", "datastoreurl", "Node,RBAC")).To(BeNil()) + g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: vals.Pointer("external"), ExternalURL: vals.Pointer("datastoreurl")}, "Node,RBAC")).To(BeNil()) g.Expect(snaputil.GetServiceArgument(s, "kube-apiserver", "--etcd-servers")).To(Equal("datastoreurl")) _, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "kube-apiserver")) @@ -151,7 +153,7 @@ func TestKubeAPIServer(t *testing.T) { s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock) // Attempt to configure kube-apiserver with an unsupported datastore - err := setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, "unsupported-datastore", "datastoreurl", "Node,RBAC") + err := setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: vals.Pointer("unsupported")}, "Node,RBAC") g.Expect(err).To(HaveOccurred()) g.Expect(err).To(MatchError(ContainSubstring("unsupported datastore"))) }) diff --git a/src/k8s/pkg/k8sd/types/cluster_config_datastore.go b/src/k8s/pkg/k8sd/types/cluster_config_datastore.go index 597f8da51..f90617a3d 100644 --- a/src/k8s/pkg/k8sd/types/cluster_config_datastore.go +++ b/src/k8s/pkg/k8sd/types/cluster_config_datastore.go @@ -1,5 +1,10 @@ package types +import ( + "fmt" + "path" +) + type Datastore struct { Type *string `json:"type,omitempty"` @@ -24,3 +29,45 @@ func (c Datastore) GetExternalClientKey() string { return getField(c.ExternalCl func (c Datastore) Empty() bool { return c.Type == nil && c.K8sDqlitePort == nil && c.K8sDqliteCert == nil && c.K8sDqliteKey == nil && c.ExternalURL == nil && c.ExternalCACert == nil && c.ExternalClientCert == nil && c.ExternalClientKey == nil } + +// DatastorePathsProvider is to avoid circular dependency for snap.Snap in Datastore.ToKubeAPIServerArguments() +type DatastorePathsProvider interface { + K8sDqliteStateDir() string + EtcdPKIDir() string +} + +// ToKubeAPIServerArguments returns updateArgs, deleteArgs that can be used with snaputil.UpdateServiceArguments() for the kube-apiserver +// according the datastore configuration. +func (c Datastore) ToKubeAPIServerArguments(p DatastorePathsProvider) (map[string]string, []string) { + var ( + updateArgs = make(map[string]string) + deleteArgs []string + ) + + switch c.GetType() { + case "k8s-dqlite": + updateArgs["--etcd-servers"] = fmt.Sprintf("unix://%s", path.Join(p.K8sDqliteStateDir(), "k8s-dqlite.sock")) + deleteArgs = []string{"--etcd-cafile", "--etcd-certfile", "--etcd-keyfile"} + case "external": + updateArgs["--etcd-servers"] = c.GetExternalURL() + + // the certificates will be written by setup.EnsureExtDatastorePKI(), here we only set the paths + for _, loop := range []struct { + arg string + cert string + path string + }{ + {cert: c.GetExternalCACert(), arg: "--etcd-cafile", path: "ca.crt"}, + {cert: c.GetExternalClientCert(), arg: "--etcd-certfile", path: "client.crt"}, + {cert: c.GetExternalClientKey(), arg: "--etcd-keyfile", path: "client.key"}, + } { + if loop.cert != "" { + updateArgs[loop.arg] = path.Join(p.EtcdPKIDir(), loop.path) + } else { + deleteArgs = append(deleteArgs, loop.arg) + } + } + } + + return updateArgs, deleteArgs +} diff --git a/src/k8s/pkg/k8sd/types/cluster_config_datastore_test.go b/src/k8s/pkg/k8sd/types/cluster_config_datastore_test.go new file mode 100644 index 000000000..3f402a030 --- /dev/null +++ b/src/k8s/pkg/k8sd/types/cluster_config_datastore_test.go @@ -0,0 +1,78 @@ +package types_test + +import ( + "testing" + + "github.com/canonical/k8s/pkg/k8sd/types" + "github.com/canonical/k8s/pkg/snap/mock" + "github.com/canonical/k8s/pkg/utils/vals" + . "github.com/onsi/gomega" +) + +func TestDatastoreToKubeAPIServerArguments(t *testing.T) { + snap := &mock.Snap{ + Mock: mock.Mock{ + K8sDqliteStateDir: "/k8s-dqlite", + EtcdPKIDir: "/pki/etcd", + }, + } + + for _, tc := range []struct { + name string + config types.Datastore + expectUpdateArgs map[string]string + expectDeleteArgs []string + }{ + { + name: "Nil", + expectUpdateArgs: map[string]string{}, + }, + { + name: "K8sDqlite", + config: types.Datastore{ + Type: vals.Pointer("k8s-dqlite"), + }, + expectUpdateArgs: map[string]string{ + "--etcd-servers": "unix:///k8s-dqlite/k8s-dqlite.sock", + }, + expectDeleteArgs: []string{"--etcd-cafile", "--etcd-certfile", "--etcd-keyfile"}, + }, + { + name: "ExternalFull", + config: types.Datastore{ + Type: vals.Pointer("external"), + ExternalURL: vals.Pointer("https://10.0.0.10:2379,https://10.0.0.11:2379"), + ExternalCACert: vals.Pointer("data"), + ExternalClientCert: vals.Pointer("data"), + ExternalClientKey: vals.Pointer("data"), + }, + expectUpdateArgs: map[string]string{ + "--etcd-servers": "https://10.0.0.10:2379,https://10.0.0.11:2379", + "--etcd-cafile": "/pki/etcd/ca.crt", + "--etcd-certfile": "/pki/etcd/client.crt", + "--etcd-keyfile": "/pki/etcd/client.key", + }, + }, + { + name: "ExternalOnlyCA", + config: types.Datastore{ + Type: vals.Pointer("external"), + ExternalURL: vals.Pointer("https://10.0.0.10:2379,https://10.0.0.11:2379"), + ExternalCACert: vals.Pointer("data"), + }, + expectUpdateArgs: map[string]string{ + "--etcd-servers": "https://10.0.0.10:2379,https://10.0.0.11:2379", + "--etcd-cafile": "/pki/etcd/ca.crt", + }, + expectDeleteArgs: []string{"--etcd-certfile", "--etcd-keyfile"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + update, delete := tc.config.ToKubeAPIServerArguments(snap) + g.Expect(update).To(Equal(tc.expectUpdateArgs)) + g.Expect(delete).To(Equal(tc.expectDeleteArgs)) + }) + } +}