Add (Datastore).ToKubeAPIServerArguments() helper (#296)

* Add (types.Datastore).ToKubeAPIServerArguments() helper
* Use (Datastore).ToKubeAPIServerArguments() when configuring kube-apiserver

src/k8s/pkg/k8sd/app/cluster_util.go

@@ -57,7 +57,7 @@ func setupControlPlaneServices(snap snap.Snap, s *state.State, cfg types.Cluster
 	if err := setup.KubeScheduler(snap); err != nil {
 		return fmt.Errorf("failed to configure kube-scheduler: %w", err)
 	}
-	if err := setup.KubeAPIServer(snap, cfg.Network.GetServiceCIDR(), s.Address().Path("1.0", "kubernetes", "auth", "webhook").String(), true, cfg.Datastore.GetType(), cfg.Datastore.GetExternalURL(), cfg.APIServer.GetAuthorizationMode()); err != nil {
+	if err := setup.KubeAPIServer(snap, cfg.Network.GetServiceCIDR(), s.Address().Path("1.0", "kubernetes", "auth", "webhook").String(), true, cfg.Datastore, cfg.APIServer.GetAuthorizationMode()); err != nil {
 		return fmt.Errorf("failed to configure kube-apiserver: %w", err)
 	}
 	return nil

src/k8s/pkg/k8sd/setup/kube_apiserver.go

@@ -6,6 +6,7 @@ import (
 	"path"
 	"strings"
 
+	"github.com/canonical/k8s/pkg/k8sd/types"
 	"github.com/canonical/k8s/pkg/snap"
 	snaputil "github.com/canonical/k8s/pkg/snap/util"
 )
@@ -45,7 +46,7 @@ var (
 )
 
 // KubeAPIServer configures kube-apiserver on the local node.
-func KubeAPIServer(snap snap.Snap, serviceCIDR string, authWebhookURL string, enableFrontProxy bool, datastore string, externalDatastoreURL string, authorizationMode string) error {
+func KubeAPIServer(snap snap.Snap, serviceCIDR string, authWebhookURL string, enableFrontProxy bool, datastore types.Datastore, authorizationMode string) error {
 	authTokenWebhookConfigFile := path.Join(snap.ServiceExtraConfigDir(), "auth-token-webhook.conf")
 	authTokenWebhookFile, err := os.OpenFile(authTokenWebhookConfigFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
 	if err != nil {
@@ -78,20 +79,15 @@ func KubeAPIServer(snap snap.Snap, serviceCIDR string, authWebhookURL string, en
 		"--tls-private-key-file":                     path.Join(snap.KubernetesPKIDir(), "apiserver.key"),
 	}
 
-	switch datastore {
-	case "k8s-dqlite":
-		args["--etcd-servers"] = fmt.Sprintf("unix://%s", path.Join(snap.K8sDqliteStateDir(), "k8s-dqlite.sock"))
-	case "external":
-		args["--etcd-servers"] = externalDatastoreURL
-		if _, err := os.Stat(path.Join(snap.EtcdPKIDir(), "ca.crt")); err == nil {
-			args["--etcd-cafile"] = path.Join(snap.EtcdPKIDir(), "ca.crt")
-		}
-		if _, err := os.Stat(path.Join(snap.EtcdPKIDir(), "client.key")); err == nil {
-			args["--etcd-keyfile"] = path.Join(snap.EtcdPKIDir(), "client.key")
-			args["--etcd-certfile"] = path.Join(snap.EtcdPKIDir(), "client.crt")
-		}
+	switch datastore.GetType() {
+	case "k8s-dqlite", "external":
 	default:
-		return fmt.Errorf("unsupported datastore %s, must be one of %v", datastore, SupportedDatastores)
+		return fmt.Errorf("unsupported datastore %s, must be one of %v", datastore.GetType(), SupportedDatastores)
+	}
+
+	datastoreUpdateArgs, deleteArgs := datastore.ToKubeAPIServerArguments(snap)
+	for key, val := range datastoreUpdateArgs {
+		args[key] = val
 	}
 
 	if enableFrontProxy {
@@ -103,7 +99,7 @@ func KubeAPIServer(snap snap.Snap, serviceCIDR string, authWebhookURL string, en
 		args["--proxy-client-cert-file"] = path.Join(snap.KubernetesPKIDir(), "front-proxy-client.crt")
 		args["--proxy-client-key-file"] = path.Join(snap.KubernetesPKIDir(), "front-proxy-client.key")
 	}
-	if _, err := snaputil.UpdateServiceArguments(snap, "kube-apiserver", args, nil); err != nil {
+	if _, err := snaputil.UpdateServiceArguments(snap, "kube-apiserver", args, deleteArgs); err != nil {
 		return fmt.Errorf("failed to render arguments file: %w", err)
 	}
 	return nil

src/k8s/pkg/k8sd/setup/kube_apiserver_test.go

@@ -7,9 +7,11 @@ import (
 	"testing"
 
 	"github.com/canonical/k8s/pkg/k8sd/setup"
+	"github.com/canonical/k8s/pkg/k8sd/types"
 	"github.com/canonical/k8s/pkg/snap/mock"
 	snaputil "github.com/canonical/k8s/pkg/snap/util"
 	"github.com/canonical/k8s/pkg/utils"
+	"github.com/canonical/k8s/pkg/utils/vals"
 	. "github.com/onsi/gomega"
 )
 
@@ -35,7 +37,7 @@ func TestKubeAPIServer(t *testing.T) {
 		s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock)
 
 		// Call the KubeAPIServer setup function with mock arguments
-		g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", true, "k8s-dqlite", "datastoreurl", "Node,RBAC")).To(BeNil())
+		g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", true, types.Datastore{Type: vals.Pointer("k8s-dqlite")}, "Node,RBAC")).To(BeNil())
 
 		// Ensure the kube-apiserver arguments file has the expected arguments and values
 		tests := []struct {
@@ -90,7 +92,7 @@ func TestKubeAPIServer(t *testing.T) {
 		s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock)
 
 		// Call the KubeAPIServer setup function with mock arguments
-		g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, "k8s-dqlite", "datastoreurl", "Node,RBAC")).To(BeNil())
+		g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: vals.Pointer("k8s-dqlite")}, "Node,RBAC")).To(BeNil())
 
 		// Ensure the kube-apiserver arguments file has the expected arguments and values
 		tests := []struct {
@@ -137,7 +139,7 @@ func TestKubeAPIServer(t *testing.T) {
 		s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock)
 
 		// Setup without proxy to simplify argument list
-		g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, "external", "datastoreurl", "Node,RBAC")).To(BeNil())
+		g.Expect(setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: vals.Pointer("external"), ExternalURL: vals.Pointer("datastoreurl")}, "Node,RBAC")).To(BeNil())
 
 		g.Expect(snaputil.GetServiceArgument(s, "kube-apiserver", "--etcd-servers")).To(Equal("datastoreurl"))
 		_, err := utils.ParseArgumentFile(path.Join(s.Mock.ServiceArgumentsDir, "kube-apiserver"))
@@ -151,7 +153,7 @@ func TestKubeAPIServer(t *testing.T) {
 		s := mustSetupSnapAndDirectories(t, setKubeAPIServerMock)
 
 		// Attempt to configure kube-apiserver with an unsupported datastore
-		err := setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, "unsupported-datastore", "datastoreurl", "Node,RBAC")
+		err := setup.KubeAPIServer(s, "10.0.0.0/24", "https://auth-webhook.url", false, types.Datastore{Type: vals.Pointer("unsupported")}, "Node,RBAC")
 		g.Expect(err).To(HaveOccurred())
 		g.Expect(err).To(MatchError(ContainSubstring("unsupported datastore")))
 	})

src/k8s/pkg/k8sd/types/cluster_config_datastore.go

@@ -1,5 +1,10 @@
 package types
 
+import (
+	"fmt"
+	"path"
+)
+
 type Datastore struct {
 	Type *string `json:"type,omitempty"`
 
@@ -24,3 +29,45 @@ func (c Datastore) GetExternalClientKey() string  { return getField(c.ExternalCl
 func (c Datastore) Empty() bool {
 	return c.Type == nil && c.K8sDqlitePort == nil && c.K8sDqliteCert == nil && c.K8sDqliteKey == nil && c.ExternalURL == nil && c.ExternalCACert == nil && c.ExternalClientCert == nil && c.ExternalClientKey == nil
 }
+
+// DatastorePathsProvider is to avoid circular dependency for snap.Snap in Datastore.ToKubeAPIServerArguments()
+type DatastorePathsProvider interface {
+	K8sDqliteStateDir() string
+	EtcdPKIDir() string
+}
+
+// ToKubeAPIServerArguments returns updateArgs, deleteArgs that can be used with snaputil.UpdateServiceArguments() for the kube-apiserver
+// according the datastore configuration.
+func (c Datastore) ToKubeAPIServerArguments(p DatastorePathsProvider) (map[string]string, []string) {
+	var (
+		updateArgs = make(map[string]string)
+		deleteArgs []string
+	)
+
+	switch c.GetType() {
+	case "k8s-dqlite":
+		updateArgs["--etcd-servers"] = fmt.Sprintf("unix://%s", path.Join(p.K8sDqliteStateDir(), "k8s-dqlite.sock"))
+		deleteArgs = []string{"--etcd-cafile", "--etcd-certfile", "--etcd-keyfile"}
+	case "external":
+		updateArgs["--etcd-servers"] = c.GetExternalURL()
+
+		// the certificates will be written by setup.EnsureExtDatastorePKI(), here we only set the paths
+		for _, loop := range []struct {
+			arg  string
+			cert string
+			path string
+		}{
+			{cert: c.GetExternalCACert(), arg: "--etcd-cafile", path: "ca.crt"},
+			{cert: c.GetExternalClientCert(), arg: "--etcd-certfile", path: "client.crt"},
+			{cert: c.GetExternalClientKey(), arg: "--etcd-keyfile", path: "client.key"},
+		} {
+			if loop.cert != "" {
+				updateArgs[loop.arg] = path.Join(p.EtcdPKIDir(), loop.path)
+			} else {
+				deleteArgs = append(deleteArgs, loop.arg)
+			}
+		}
+	}
+
+	return updateArgs, deleteArgs
+}

src/k8s/pkg/k8sd/types/cluster_config_datastore_test.go

@@ -0,0 +1,78 @@
+package types_test
+
+import (
+	"testing"
+
+	"github.com/canonical/k8s/pkg/k8sd/types"
+	"github.com/canonical/k8s/pkg/snap/mock"
+	"github.com/canonical/k8s/pkg/utils/vals"
+	. "github.com/onsi/gomega"
+)
+
+func TestDatastoreToKubeAPIServerArguments(t *testing.T) {
+	snap := &mock.Snap{
+		Mock: mock.Mock{
+			K8sDqliteStateDir: "/k8s-dqlite",
+			EtcdPKIDir:        "/pki/etcd",
+		},
+	}
+
+	for _, tc := range []struct {
+		name             string
+		config           types.Datastore
+		expectUpdateArgs map[string]string
+		expectDeleteArgs []string
+	}{
+		{
+			name:             "Nil",
+			expectUpdateArgs: map[string]string{},
+		},
+		{
+			name: "K8sDqlite",
+			config: types.Datastore{
+				Type: vals.Pointer("k8s-dqlite"),
+			},
+			expectUpdateArgs: map[string]string{
+				"--etcd-servers": "unix:///k8s-dqlite/k8s-dqlite.sock",
+			},
+			expectDeleteArgs: []string{"--etcd-cafile", "--etcd-certfile", "--etcd-keyfile"},
+		},
+		{
+			name: "ExternalFull",
+			config: types.Datastore{
+				Type:               vals.Pointer("external"),
+				ExternalURL:        vals.Pointer("https://10.0.0.10:2379,https://10.0.0.11:2379"),
+				ExternalCACert:     vals.Pointer("data"),
+				ExternalClientCert: vals.Pointer("data"),
+				ExternalClientKey:  vals.Pointer("data"),
+			},
+			expectUpdateArgs: map[string]string{
+				"--etcd-servers":  "https://10.0.0.10:2379,https://10.0.0.11:2379",
+				"--etcd-cafile":   "/pki/etcd/ca.crt",
+				"--etcd-certfile": "/pki/etcd/client.crt",
+				"--etcd-keyfile":  "/pki/etcd/client.key",
+			},
+		},
+		{
+			name: "ExternalOnlyCA",
+			config: types.Datastore{
+				Type:           vals.Pointer("external"),
+				ExternalURL:    vals.Pointer("https://10.0.0.10:2379,https://10.0.0.11:2379"),
+				ExternalCACert: vals.Pointer("data"),
+			},
+			expectUpdateArgs: map[string]string{
+				"--etcd-servers": "https://10.0.0.10:2379,https://10.0.0.11:2379",
+				"--etcd-cafile":  "/pki/etcd/ca.crt",
+			},
+			expectDeleteArgs: []string{"--etcd-certfile", "--etcd-keyfile"},
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			g := NewWithT(t)
+
+			update, delete := tc.config.ToKubeAPIServerArguments(snap)
+			g.Expect(update).To(Equal(tc.expectUpdateArgs))
+			g.Expect(delete).To(Equal(tc.expectDeleteArgs))
+		})
+	}
+}