diff --git a/src/k8s/api/v1/annotations.go b/src/k8s/api/v1/annotations.go new file mode 100644 index 000000000..3597c0680 --- /dev/null +++ b/src/k8s/api/v1/annotations.go @@ -0,0 +1,8 @@ +package apiv1 + +const ( + // AnnotationSkipCleanupKubernetesNodeOnRemove if set, only the microcluster & file cleanup is done. + // This is useful, if an external controller (e.g. CAPI) is responsible for the Kubernetes node life cycle. + // By default, the Kubernetes node is removed by k8sd if a node is removed from the cluster. + AnnotationSkipCleanupKubernetesNodeOnRemove = "k8sd/v1alpha/lifecycle/skip-cleanup-kubernetes-node-on-remove" +) diff --git a/src/k8s/cmd/k8s/k8s_bootstrap_test.go b/src/k8s/cmd/k8s/k8s_bootstrap_test.go index c82839383..e8fdba3d4 100644 --- a/src/k8s/cmd/k8s/k8s_bootstrap_test.go +++ b/src/k8s/cmd/k8s/k8s_bootstrap_test.go @@ -63,6 +63,7 @@ var testCases = []testCase{ Enabled: utils.Pointer(true), }, CloudProvider: utils.Pointer("external"), + Annotations: map[string]string{apiv1.AnnotationSkipCleanupKubernetesNodeOnRemove: "true"}, }, ControlPlaneTaints: []string{"node-role.kubernetes.io/control-plane:NoSchedule"}, PodCIDR: utils.Pointer("10.100.0.0/16"), diff --git a/src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml b/src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml index 8431de74d..79def822f 100644 --- a/src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml +++ b/src/k8s/cmd/k8s/testdata/bootstrap-config-full.yaml @@ -21,6 +21,8 @@ cluster-config: metrics-server: enabled: true cloud-provider: external + annotations: + k8sd/v1alpha/lifecycle/skip-cleanup-kubernetes-node-on-remove: true control-plane-taints: - node-role.kubernetes.io/control-plane:NoSchedule pod-cidr: 10.100.0.0/16 diff --git a/src/k8s/pkg/k8sd/api/cluster_remove.go b/src/k8s/pkg/k8sd/api/cluster_remove.go index bd8b6e60b..b44c26b94 100644 --- a/src/k8s/pkg/k8sd/api/cluster_remove.go +++ b/src/k8s/pkg/k8sd/api/cluster_remove.go @@ -7,6 +7,7 @@ import ( "net/http" apiv1 "github.com/canonical/k8s/api/v1" + databaseutil "github.com/canonical/k8s/pkg/k8sd/database/util" "github.com/canonical/k8s/pkg/log" "github.com/canonical/k8s/pkg/utils" "github.com/canonical/k8s/pkg/utils/control" @@ -72,6 +73,17 @@ func (e *Endpoints) postClusterRemove(s state.State, r *http.Request) response.R return response.SyncResponse(true, nil) } + cfg, err := databaseutil.GetClusterConfig(ctx, s) + if err != nil { + return response.InternalError(fmt.Errorf("failed to get cluster config: %w", err)) + } + + if _, ok := cfg.Annotations[apiv1.AnnotationSkipCleanupKubernetesNodeOnRemove]; ok { + // Explicitly skip removing the node from Kubernetes. + log.Info("Skipping Kubernetes worker node removal") + return response.SyncResponse(true, nil) + } + client, err := snap.KubernetesClient("") if err != nil { return response.InternalError(fmt.Errorf("failed to create k8s client: %w", err)) diff --git a/src/k8s/pkg/k8sd/app/hooks_remove.go b/src/k8s/pkg/k8sd/app/hooks_remove.go index 1989b3f0b..730d453e7 100644 --- a/src/k8s/pkg/k8sd/app/hooks_remove.go +++ b/src/k8s/pkg/k8sd/app/hooks_remove.go @@ -7,6 +7,7 @@ import ( "net" "os" + apiv1 "github.com/canonical/k8s/api/v1" databaseutil "github.com/canonical/k8s/pkg/k8sd/database/util" "github.com/canonical/k8s/pkg/k8sd/pki" "github.com/canonical/k8s/pkg/k8sd/setup" @@ -23,7 +24,7 @@ import ( func (a *App) onPreRemove(ctx context.Context, s state.State, force bool) (rerr error) { snap := a.Snap() - log := log.FromContext(ctx).WithValues("hook", "preremove") + log := log.FromContext(ctx).WithValues("hook", "preremove", "node", s.Name()) log.Info("Running preremove hook") log.Info("Waiting for node to finish microcluster join before removing") @@ -43,8 +44,19 @@ func (a *App) onPreRemove(ctx context.Context, s state.State, force bool) (rerr return notPending, nil }) - cfg, clusterConfigErr := databaseutil.GetClusterConfig(ctx, s) - if clusterConfigErr == nil { + if cfg, err := databaseutil.GetClusterConfig(ctx, s); err == nil { + if _, ok := cfg.Annotations[apiv1.AnnotationSkipCleanupKubernetesNodeOnRemove]; !ok { + c, err := snap.KubernetesClient("") + if err != nil { + log.Error(err, "Failed to create Kubernetes client", err) + } + + log.Info("Deleting node from Kubernetes cluster") + if err := c.DeleteNode(ctx, s.Name()); err != nil { + log.Error(err, "Failed to remove k8s node %q: %w", s.Name(), err) + } + } + switch cfg.Datastore.GetType() { case "k8s-dqlite": client, err := snap.K8sDqliteClient(ctx) @@ -72,17 +84,7 @@ func (a *App) onPreRemove(ctx context.Context, s state.State, force bool) (rerr default: } } else { - log.Error(clusterConfigErr, "Failed to retrieve cluster config") - } - - c, err := snap.KubernetesClient("") - if err != nil { - log.Error(err, "Failed to create Kubernetes client", err) - } - - log.Info("Deleting node from Kubernetes cluster") - if err := c.DeleteNode(ctx, s.Name()); err != nil { - log.Error(err, "Failed to remove k8s node %q: %w", s.Name(), err) + log.Error(err, "Failed to retrieve cluster config") } for _, dir := range []string{snap.ServiceArgumentsDir()} { @@ -115,11 +117,9 @@ func (a *App) onPreRemove(ctx context.Context, s state.State, force bool) (rerr log.Error(err, "failed to cleanup control plane certificates") } - if clusterConfigErr == nil { - log.Info("Stopping control plane services") - if err := snaputil.StopControlPlaneServices(ctx, snap); err != nil { - log.Error(err, "Failed to stop control-plane services") - } + log.Info("Stopping control plane services") + if err := snaputil.StopControlPlaneServices(ctx, snap); err != nil { + log.Error(err, "Failed to stop control-plane services") } return nil diff --git a/src/k8s/pkg/k8sd/types/cluster_config_defaults_test.go b/src/k8s/pkg/k8sd/types/cluster_config_defaults_test.go index 6b2e5ac86..ae8b8581c 100644 --- a/src/k8s/pkg/k8sd/types/cluster_config_defaults_test.go +++ b/src/k8s/pkg/k8sd/types/cluster_config_defaults_test.go @@ -1,10 +1,10 @@ package types_test import ( - "github.com/canonical/k8s/pkg/utils" "testing" "github.com/canonical/k8s/pkg/k8sd/types" + "github.com/canonical/k8s/pkg/utils" . "github.com/onsi/gomega" ) diff --git a/tests/integration/templates/bootstrap-no-k8s-node-remove.yaml b/tests/integration/templates/bootstrap-no-k8s-node-remove.yaml new file mode 100644 index 000000000..24bcc26d6 --- /dev/null +++ b/tests/integration/templates/bootstrap-no-k8s-node-remove.yaml @@ -0,0 +1,9 @@ +cluster-config: + network: + enabled: true + dns: + enabled: true + metrics-server: + enabled: true + annotations: + k8sd/v1alpha/lifecycle/skip-cleanup-kubernetes-node-on-remove: true diff --git a/tests/integration/tests/test_clustering.py b/tests/integration/tests/test_clustering.py index 5a4e03abe..4b076eb74 100644 --- a/tests/integration/tests/test_clustering.py +++ b/tests/integration/tests/test_clustering.py @@ -5,7 +5,7 @@ from typing import List import pytest -from test_util import harness, util +from test_util import config, harness, util LOG = logging.getLogger(__name__) @@ -64,3 +64,33 @@ def test_worker_nodes(instances: List[harness.Instance]): ] and other_joining_node.id in [ node["metadata"]["name"] for node in nodes ], f"only {cluster_node.id} should be left in cluster" + + +@pytest.mark.node_count(3) +@pytest.mark.bootstrap_config( + (config.MANIFESTS_DIR / "bootstrap-no-k8s-node-remove.yaml").read_text() +) +def test_no_remove(instances: List[harness.Instance]): + cluster_node = instances[0] + joining_cp = instances[1] + joining_worker = instances[2] + + join_token = util.get_join_token(cluster_node, joining_cp) + join_token_worker = util.get_join_token(cluster_node, joining_worker, "--worker") + util.join_cluster(joining_cp, join_token) + util.join_cluster(joining_worker, join_token_worker) + + util.wait_until_k8s_ready(cluster_node, instances) + nodes = util.ready_nodes(cluster_node) + assert len(nodes) == 3, "nodes should have joined cluster" + + assert "control-plane" in util.get_local_node_status(cluster_node) + assert "control-plane" in util.get_local_node_status(joining_cp) + assert "worker" in util.get_local_node_status(joining_worker) + + nodes = util.ready_nodes(cluster_node) + + cluster_node.exec(["k8s", "remove-node", joining_cp.id]) + assert len(nodes) == 3, "cp node should not have been removed from cluster" + cluster_node.exec(["k8s", "remove-node", joining_worker.id]) + assert len(nodes) == 3, "worker node should not have been removed from cluster"