From b83b0e0522790f4a9a1a110f8d92fe32cc5e6e4b Mon Sep 17 00:00:00 2001 From: eaudetcobello Date: Tue, 4 Jun 2024 14:59:55 -0400 Subject: [PATCH] lb implementation --- src/k8s/pkg/k8sd/features/metallb/chart.go | 4 +- .../pkg/k8sd/features/metallb/loadbalancer.go | 146 ++++++++++++++++++ 2 files changed, 148 insertions(+), 2 deletions(-) create mode 100644 src/k8s/pkg/k8sd/features/metallb/loadbalancer.go diff --git a/src/k8s/pkg/k8sd/features/metallb/chart.go b/src/k8s/pkg/k8sd/features/metallb/chart.go index 0a5899ee0c..aacdb92503 100644 --- a/src/k8s/pkg/k8sd/features/metallb/chart.go +++ b/src/k8s/pkg/k8sd/features/metallb/chart.go @@ -9,13 +9,13 @@ import ( var ( chartMetalLB = helm.InstallableChart{ Name: "metallb", - Namespace: "metallb-system", + Namespace: "kube-system", ManifestPath: path.Join("charts", "metallb-0.14.5.tgz"), } chartMetalLBLoadBalancer = helm.InstallableChart{ Name: "metallb-loadbalancer", - Namespace: "metallb-system", + Namespace: "kube-system", ManifestPath: path.Join("charts", "ck-loadbalancer"), } ) diff --git a/src/k8s/pkg/k8sd/features/metallb/loadbalancer.go b/src/k8s/pkg/k8sd/features/metallb/loadbalancer.go new file mode 100644 index 0000000000..a9e36725b0 --- /dev/null +++ b/src/k8s/pkg/k8sd/features/metallb/loadbalancer.go @@ -0,0 +1,146 @@ +package metallb + +import ( + "context" + "fmt" + + "github.com/canonical/k8s/pkg/client/helm" + "github.com/canonical/k8s/pkg/k8sd/types" + "github.com/canonical/k8s/pkg/snap" + "github.com/canonical/k8s/pkg/utils/control" +) + +func ApplyLoadBalancer(ctx context.Context, snap snap.Snap, loadbalancer types.LoadBalancer, network types.Network, _ types.Annotations) error { + if !loadbalancer.GetEnabled() { + if err := disableLoadBalancer(ctx, snap, network); err != nil { + return fmt.Errorf("failed to disable LoadBalancer: %w", err) + } + return nil + } + + if err := enableLoadBalancer(ctx, snap, loadbalancer, network); err != nil { + return fmt.Errorf("failed to enable LoadBalancer: %w", err) + } + return nil +} + +func disableLoadBalancer(ctx context.Context, snap snap.Snap, network types.Network) error { + m := snap.HelmClient() + + if _, err := m.Apply(ctx, chartMetalLBLoadBalancer, helm.StateDeleted, nil); err != nil { + return fmt.Errorf("failed to uninstall LoadBalancer manifests: %w", err) + } + + if _, err := m.Apply(ctx, chartMetalLB, helm.StateDeleted, nil); err != nil { + return fmt.Errorf("failed to refresh network to apply LoadBalancer configuration: %w", err) + } + return nil +} + +func enableLoadBalancer(ctx context.Context, snap snap.Snap, loadbalancer types.LoadBalancer, network types.Network) error { + m := snap.HelmClient() + + if _, err := m.Apply(ctx, chartMetalLB, helm.StatePresent, nil); err != nil { + return fmt.Errorf("failed to apply MetalLB configuration: %w", err) + } + + if err := waitForRequiredLoadBalancerCRDs(ctx, snap, loadbalancer.GetBGPMode()); err != nil { + return fmt.Errorf("failed to wait for required LoadBalancer CRDs: %w", err) + } + + cidrs := []map[string]any{} + for _, cidr := range loadbalancer.GetCIDRs() { + cidrs = append(cidrs, map[string]any{"cidr": cidr}) + } + for _, ipRange := range loadbalancer.GetIPRanges() { + cidrs = append(cidrs, map[string]any{"start": ipRange.Start, "stop": ipRange.Stop}) + } + + values := map[string]any{ + "l2": map[string]any{ + "enabled": loadbalancer.GetL2Mode(), + "interfaces": loadbalancer.GetL2Interfaces(), + }, + "ipPool": map[string]any{ + "cidrs": cidrs, + }, + "bgp": map[string]any{ + "enabled": loadbalancer.GetBGPMode(), + "localASN": loadbalancer.GetBGPLocalASN(), + "neighbors": []map[string]any{ + { + "peerAddress": loadbalancer.GetBGPPeerAddress(), + "peerASN": loadbalancer.GetBGPPeerASN(), + "peerPort": loadbalancer.GetBGPPeerPort(), + }, + }, + }, + } + + if _, err := m.Apply(ctx, chartMetalLBLoadBalancer, helm.StatePresent, values); err != nil { + return fmt.Errorf("failed to apply LoadBalancer configuration: %w", err) + } + + if err := rolloutRestartMetalLB(ctx, snap, 3); err != nil { + return fmt.Errorf("failed to rollout restart metallb to apply LoadBalancer configuration: %w", err) + } + + return nil +} + +func waitForRequiredLoadBalancerCRDs(ctx context.Context, snap snap.Snap, bgpMode bool) error { + client, err := snap.KubernetesClient("") + if err != nil { + return fmt.Errorf("failed to create Kubernetes client: %w", err) + } + + return control.WaitUntilReady(ctx, func() (bool, error) { + resources, err := client.ListResourcesForGroupVersion("metallb.io/v1beta1") + if err != nil { + // This error is expected if the group version is not yet deployed. + return false, nil + } + + requiredCRDs := map[string]struct{}{ + "ipaddresspools": {}, + "l2advertisements": {}, + } + if bgpMode { + requiredCRDs["ciliumbgppeeringpolicies"] = struct{}{} + } + requiredCount := len(requiredCRDs) + for _, resource := range resources.APIResources { + if _, ok := requiredCRDs[resource.Name]; ok { + requiredCount = requiredCount - 1 + } + } + return requiredCount == 0, nil + }) +} + +func rolloutRestartMetalLB(ctx context.Context, snap snap.Snap, attempts int) error { + client, err := snap.KubernetesClient("") + if err != nil { + return fmt.Errorf("failed to create kubernetes client: %w", err) + } + + if err := control.RetryFor(ctx, attempts, 0, func() error { + if err := client.RestartDeployment(ctx, "metallb-controller", "kube-system"); err != nil { + return fmt.Errorf("failed to restart metallb-controller deployment: %w", err) + } + return nil + }); err != nil { + return fmt.Errorf("failed to restart metallb-controller deployment after %d attempts: %w", attempts, err) + } + + if err := control.RetryFor(ctx, attempts, 0, func() error { + if err := client.RestartDaemonset(ctx, "metallb-speaker", "kube-system"); err != nil { + return fmt.Errorf("failed to restart metallb-speaker daemonset: %w", err) + } + return nil + }); err != nil { + return fmt.Errorf("failed to restart metallb-speaker daemonset after %d attempts: %w", attempts, err) + } + + return nil +}