diff --git a/build-scripts/hack/update-component-versions.py b/build-scripts/hack/update-component-versions.py index 21c3d865e..7dba88081 100755 --- a/build-scripts/hack/update-component-versions.py +++ b/build-scripts/hack/update-component-versions.py @@ -41,6 +41,11 @@ CONTOUR_HELM_REPO = "https://charts.bitnami.com/bitnami" CONTOUR_CHART_VERSION = "17.0.4" +# MetalLB Helm repository and chart version +METALLB_REPO = "https://metallb.github.io/metallb" +METALLB_CHART_VERSION = "0.14.5" + + def get_kubernetes_version() -> str: """Update Kubernetes version based on the specified marker file""" LOG.info("Checking latest Kubernetes version from %s", KUBERNETES_VERSION_MARKER) @@ -63,10 +68,16 @@ def get_cni_version() -> str: raise Exception(f"Failed to find cni dependency in {deps_file}") + def pull_contour_chart() -> None: - LOG.info("Pulling Contour Helm chart from %s with version %s", CONTOUR_HELM_REPO, CONTOUR_CHART_VERSION) + LOG.info( + "Pulling Contour Helm chart from %s with version %s", + CONTOUR_HELM_REPO, + CONTOUR_CHART_VERSION, + ) util.helm_pull("contour", CONTOUR_HELM_REPO, CONTOUR_CHART_VERSION, CHARTS) + def get_containerd_version() -> str: """Update containerd version using latest tag of specified branch""" containerd_repo = util.read_file(COMPONENTS / "containerd/repository") @@ -93,6 +104,11 @@ def get_helm_version() -> str: return util.parse_output(["git", "describe", "--tags", "--abbrev=0"], cwd=dir) +def pull_metallb_chart() -> None: + LOG.info("Pulling MetalLB chart @ %s", METALLB_CHART_VERSION) + util.helm_pull("metallb", METALLB_REPO, METALLB_CHART_VERSION, CHARTS) + + def update_component_versions(dry_run: bool): for component, get_version in [ ("kubernetes", get_kubernetes_version), @@ -110,6 +126,7 @@ def update_component_versions(dry_run: bool): for component, pull_helm_chart in [ ("bitnami/contour", pull_contour_chart), + ("metallb", pull_metallb_chart), ]: LOG.info("Updating chart for %s", component) if not dry_run: diff --git a/build-scripts/hack/util.py b/build-scripts/hack/util.py index 090f1d4f4..05bc961eb 100644 --- a/build-scripts/hack/util.py +++ b/build-scripts/hack/util.py @@ -49,6 +49,7 @@ def read_file(path: Path) -> str: def read_url(url: str) -> str: return urlopen(url).read().decode().strip() + def helm_pull(chart, repo_url: str, version: str, destination: Path) -> None: parse_output( [ diff --git a/k8s/manifests/charts/ck-loadbalancer/templates/bgp-policy.yaml b/k8s/manifests/charts/ck-loadbalancer/templates/cilium/bgp-policy.yaml similarity index 84% rename from k8s/manifests/charts/ck-loadbalancer/templates/bgp-policy.yaml rename to k8s/manifests/charts/ck-loadbalancer/templates/cilium/bgp-policy.yaml index 94e31921b..3affe934c 100644 --- a/k8s/manifests/charts/ck-loadbalancer/templates/bgp-policy.yaml +++ b/k8s/manifests/charts/ck-loadbalancer/templates/cilium/bgp-policy.yaml @@ -1,4 +1,6 @@ -{{- if .Values.bgp.enabled }} +{{- if (eq .Values.driver "cilium") }} +{{- if (.Values.bgp.enabled) }} + apiVersion: "cilium.io/v2alpha1" kind: CiliumBGPPeeringPolicy metadata: @@ -15,4 +17,6 @@ spec: neighbors: {{- toYaml . | nindent 4 }} {{- end }} + +{{- end}} {{- end}} diff --git a/k8s/manifests/charts/ck-loadbalancer/templates/l2-policy.yaml b/k8s/manifests/charts/ck-loadbalancer/templates/cilium/l2-policy.yaml similarity index 87% rename from k8s/manifests/charts/ck-loadbalancer/templates/l2-policy.yaml rename to k8s/manifests/charts/ck-loadbalancer/templates/cilium/l2-policy.yaml index 3a0862ca9..7540cd866 100644 --- a/k8s/manifests/charts/ck-loadbalancer/templates/l2-policy.yaml +++ b/k8s/manifests/charts/ck-loadbalancer/templates/cilium/l2-policy.yaml @@ -1,4 +1,6 @@ +{{- if (eq .Values.driver "cilium") }} {{- if .Values.l2.enabled }} + apiVersion: "cilium.io/v2alpha1" kind: CiliumL2AnnouncementPolicy metadata: @@ -12,4 +14,6 @@ spec: {{- end }} externalIPs: true loadBalancerIPs: true + +{{- end }} {{- end }} diff --git a/k8s/manifests/charts/ck-loadbalancer/templates/lb-ip-pool.yaml b/k8s/manifests/charts/ck-loadbalancer/templates/cilium/lb-ip-pool.yaml similarity index 86% rename from k8s/manifests/charts/ck-loadbalancer/templates/lb-ip-pool.yaml rename to k8s/manifests/charts/ck-loadbalancer/templates/cilium/lb-ip-pool.yaml index 044512b55..a5d2190a3 100644 --- a/k8s/manifests/charts/ck-loadbalancer/templates/lb-ip-pool.yaml +++ b/k8s/manifests/charts/ck-loadbalancer/templates/cilium/lb-ip-pool.yaml @@ -1,4 +1,6 @@ +{{- if (eq .Values.driver "cilium") }} {{- if .Values.ipPool.cidrs }} + apiVersion: "cilium.io/v2alpha1" kind: CiliumLoadBalancerIPPool metadata: @@ -10,4 +12,6 @@ spec: blocks: {{- toYaml . | nindent 4 }} {{- end }} + +{{- end }} {{- end }} diff --git a/k8s/manifests/charts/ck-loadbalancer/templates/metallb/bgp-policy.yaml b/k8s/manifests/charts/ck-loadbalancer/templates/metallb/bgp-policy.yaml new file mode 100644 index 000000000..f85027230 --- /dev/null +++ b/k8s/manifests/charts/ck-loadbalancer/templates/metallb/bgp-policy.yaml @@ -0,0 +1,31 @@ +{{- if (eq .Values.driver "metallb") }} +{{- if .Values.bgp.enabled }} + +apiVersion: "metallb.io/v1beta2" +kind: BGPPeer +metadata: + name: {{ include "ck-loadbalancer.fullname" . }} + labels: + {{- include "ck-loadbalancer.labels" . | nindent 4 }} +spec: + myASN: {{ .Values.bgp.localASN }} + {{- with (index .Values.bgp.neighbors 0) }} + peerASN: {{ .peerASN }} + peerAddress: {{ .peerAddress }} + peerPort: {{ .peerPort }} + {{- end }} + +--- + +apiVersion: "metallb.io/v1beta1" +kind: BGPAdvertisement +metadata: + name: {{ include "ck-loadbalancer.fullname" . }} + labels: + {{- include "ck-loadbalancer.labels" . | nindent 4 }} +spec: + ipAddressPools: + - {{ include "ck-loadbalancer.fullname" . }} + +{{- end }} +{{- end}} diff --git a/k8s/manifests/charts/ck-loadbalancer/templates/metallb/l2-policy.yaml b/k8s/manifests/charts/ck-loadbalancer/templates/metallb/l2-policy.yaml new file mode 100644 index 000000000..b97bfd4ad --- /dev/null +++ b/k8s/manifests/charts/ck-loadbalancer/templates/metallb/l2-policy.yaml @@ -0,0 +1,19 @@ +{{- if (eq .Values.driver "metallb") -}} +{{- if .Values.l2.enabled }} + +apiVersion: "metallb.io/v1beta1" +kind: L2Advertisement +metadata: + name: {{ include "ck-loadbalancer.fullname" . }} + labels: + {{- include "ck-loadbalancer.labels" . | nindent 4 }} +spec: + ipAddressPools: + - {{ include "ck-loadbalancer.fullname" . }} + {{- with .Values.l2.interfaces }} + interfaces: + {{- toYaml . | nindent 4 }} + {{- end }} + +{{- end }} +{{- end }} diff --git a/k8s/manifests/charts/ck-loadbalancer/templates/metallb/lb-ip-pool.yaml b/k8s/manifests/charts/ck-loadbalancer/templates/metallb/lb-ip-pool.yaml new file mode 100644 index 000000000..c5a18df7a --- /dev/null +++ b/k8s/manifests/charts/ck-loadbalancer/templates/metallb/lb-ip-pool.yaml @@ -0,0 +1,19 @@ +{{ if (eq .Values.driver "metallb") }} +{{ if .Values.ipPool.cidrs }} +apiVersion: "metallb.io/v1beta1" +kind: IPAddressPool +metadata: + name: {{ include "ck-loadbalancer.fullname" . }} + labels: + {{- include "ck-loadbalancer.labels" . | nindent 4 }} +spec: + addresses: + {{- range .Values.ipPool.cidrs }} + {{- if .cidr }} + - {{ .cidr }} + {{- else if and .start .stop }} + - {{ printf "%s-%s" .start .stop }} + {{ end }} + {{ end }} +{{ end }} +{{ end }} diff --git a/k8s/manifests/charts/ck-loadbalancer/values.yaml b/k8s/manifests/charts/ck-loadbalancer/values.yaml index a9bdafa96..e1a9eef69 100644 --- a/k8s/manifests/charts/ck-loadbalancer/values.yaml +++ b/k8s/manifests/charts/ck-loadbalancer/values.yaml @@ -1,3 +1,4 @@ +driver: l2: enabled: true diff --git a/k8s/manifests/charts/metallb-0.14.5.tgz b/k8s/manifests/charts/metallb-0.14.5.tgz new file mode 100644 index 000000000..c37846191 Binary files /dev/null and b/k8s/manifests/charts/metallb-0.14.5.tgz differ diff --git a/src/k8s/pkg/k8sd/features/cilium/loadbalancer.go b/src/k8s/pkg/k8sd/features/cilium/loadbalancer.go index a2e347187..5ea8549fb 100644 --- a/src/k8s/pkg/k8sd/features/cilium/loadbalancer.go +++ b/src/k8s/pkg/k8sd/features/cilium/loadbalancer.go @@ -99,6 +99,7 @@ func enableLoadBalancer(ctx context.Context, snap snap.Snap, loadbalancer types. } values := map[string]any{ + "driver": "cilium", "l2": map[string]any{ "enabled": loadbalancer.GetL2Mode(), "interfaces": loadbalancer.GetL2Interfaces(), diff --git a/src/k8s/pkg/k8sd/features/implementation_moonray.go b/src/k8s/pkg/k8sd/features/implementation_moonray.go index 9e6309ff0..cbaacad91 100644 --- a/src/k8s/pkg/k8sd/features/implementation_moonray.go +++ b/src/k8s/pkg/k8sd/features/implementation_moonray.go @@ -4,10 +4,10 @@ package features import ( "github.com/canonical/k8s/pkg/k8sd/features/calico" - "github.com/canonical/k8s/pkg/k8sd/features/cilium" "github.com/canonical/k8s/pkg/k8sd/features/contour" "github.com/canonical/k8s/pkg/k8sd/features/coredns" "github.com/canonical/k8s/pkg/k8sd/features/localpv" + "github.com/canonical/k8s/pkg/k8sd/features/metallb" metrics_server "github.com/canonical/k8s/pkg/k8sd/features/metrics-server" ) @@ -16,7 +16,7 @@ import ( var Implementation Interface = &implementation{ applyDNS: coredns.ApplyDNS, applyNetwork: calico.ApplyNetwork, - applyLoadBalancer: cilium.ApplyLoadBalancer, + applyLoadBalancer: metallb.ApplyLoadBalancer, applyIngress: contour.ApplyIngress, applyGateway: contour.ApplyGateway, applyMetricsServer: metrics_server.ApplyMetricsServer, diff --git a/src/k8s/pkg/k8sd/features/metallb/chart.go b/src/k8s/pkg/k8sd/features/metallb/chart.go new file mode 100644 index 000000000..6be802183 --- /dev/null +++ b/src/k8s/pkg/k8sd/features/metallb/chart.go @@ -0,0 +1,41 @@ +package metallb + +import ( + "path" + + "github.com/canonical/k8s/pkg/client/helm" +) + +var ( + // chartMetalLB represents manifests to deploy MetalLB speaker and controller. + chartMetalLB = helm.InstallableChart{ + Name: "metallb", + Namespace: "metallb-system", + ManifestPath: path.Join("charts", "metallb-0.14.5.tgz"), + } + + // chartMetalLBLoadBalancer represents manifests to deploy MetalLB L2 or BGP resources. + chartMetalLBLoadBalancer = helm.InstallableChart{ + Name: "metallb-loadbalancer", + Namespace: "metallb-system", + ManifestPath: path.Join("charts", "ck-loadbalancer"), + } + + // controllerImageRepo is the image to use for metallb-controller. + controllerImageRepo = "quay.io/metallb/controller" + + // controllerImageTag is the tag to use for metallb-controller. + controllerImageTag = "v0.14.5" + + // speakerImageRepo is the image to use for metallb-speaker. + speakerImageRepo = "quay.io/metallb/speaker" + + // speakerImageTag is the tag to use for metallb-speaker. + speakerImageTag = "v0.14.5" + + // frrImageRepo is the image to use for frrouting. + frrImageRepo = "quay.io/frrouting/frr" + + // frrImageTag is the tag to use for frrouting. + frrImageTag = "9.0.2" +) diff --git a/src/k8s/pkg/k8sd/features/metallb/loadbalancer.go b/src/k8s/pkg/k8sd/features/metallb/loadbalancer.go new file mode 100644 index 000000000..e556a8984 --- /dev/null +++ b/src/k8s/pkg/k8sd/features/metallb/loadbalancer.go @@ -0,0 +1,154 @@ +package metallb + +import ( + "context" + "fmt" + + "github.com/canonical/k8s/pkg/client/helm" + "github.com/canonical/k8s/pkg/k8sd/types" + "github.com/canonical/k8s/pkg/snap" + "github.com/canonical/k8s/pkg/utils/control" +) + +func ApplyLoadBalancer(ctx context.Context, snap snap.Snap, loadbalancer types.LoadBalancer, network types.Network, _ types.Annotations) error { + if !loadbalancer.GetEnabled() { + if err := disableLoadBalancer(ctx, snap, network); err != nil { + return fmt.Errorf("failed to disable LoadBalancer: %w", err) + } + return nil + } + + if err := enableLoadBalancer(ctx, snap, loadbalancer, network); err != nil { + return fmt.Errorf("failed to enable LoadBalancer: %w", err) + } + return nil +} + +func disableLoadBalancer(ctx context.Context, snap snap.Snap, network types.Network) error { + m := snap.HelmClient() + + if _, err := m.Apply(ctx, chartMetalLBLoadBalancer, helm.StateDeleted, nil); err != nil { + return fmt.Errorf("failed to uninstall MetalLB LoadBalancer chart: %w", err) + } + + if _, err := m.Apply(ctx, chartMetalLB, helm.StateDeleted, nil); err != nil { + return fmt.Errorf("failed to uninstall MetalLB chart: %w", err) + } + return nil +} + +func enableLoadBalancer(ctx context.Context, snap snap.Snap, loadbalancer types.LoadBalancer, network types.Network) error { + m := snap.HelmClient() + + metalLBValues := map[string]any{ + "controller": map[string]any{ + "image": map[string]any{ + "repository": controllerImageRepo, + "tag": controllerImageTag, + }, + }, + "speaker": map[string]any{ + "image": map[string]any{ + "repository": speakerImageRepo, + "tag": speakerImageTag, + }, + // TODO(neoaggelos): make frr enable/disable configurable through an annotation + // We keep it disabled by default + "frr": map[string]any{ + "enabled": false, + "image": map[string]any{ + "repository": frrImageRepo, + "tag": frrImageTag, + }, + }, + }, + } + if _, err := m.Apply(ctx, chartMetalLB, helm.StatePresent, metalLBValues); err != nil { + return fmt.Errorf("failed to apply MetalLB configuration: %w", err) + } + + if err := waitForRequiredLoadBalancerCRDs(ctx, snap, loadbalancer.GetBGPMode()); err != nil { + return fmt.Errorf("failed to wait for required MetalLB CRDs: %w", err) + } + + cidrs := []map[string]any{} + for _, cidr := range loadbalancer.GetCIDRs() { + cidrs = append(cidrs, map[string]any{"cidr": cidr}) + } + for _, ipRange := range loadbalancer.GetIPRanges() { + cidrs = append(cidrs, map[string]any{"start": ipRange.Start, "stop": ipRange.Stop}) + } + + values := map[string]any{ + "driver": "metallb", + "l2": map[string]any{ + "enabled": loadbalancer.GetL2Mode(), + "interfaces": loadbalancer.GetL2Interfaces(), + }, + "ipPool": map[string]any{ + "cidrs": cidrs, + }, + "bgp": map[string]any{ + "enabled": loadbalancer.GetBGPMode(), + "localASN": loadbalancer.GetBGPLocalASN(), + "neighbors": []map[string]any{ + { + "peerAddress": loadbalancer.GetBGPPeerAddress(), + "peerASN": loadbalancer.GetBGPPeerASN(), + "peerPort": loadbalancer.GetBGPPeerPort(), + }, + }, + }, + } + + if _, err := m.Apply(ctx, chartMetalLBLoadBalancer, helm.StatePresent, values); err != nil { + return fmt.Errorf("failed to apply MetalLB LoadBalancer configuration: %w", err) + } + + return nil +} + +func waitForRequiredLoadBalancerCRDs(ctx context.Context, snap snap.Snap, bgpMode bool) error { + client, err := snap.KubernetesClient("") + if err != nil { + return fmt.Errorf("failed to create Kubernetes client: %w", err) + } + + return control.WaitUntilReady(ctx, func() (bool, error) { + resourcesv1beta1, err := client.ListResourcesForGroupVersion("metallb.io/v1beta1") + if err != nil { + // This error is expected if the group version is not yet deployed. + return false, nil + } + resourcesv1beta2, err := client.ListResourcesForGroupVersion("metallb.io/v1beta2") + if err != nil { + // This error is expected if the group version is not yet deployed. + return false, nil + } + + requiredCRDs := map[string]bool{ + "metallb.io/v1beta1:ipaddresspools": true, + "metallb.io/v1beta1:l2advertisements": true, + } + if bgpMode { + requiredCRDs["metallb.io/v1beta2:bgppeers"] = true + requiredCRDs["metallb.io/v1beta1:bgpadvertisements"] = true + } + + requiredCount := len(requiredCRDs) + + for _, resource := range resourcesv1beta1.APIResources { + if _, ok := requiredCRDs[fmt.Sprintf("metallb.io/v1beta1:%s", resource.Name)]; ok { + requiredCount-- + } + } + + for _, resource := range resourcesv1beta2.APIResources { + if _, ok := requiredCRDs[fmt.Sprintf("metallb.io/v1beta2:%s", resource.Name)]; ok { + requiredCount-- + } + } + + return requiredCount == 0, nil + }) +} diff --git a/src/k8s/pkg/k8sd/features/metallb/register.go b/src/k8s/pkg/k8sd/features/metallb/register.go new file mode 100644 index 000000000..817f5408d --- /dev/null +++ b/src/k8s/pkg/k8sd/features/metallb/register.go @@ -0,0 +1,15 @@ +package metallb + +import ( + "fmt" + + "github.com/canonical/k8s/pkg/k8sd/images" +) + +func init() { + images.Register( + fmt.Sprintf("%s:%s", controllerImageRepo, controllerImageTag), + fmt.Sprintf("%s:%s", speakerImageRepo, speakerImageTag), + fmt.Sprintf("%s:%s", frrImageRepo, frrImageTag), + ) +}