From 1698316ac79ce654a872786521276165bfd2269d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Berkay=20Tekin=20=C3=96z?= Date: Tue, 6 Feb 2024 17:30:44 +0300 Subject: [PATCH] Added loadbalancer support (#106) --- .../charts/ck-loadbalancer/.helmignore | 23 +++ .../charts/ck-loadbalancer/Chart.yaml | 24 +++ .../ck-loadbalancer/templates/_helpers.tpl | 62 +++++++ .../ck-loadbalancer/templates/bgp-policy.yaml | 18 ++ .../ck-loadbalancer/templates/l2-policy.yaml | 15 ++ .../ck-loadbalancer/templates/lb-ip-pool.yaml | 13 ++ .../charts/ck-loadbalancer/values.schema.json | 75 ++++++++ .../charts/ck-loadbalancer/values.yaml | 20 +++ k8s/components/components.yaml | 4 + src/k8s/api/v1/component.go | 21 +++ src/k8s/cmd/k8s/k8s_disable.go | 4 + src/k8s/cmd/k8s/k8s_disable_dns.go | 3 +- src/k8s/cmd/k8s/k8s_disable_gateway.go | 35 ++++ src/k8s/cmd/k8s/k8s_disable_ingress.go | 35 ++++ src/k8s/cmd/k8s/k8s_disable_loadbalancer.go | 35 ++++ src/k8s/cmd/k8s/k8s_disable_network.go | 3 +- src/k8s/cmd/k8s/k8s_disable_storage.go | 35 ++++ src/k8s/cmd/k8s/k8s_enable.go | 3 +- src/k8s/cmd/k8s/k8s_enable_dns.go | 3 +- src/k8s/cmd/k8s/k8s_enable_gateway.go | 3 +- src/k8s/cmd/k8s/k8s_enable_ingress.go | 3 +- src/k8s/cmd/k8s/k8s_enable_loadbalancer.go | 68 ++++++++ src/k8s/cmd/k8s/k8s_enable_network.go | 3 +- src/k8s/cmd/k8s/k8s_enable_storage.go | 3 +- src/k8s/pkg/component/loadbalancer.go | 123 +++++++++++++ src/k8s/pkg/component/network.go | 4 - src/k8s/pkg/k8s/client/component.go | 11 ++ src/k8s/pkg/k8sd/api/component.go | 34 ++++ src/k8s/pkg/k8sd/api/endpoints.go | 5 + tests/e2e/templates/loadbalancer-test.yaml | 33 ++++ tests/e2e/tests/test_loadbalancer.py | 162 ++++++++++++++++++ 31 files changed, 864 insertions(+), 19 deletions(-) create mode 100644 k8s/components/charts/ck-loadbalancer/.helmignore create mode 100644 k8s/components/charts/ck-loadbalancer/Chart.yaml create mode 100644 k8s/components/charts/ck-loadbalancer/templates/_helpers.tpl create mode 100644 k8s/components/charts/ck-loadbalancer/templates/bgp-policy.yaml create mode 100644 k8s/components/charts/ck-loadbalancer/templates/l2-policy.yaml create mode 100644 k8s/components/charts/ck-loadbalancer/templates/lb-ip-pool.yaml create mode 100644 k8s/components/charts/ck-loadbalancer/values.schema.json create mode 100644 k8s/components/charts/ck-loadbalancer/values.yaml create mode 100644 src/k8s/cmd/k8s/k8s_disable_gateway.go create mode 100644 src/k8s/cmd/k8s/k8s_disable_ingress.go create mode 100644 src/k8s/cmd/k8s/k8s_disable_loadbalancer.go create mode 100644 src/k8s/cmd/k8s/k8s_disable_storage.go create mode 100644 src/k8s/cmd/k8s/k8s_enable_loadbalancer.go create mode 100644 src/k8s/pkg/component/loadbalancer.go create mode 100644 tests/e2e/templates/loadbalancer-test.yaml create mode 100644 tests/e2e/tests/test_loadbalancer.py diff --git a/k8s/components/charts/ck-loadbalancer/.helmignore b/k8s/components/charts/ck-loadbalancer/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/k8s/components/charts/ck-loadbalancer/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/k8s/components/charts/ck-loadbalancer/Chart.yaml b/k8s/components/charts/ck-loadbalancer/Chart.yaml new file mode 100644 index 000000000..62c410230 --- /dev/null +++ b/k8s/components/charts/ck-loadbalancer/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: ck-loadbalancer +description: A Helm chart containing LoadBalancer manifests for Canonical Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.1.0" diff --git a/k8s/components/charts/ck-loadbalancer/templates/_helpers.tpl b/k8s/components/charts/ck-loadbalancer/templates/_helpers.tpl new file mode 100644 index 000000000..920697e59 --- /dev/null +++ b/k8s/components/charts/ck-loadbalancer/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "ck-loadbalancer.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ck-loadbalancer.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "ck-loadbalancer.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "ck-loadbalancer.labels" -}} +helm.sh/chart: {{ include "ck-loadbalancer.chart" . }} +{{ include "ck-loadbalancer.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "ck-loadbalancer.selectorLabels" -}} +app.kubernetes.io/name: {{ include "ck-loadbalancer.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "ck-loadbalancer.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "ck-loadbalancer.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/k8s/components/charts/ck-loadbalancer/templates/bgp-policy.yaml b/k8s/components/charts/ck-loadbalancer/templates/bgp-policy.yaml new file mode 100644 index 000000000..94e31921b --- /dev/null +++ b/k8s/components/charts/ck-loadbalancer/templates/bgp-policy.yaml @@ -0,0 +1,18 @@ +{{- if .Values.bgp.enabled }} +apiVersion: "cilium.io/v2alpha1" +kind: CiliumBGPPeeringPolicy +metadata: + name: {{ include "ck-loadbalancer.fullname" . }} + labels: + {{- include "ck-loadbalancer.labels" . | nindent 4 }} +spec: + virtualRouters: + - localASN: {{ .Values.bgp.localASN }} + serviceSelector: + matchExpressions: + - {key: somekey, operator: NotIn, values: ['never-used-value']} + {{- with .Values.bgp.neighbors }} + neighbors: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end}} diff --git a/k8s/components/charts/ck-loadbalancer/templates/l2-policy.yaml b/k8s/components/charts/ck-loadbalancer/templates/l2-policy.yaml new file mode 100644 index 000000000..3a0862ca9 --- /dev/null +++ b/k8s/components/charts/ck-loadbalancer/templates/l2-policy.yaml @@ -0,0 +1,15 @@ +{{- if .Values.l2.enabled }} +apiVersion: "cilium.io/v2alpha1" +kind: CiliumL2AnnouncementPolicy +metadata: + name: {{ include "ck-loadbalancer.fullname" . }} + labels: + {{- include "ck-loadbalancer.labels" . | nindent 4 }} +spec: + {{- with .Values.l2.interfaces }} + interfaces: + {{- toYaml . | nindent 4 }} + {{- end }} + externalIPs: true + loadBalancerIPs: true +{{- end }} diff --git a/k8s/components/charts/ck-loadbalancer/templates/lb-ip-pool.yaml b/k8s/components/charts/ck-loadbalancer/templates/lb-ip-pool.yaml new file mode 100644 index 000000000..7f2bc9d12 --- /dev/null +++ b/k8s/components/charts/ck-loadbalancer/templates/lb-ip-pool.yaml @@ -0,0 +1,13 @@ +{{- if .Values.ipPool.cidrs }} +apiVersion: "cilium.io/v2alpha1" +kind: CiliumLoadBalancerIPPool +metadata: + name: {{ include "ck-loadbalancer.fullname" . }} + labels: + {{- include "ck-loadbalancer.labels" . | nindent 4 }} +spec: + {{- with .Values.ipPool.cidrs }} + cidrs: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/k8s/components/charts/ck-loadbalancer/values.schema.json b/k8s/components/charts/ck-loadbalancer/values.schema.json new file mode 100644 index 000000000..6a793b079 --- /dev/null +++ b/k8s/components/charts/ck-loadbalancer/values.schema.json @@ -0,0 +1,75 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema#", + "type": "object", + "properties": { + "l2": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "interfaces": { + "type": [ "array", "null" ], + "items": [ + { + "type": "string" + } + ] + } + } + }, + "ipPool": { + "type": "object", + "properties": { + "cidrs": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "cidr": { + "type": "string" + } + }, + "required": ["cidr"] + } + ] + } + }, + "required": ["cidrs"] + }, + "bgp": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "localASN": { + "type": "integer" + }, + "neighbors": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "peerAddress": { + "type": "string" + }, + "peerASN": { + "type": "integer" + }, + "peerPort": { + "type": "integer" + } + }, + "required": ["peerAddress", "peerASN", "peerPort"] + } + ] + } + }, + "required": ["localASN", "neighbors"] + } + }, + "required": ["l2", "ipPool", "bgp"] +} diff --git a/k8s/components/charts/ck-loadbalancer/values.yaml b/k8s/components/charts/ck-loadbalancer/values.yaml new file mode 100644 index 000000000..a9bdafa96 --- /dev/null +++ b/k8s/components/charts/ck-loadbalancer/values.yaml @@ -0,0 +1,20 @@ + +l2: + enabled: true + # interfaces: + # - "^eth[0-9]+" + interfaces: [] + +ipPool: + # cidrs: + # - cidr: "10.42.254.176/28" + cidrs: [] + +bgp: + enabled: false + localASN: 64512 + # neighbors: + # - peerAddress: '10.0.0.60/24' + # peerASN: 65100 + # peerPort: 179 + neighbors: [] diff --git a/k8s/components/components.yaml b/k8s/components/components.yaml index cab09076b..a5ff3e05e 100644 --- a/k8s/components/components.yaml +++ b/k8s/components/components.yaml @@ -16,3 +16,7 @@ gateway: release: "ck-gateway" chart: "gateway-api-0.7.1.tgz" namespace: "kube-system" +loadbalancer: + release: "ck-loadbalancer" + chart: "ck-loadbalancer" + namespace: "kube-system" diff --git a/src/k8s/api/v1/component.go b/src/k8s/api/v1/component.go index 5a3bea29a..a1880440b 100644 --- a/src/k8s/api/v1/component.go +++ b/src/k8s/api/v1/component.go @@ -48,6 +48,24 @@ type UpdateGatewayComponentRequest struct { Status ComponentStatus `json:"status"` } +// UpdateLoadBalancerComponentRequest is used to update the LoadBalancer component state. +type UpdateLoadBalancerComponentRequest struct { + Status ComponentStatus `json:"status"` + Config LoadBalancerComponentConfig `json:"config,omitempty"` +} + +// LoadBalancerComponentConfig holds the configuration values for the LoadBalancer component. +type LoadBalancerComponentConfig struct { + CIDRs []string `json:"cidrs,omitempty"` + L2Enabled bool `json:"l2Enabled,omitempty"` + L2Interfaces []string `json:"l2Interfaces,omitempty"` + BGPEnabled bool `json:"bgpEnabled,omitempty"` + BGPLocalASN int `json:"bgpLocalAsn,omitempty"` + BGPPeerAddress string `json:"bgpPeerAddress,omitempty"` + BGPPeerASN int `json:"bgpPeerAsn,omitempty"` + BGPPeerPort int `json:"bgpPeerPort,omitempty"` +} + // UpdateDNSComponentResponse is the response for "PUT 1.0/k8sd/components/dns". type UpdateDNSComponentResponse struct{} @@ -63,6 +81,9 @@ type UpdateIngressComponentResponse struct{} // UpdateGatewayComponentResponse is the response for "PUT 1.0/k8sd/components/gateway". type UpdateGatewayComponentResponse struct{} +// UpdateLoadBalancerComponentResponse is the response for "PUT 1.0/k8sd/components/loadbalancer". +type UpdateLoadBalancerComponentResponse struct{} + // Component holds information about a k8s component. type Component struct { Name string `json:"name"` diff --git a/src/k8s/cmd/k8s/k8s_disable.go b/src/k8s/cmd/k8s/k8s_disable.go index b7d4bd9d6..4fca1b9fe 100644 --- a/src/k8s/cmd/k8s/k8s_disable.go +++ b/src/k8s/cmd/k8s/k8s_disable.go @@ -19,4 +19,8 @@ func init() { rootCmd.AddCommand(disableCmd) disableCmd.AddCommand(disableDNSCmd) disableCmd.AddCommand(disableNetworkCmd) + disableCmd.AddCommand(disableStorageCmd) + disableCmd.AddCommand(disableIngressCmd) + disableCmd.AddCommand(disableGatewayCmd) + disableCmd.AddCommand(disableLoadBalancerCmd) } diff --git a/src/k8s/cmd/k8s/k8s_disable_dns.go b/src/k8s/cmd/k8s/k8s_disable_dns.go index 4426613a2..985a40247 100644 --- a/src/k8s/cmd/k8s/k8s_disable_dns.go +++ b/src/k8s/cmd/k8s/k8s_disable_dns.go @@ -25,8 +25,7 @@ var disableDNSCmd = &cobra.Command{ Status: api.ComponentDisable, } - err = client.UpdateDNSComponent(cmd.Context(), request) - if err != nil { + if err := client.UpdateDNSComponent(cmd.Context(), request); err != nil { return fmt.Errorf("failed to disable DNS component: %w", err) } diff --git a/src/k8s/cmd/k8s/k8s_disable_gateway.go b/src/k8s/cmd/k8s/k8s_disable_gateway.go new file mode 100644 index 000000000..522f7297b --- /dev/null +++ b/src/k8s/cmd/k8s/k8s_disable_gateway.go @@ -0,0 +1,35 @@ +package k8s + +import ( + "fmt" + + api "github.com/canonical/k8s/api/v1" + "github.com/canonical/k8s/pkg/k8s/client" + "github.com/spf13/cobra" +) + +var disableGatewayCmd = &cobra.Command{ + Use: "gateway", + Short: "Disable the Gateway component in the cluster.", + RunE: func(cmd *cobra.Command, args []string) error { + client, err := client.NewClient(cmd.Context(), client.ClusterOpts{ + StateDir: clusterCmdOpts.stateDir, + Verbose: rootCmdOpts.logVerbose, + Debug: rootCmdOpts.logDebug, + }) + if err != nil { + return fmt.Errorf("failed to create client: %w", err) + } + + request := api.UpdateGatewayComponentRequest{ + Status: api.ComponentDisable, + } + + if err := client.UpdateGatewayComponent(cmd.Context(), request); err != nil { + return fmt.Errorf("failed to disable Gateway component: %w", err) + } + + cmd.Println("Component 'Gateway' disabled") + return nil + }, +} diff --git a/src/k8s/cmd/k8s/k8s_disable_ingress.go b/src/k8s/cmd/k8s/k8s_disable_ingress.go new file mode 100644 index 000000000..17ffc9358 --- /dev/null +++ b/src/k8s/cmd/k8s/k8s_disable_ingress.go @@ -0,0 +1,35 @@ +package k8s + +import ( + "fmt" + + api "github.com/canonical/k8s/api/v1" + "github.com/canonical/k8s/pkg/k8s/client" + "github.com/spf13/cobra" +) + +var disableIngressCmd = &cobra.Command{ + Use: "ingress", + Short: "Disable the Ingress component in the cluster.", + RunE: func(cmd *cobra.Command, args []string) error { + client, err := client.NewClient(cmd.Context(), client.ClusterOpts{ + StateDir: clusterCmdOpts.stateDir, + Verbose: rootCmdOpts.logVerbose, + Debug: rootCmdOpts.logDebug, + }) + if err != nil { + return fmt.Errorf("failed to create client: %w", err) + } + + request := api.UpdateIngressComponentRequest{ + Status: api.ComponentDisable, + } + + if err := client.UpdateIngressComponent(cmd.Context(), request); err != nil { + return fmt.Errorf("failed to disable Ingress component: %w", err) + } + + cmd.Println("Component 'Ingress' disabled") + return nil + }, +} diff --git a/src/k8s/cmd/k8s/k8s_disable_loadbalancer.go b/src/k8s/cmd/k8s/k8s_disable_loadbalancer.go new file mode 100644 index 000000000..a53dcf125 --- /dev/null +++ b/src/k8s/cmd/k8s/k8s_disable_loadbalancer.go @@ -0,0 +1,35 @@ +package k8s + +import ( + "fmt" + + api "github.com/canonical/k8s/api/v1" + "github.com/canonical/k8s/pkg/k8s/client" + "github.com/spf13/cobra" +) + +var disableLoadBalancerCmd = &cobra.Command{ + Use: "loadbalancer", + Short: "Disable the LoadBalancer component in the cluster.", + RunE: func(cmd *cobra.Command, args []string) error { + client, err := client.NewClient(cmd.Context(), client.ClusterOpts{ + StateDir: clusterCmdOpts.stateDir, + Verbose: rootCmdOpts.logVerbose, + Debug: rootCmdOpts.logDebug, + }) + if err != nil { + return fmt.Errorf("failed to create client: %w", err) + } + + request := api.UpdateLoadBalancerComponentRequest{ + Status: api.ComponentDisable, + } + + if err := client.UpdateLoadBalancerComponent(cmd.Context(), request); err != nil { + return fmt.Errorf("failed to disable LoadBalancer component: %w", err) + } + + cmd.Println("Component 'LoadBalancer' disabled") + return nil + }, +} diff --git a/src/k8s/cmd/k8s/k8s_disable_network.go b/src/k8s/cmd/k8s/k8s_disable_network.go index f4f292b77..9286f09af 100644 --- a/src/k8s/cmd/k8s/k8s_disable_network.go +++ b/src/k8s/cmd/k8s/k8s_disable_network.go @@ -25,8 +25,7 @@ var disableNetworkCmd = &cobra.Command{ Status: api.ComponentDisable, } - err = client.UpdateNetworkComponent(cmd.Context(), request) - if err != nil { + if err := client.UpdateNetworkComponent(cmd.Context(), request); err != nil { return fmt.Errorf("failed to disable Network component: %w", err) } diff --git a/src/k8s/cmd/k8s/k8s_disable_storage.go b/src/k8s/cmd/k8s/k8s_disable_storage.go new file mode 100644 index 000000000..6e0abcbd1 --- /dev/null +++ b/src/k8s/cmd/k8s/k8s_disable_storage.go @@ -0,0 +1,35 @@ +package k8s + +import ( + "fmt" + + api "github.com/canonical/k8s/api/v1" + "github.com/canonical/k8s/pkg/k8s/client" + "github.com/spf13/cobra" +) + +var disableStorageCmd = &cobra.Command{ + Use: "storage", + Short: "Disable the Network component in the cluster.", + RunE: func(cmd *cobra.Command, args []string) error { + client, err := client.NewClient(cmd.Context(), client.ClusterOpts{ + StateDir: clusterCmdOpts.stateDir, + Verbose: rootCmdOpts.logVerbose, + Debug: rootCmdOpts.logDebug, + }) + if err != nil { + return fmt.Errorf("failed to create client: %w", err) + } + + request := api.UpdateStorageComponentRequest{ + Status: api.ComponentDisable, + } + + if err := client.UpdateStorageComponent(cmd.Context(), request); err != nil { + return fmt.Errorf("failed to disable Storage component: %w", err) + } + + cmd.Println("Component 'Storage' disabled") + return nil + }, +} diff --git a/src/k8s/cmd/k8s/k8s_enable.go b/src/k8s/cmd/k8s/k8s_enable.go index 956d7ec3b..eb06b0380 100644 --- a/src/k8s/cmd/k8s/k8s_enable.go +++ b/src/k8s/cmd/k8s/k8s_enable.go @@ -8,7 +8,7 @@ import ( ) var ( - componentList = []string{"network", "dns", "gateway", "ingress", "rbac", "storage"} + componentList = []string{"network", "dns", "gateway", "ingress", "rbac", "storage", "loadbalancer"} enableCmd = &cobra.Command{ Use: "enable ", @@ -26,4 +26,5 @@ func init() { enableCmd.AddCommand(enableStorageCmd) enableCmd.AddCommand(enableIngressCmd) enableCmd.AddCommand(enableGatewayCmd) + enableCmd.AddCommand(enableLoadBalancerCmd) } diff --git a/src/k8s/cmd/k8s/k8s_enable_dns.go b/src/k8s/cmd/k8s/k8s_enable_dns.go index bd1eaa84f..3c9a48edf 100644 --- a/src/k8s/cmd/k8s/k8s_enable_dns.go +++ b/src/k8s/cmd/k8s/k8s_enable_dns.go @@ -35,8 +35,7 @@ var enableDNSCmd = &cobra.Command{ }, } - err = client.UpdateDNSComponent(cmd.Context(), request) - if err != nil { + if err := client.UpdateDNSComponent(cmd.Context(), request); err != nil { return fmt.Errorf("failed to enable DNS component: %w", err) } diff --git a/src/k8s/cmd/k8s/k8s_enable_gateway.go b/src/k8s/cmd/k8s/k8s_enable_gateway.go index ee8c7772d..794df92b1 100644 --- a/src/k8s/cmd/k8s/k8s_enable_gateway.go +++ b/src/k8s/cmd/k8s/k8s_enable_gateway.go @@ -25,8 +25,7 @@ var enableGatewayCmd = &cobra.Command{ Status: api.ComponentEnable, } - err = client.UpdateGatewayComponent(cmd.Context(), request) - if err != nil { + if err := client.UpdateGatewayComponent(cmd.Context(), request); err != nil { return fmt.Errorf("failed to enable Storage component: %w", err) } diff --git a/src/k8s/cmd/k8s/k8s_enable_ingress.go b/src/k8s/cmd/k8s/k8s_enable_ingress.go index 57986e2d3..62b2d8f6b 100644 --- a/src/k8s/cmd/k8s/k8s_enable_ingress.go +++ b/src/k8s/cmd/k8s/k8s_enable_ingress.go @@ -33,8 +33,7 @@ var enableIngressCmd = &cobra.Command{ }, } - err = client.UpdateIngressComponent(cmd.Context(), request) - if err != nil { + if err := client.UpdateIngressComponent(cmd.Context(), request); err != nil { return fmt.Errorf("failed to enable Ingress component: %w", err) } diff --git a/src/k8s/cmd/k8s/k8s_enable_loadbalancer.go b/src/k8s/cmd/k8s/k8s_enable_loadbalancer.go new file mode 100644 index 000000000..714d06227 --- /dev/null +++ b/src/k8s/cmd/k8s/k8s_enable_loadbalancer.go @@ -0,0 +1,68 @@ +package k8s + +import ( + "fmt" + + api "github.com/canonical/k8s/api/v1" + "github.com/canonical/k8s/pkg/k8s/client" + "github.com/spf13/cobra" +) + +var enableLoadBalancerCmdOpts struct { + CIDRs []string + L2Enabled bool + L2Interfaces []string + BGPEnabled bool + BGPLocalASN int + BGPPeerAddress string + BGPPeerASN int + BGPPeerPort int +} + +var enableLoadBalancerCmd = &cobra.Command{ + Use: "loadbalancer", + Short: "Enable the LoadBalancer component in the cluster", + RunE: func(cmd *cobra.Command, args []string) error { + client, err := client.NewClient(cmd.Context(), client.ClusterOpts{ + StateDir: clusterCmdOpts.stateDir, + Verbose: rootCmdOpts.logVerbose, + Debug: rootCmdOpts.logDebug, + }) + if err != nil { + return fmt.Errorf("failed to create client: %w", err) + } + + request := api.UpdateLoadBalancerComponentRequest{ + Status: api.ComponentEnable, + Config: api.LoadBalancerComponentConfig{ + CIDRs: enableLoadBalancerCmdOpts.CIDRs, + L2Enabled: enableLoadBalancerCmdOpts.L2Enabled, + L2Interfaces: enableLoadBalancerCmdOpts.L2Interfaces, + BGPEnabled: enableLoadBalancerCmdOpts.BGPEnabled, + BGPPeerAddress: enableLoadBalancerCmdOpts.BGPPeerAddress, + BGPPeerASN: enableLoadBalancerCmdOpts.BGPPeerASN, + BGPPeerPort: enableLoadBalancerCmdOpts.BGPPeerPort, + }, + } + + if err := client.UpdateLoadBalancerComponent(cmd.Context(), request); err != nil { + return fmt.Errorf("failed to enable LoadBalancer component: %w", err) + } + + cmd.Println("Component 'LoadBalancer' enabled") + return nil + }, +} + +func init() { + enableLoadBalancerCmd.Flags().StringSliceVar(&enableLoadBalancerCmdOpts.CIDRs, "cidrs", []string{}, "List of CIDRs that will be used for LoadBalancer IP addresses.") + enableLoadBalancerCmd.MarkFlagRequired("cidrs") + enableLoadBalancerCmd.Flags().BoolVar(&enableLoadBalancerCmdOpts.L2Enabled, "l2-mode", true, "If set, L2 mode will be enabled for the LoadBalancer") + enableLoadBalancerCmd.Flags().StringSliceVar(&enableLoadBalancerCmdOpts.L2Interfaces, "l2-interfaces", []string{}, "List of interface names that will be used to announce services in L2 mode. All interfaces used by default.") + enableLoadBalancerCmd.Flags().BoolVar(&enableLoadBalancerCmdOpts.BGPEnabled, "bgp-mode", false, "If set, BGP mode will be enabled for the LoadBalancer") + enableLoadBalancerCmd.Flags().IntVar(&enableLoadBalancerCmdOpts.BGPLocalASN, "bgp-local-asn", 64512, "ASN number to use for the cluster's BGP router.") + enableLoadBalancerCmd.Flags().StringVar(&enableLoadBalancerCmdOpts.BGPPeerAddress, "bgp-peer-address", "", "Address(with slash notation) of the BGP peer.") + enableLoadBalancerCmd.Flags().IntVar(&enableLoadBalancerCmdOpts.BGPPeerASN, "bgp-peer-asn", 0, "ASN number of the BGP peer.") + enableLoadBalancerCmd.Flags().IntVar(&enableLoadBalancerCmdOpts.BGPPeerPort, "bgp-peer-port", 0, "Port number of the BGP peer.") + enableLoadBalancerCmd.MarkFlagsRequiredTogether("bgp-mode", "bgp-local-asn", "bgp-peer-address", "bgp-peer-asn", "bgp-peer-port") +} diff --git a/src/k8s/cmd/k8s/k8s_enable_network.go b/src/k8s/cmd/k8s/k8s_enable_network.go index 8719ad26b..cfaf4a59d 100644 --- a/src/k8s/cmd/k8s/k8s_enable_network.go +++ b/src/k8s/cmd/k8s/k8s_enable_network.go @@ -25,8 +25,7 @@ var enableNetworkCmd = &cobra.Command{ Status: api.ComponentEnable, } - err = client.UpdateNetworkComponent(cmd.Context(), request) - if err != nil { + if err := client.UpdateNetworkComponent(cmd.Context(), request); err != nil { return fmt.Errorf("failed to enable Network component: %w", err) } diff --git a/src/k8s/cmd/k8s/k8s_enable_storage.go b/src/k8s/cmd/k8s/k8s_enable_storage.go index cd4567008..58746a2c6 100644 --- a/src/k8s/cmd/k8s/k8s_enable_storage.go +++ b/src/k8s/cmd/k8s/k8s_enable_storage.go @@ -25,8 +25,7 @@ var enableStorageCmd = &cobra.Command{ Status: api.ComponentEnable, } - err = client.UpdateStorageComponent(cmd.Context(), request) - if err != nil { + if err := client.UpdateStorageComponent(cmd.Context(), request); err != nil { return fmt.Errorf("failed to enable Storage component: %w", err) } diff --git a/src/k8s/pkg/component/loadbalancer.go b/src/k8s/pkg/component/loadbalancer.go new file mode 100644 index 000000000..1b0c9f746 --- /dev/null +++ b/src/k8s/pkg/component/loadbalancer.go @@ -0,0 +1,123 @@ +package component + +import ( + "context" + "fmt" + "time" + + "github.com/canonical/k8s/pkg/snap" + "github.com/canonical/k8s/pkg/utils/k8s" +) + +func EnableLoadBalancerComponent(s snap.Snap, cidrs []string, l2Enabled bool, l2Interfaces []string, bgpEnabled bool, bgpLocalASN int, bgpPeerAddress string, bgpPeerASN int, bgpPeerPort int) error { + manager, err := NewHelmClient(s, nil) + if err != nil { + return fmt.Errorf("failed to get component manager: %w", err) + } + + networkValues := map[string]any{ + "l2announcements": map[string]any{ + "enabled": l2Enabled, + }, + "bgpControlPlane": map[string]any{ + "enabled": bgpEnabled, + }, + "externalIPs": map[string]any{ + "enabled": true, + }, + // https://docs.cilium.io/en/v1.14/network/l2-announcements/#sizing-client-rate-limit + // Assuming for 50 LB services + "k8sClientRateLimit": map[string]any{ + "qps": 10, + "burst": 20, + }, + } + + if err := manager.Refresh("network", networkValues); err != nil { + return fmt.Errorf("failed to enable ingress component: %w", err) + } + + formattedCidrs := []map[string]any{} + + for _, cidr := range cidrs { + formattedCidrs = append(formattedCidrs, map[string]any{"cidr": cidr}) + } + + values := map[string]any{ + "l2": map[string]any{ + "enabled": l2Enabled, + "interfaces": l2Interfaces, + }, + "ipPool": map[string]any{ + "cidrs": formattedCidrs, + }, + "bgp": map[string]any{ + "enabled": bgpEnabled, + "localASN": bgpLocalASN, + "neighbors": []map[string]any{ + map[string]any{ + "peerAddress": bgpPeerAddress, + "peerASN": bgpPeerASN, + "peerPort": bgpPeerPort, + }, + }, + }, + } + + if err := manager.Enable("loadbalancer", values); err != nil { + return fmt.Errorf("failed to enable loadbalancer component: %w", err) + } + + client, err := k8s.NewClient() + if err != nil { + return fmt.Errorf("failed to create kubernetes client: %w", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + if err := k8s.RestartDeployment(ctx, client, "cilium-operator", "kube-system"); err != nil { + return fmt.Errorf("failed to restart cilium-operator deployment: %w", err) + } + + if err := k8s.RestartDaemonset(ctx, client, "cilium", "kube-system"); err != nil { + return fmt.Errorf("failed to restart cilium-operator deployment: %w", err) + } + + return nil +} + +func DisableLoadBalancerComponent(s snap.Snap) error { + manager, err := NewHelmClient(s, nil) + if err != nil { + return fmt.Errorf("failed to get component manager: %w", err) + } + + if err := manager.Disable("loadbalancer"); err != nil { + return fmt.Errorf("failed to disable loadbalancer component: %w", err) + } + + networkValues := map[string]any{ + "l2announcements": map[string]any{ + "enabled": false, + }, + "bgpControlPlane": map[string]any{ + "enabled": false, + }, + "externalIPs": map[string]any{ + "enabled": false, + }, + // https://docs.cilium.io/en/v1.14/network/l2-announcements/#sizing-client-rate-limit + // Setting back to defaults + "k8sClientRateLimit": map[string]any{ + "qps": 5, + "burst": 10, + }, + } + + if err := manager.Refresh("network", networkValues); err != nil { + return fmt.Errorf("failed to disable ingress component: %w", err) + } + + return nil +} diff --git a/src/k8s/pkg/component/network.go b/src/k8s/pkg/component/network.go index 7d4a44224..f52109596 100644 --- a/src/k8s/pkg/component/network.go +++ b/src/k8s/pkg/component/network.go @@ -58,10 +58,6 @@ func EnableNetworkComponent(s snap.Snap) error { "nodePort": map[string]any{ "enabled": true, }, - - "l2announcements": map[string]any{ - "enabled": true, - }, } if s.IsStrict() { diff --git a/src/k8s/pkg/k8s/client/component.go b/src/k8s/pkg/k8s/client/component.go index 670182282..13e67c86e 100644 --- a/src/k8s/pkg/k8s/client/component.go +++ b/src/k8s/pkg/k8s/client/component.go @@ -82,3 +82,14 @@ func (c *Client) UpdateGatewayComponent(ctx context.Context, request api.UpdateG } return nil } + +func (c *Client) UpdateLoadBalancerComponent(ctx context.Context, request api.UpdateLoadBalancerComponentRequest) error { + queryCtx, cancel := context.WithTimeout(ctx, time.Second*30) + defer cancel() + + var response api.UpdateLoadBalancerComponentResponse + if err := c.mc.Query(queryCtx, "PUT", lxdApi.NewURL().Path("k8sd", "components", "loadbalancer"), request, &response); err != nil { + return fmt.Errorf("failed to enable loadbalancer component: %w", err) + } + return nil +} diff --git a/src/k8s/pkg/k8sd/api/component.go b/src/k8s/pkg/k8sd/api/component.go index 8653f385b..ad32ef143 100644 --- a/src/k8s/pkg/k8sd/api/component.go +++ b/src/k8s/pkg/k8sd/api/component.go @@ -147,3 +147,37 @@ func putGatewayComponent(s *state.State, r *http.Request) response.Response { return response.SyncResponse(true, &api.UpdateGatewayComponentResponse{}) } + +func putLoadBalancerComponent(s *state.State, r *http.Request) response.Response { + var req api.UpdateLoadBalancerComponentRequest + snap := snap.SnapFromContext(s.Context) + + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + return response.SmartError(fmt.Errorf("failed to decode request: %w", err)) + } + + switch req.Status { + case api.ComponentEnable: + if err := component.EnableLoadBalancerComponent( + snap, + req.Config.CIDRs, + req.Config.L2Enabled, + req.Config.L2Interfaces, + req.Config.BGPEnabled, + req.Config.BGPLocalASN, + req.Config.BGPPeerAddress, + req.Config.BGPPeerASN, + req.Config.BGPPeerPort, + ); err != nil { + return response.SmartError(fmt.Errorf("failed to enable loadbalancer: %w", err)) + } + case api.ComponentDisable: + if err := component.DisableLoadBalancerComponent(snap); err != nil { + return response.SmartError(fmt.Errorf("failed to disable loadbalancer: %w", err)) + } + default: + return response.SmartError(fmt.Errorf("invalid component status %s", req.Status)) + } + + return response.SyncResponse(true, &api.UpdateLoadBalancerComponentResponse{}) +} diff --git a/src/k8s/pkg/k8sd/api/endpoints.go b/src/k8s/pkg/k8sd/api/endpoints.go index 0803f0acc..48ff60d3c 100644 --- a/src/k8s/pkg/k8sd/api/endpoints.go +++ b/src/k8s/pkg/k8sd/api/endpoints.go @@ -71,6 +71,11 @@ var Endpoints = []rest.Endpoint{ Path: "k8sd/components/gateway", Put: rest.EndpointAction{Handler: putGatewayComponent}, }, + { + Name: "LoadBalancerComponent", + Path: "k8sd/components/loadbalancer", + Put: rest.EndpointAction{Handler: putLoadBalancerComponent}, + }, // Kubernetes auth tokens and token review webhook for kube-apiserver { Name: "KubernetesAuthTokens", diff --git a/tests/e2e/templates/loadbalancer-test.yaml b/tests/e2e/templates/loadbalancer-test.yaml new file mode 100644 index 000000000..8703881d9 --- /dev/null +++ b/tests/e2e/templates/loadbalancer-test.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: my-nginx +spec: + selector: + matchLabels: + run: my-nginx + replicas: 1 + template: + metadata: + labels: + run: my-nginx + spec: + containers: + - name: my-nginx + image: nginx + ports: + - containerPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: my-nginx + labels: + run: my-nginx +spec: + type: LoadBalancer + ports: + - port: 80 + protocol: TCP + selector: + run: my-nginx diff --git a/tests/e2e/tests/test_loadbalancer.py b/tests/e2e/tests/test_loadbalancer.py new file mode 100644 index 000000000..65982e110 --- /dev/null +++ b/tests/e2e/tests/test_loadbalancer.py @@ -0,0 +1,162 @@ +# +# Copyright 2024 Canonical, Ltd. +# +import ipaddress +import logging +from pathlib import Path +from typing import List + +import pytest +from e2e_util import harness, util +from e2e_util.config import MANIFESTS_DIR + +LOG = logging.getLogger(__name__) + + +def get_default_cidr(instance: harness.Instance, instance_default_ip: str): + # ---- + # 1: lo inet 127.0.0.1/8 scope host lo ..... + # 28: eth0 inet 10.42.254.197/24 metric 100 brd 10.42.254.255 scope global dynamic eth0 .... + # ---- + # Fetching the cidr for the default interface by matching with instance ip from the output + p = instance.exec(["ip", "-o", "-f", "inet", "addr", "show"], capture_output=True) + out = p.stdout.decode().split(" ") + return [i for i in out if instance_default_ip in i][0] + + +def get_default_ip(instance: harness.Instance): + # --- + # default via 10.42.254.1 dev eth0 proto dhcp src 10.42.254.197 metric 100 + # --- + # Fetching the default IP address from the output, e.g. 10.42.254.197 + p = instance.exec( + ["ip", "-o", "-4", "route", "show", "to", "default"], capture_output=True + ) + return p.stdout.decode().split(" ")[8] + + +def find_suitable_cidr(parent_cidr: str, excluded_ips: List[str]): + net = ipaddress.IPv4Network(parent_cidr, False) + + # Starting from the first IP address from the parent cidr, + # we search for a /30 cidr block(4 total ips, 2 available) + # that doesn't contain the excluded ips to avoid collisions + # /30 because this is the smallest CIDR cilium hands out IPs from + for i in range(4, 255, 4): + lb_net = ipaddress.IPv4Network(f"{str(net[0]+i)}/30", False) + + contains_excluded = False + for excluded in excluded_ips: + if ipaddress.ip_address(excluded) in lb_net: + contains_excluded = True + break + + if contains_excluded: + continue + + return str(lb_net) + raise RuntimeError("Could not find a suitable CIDR for LoadBalancer services") + + +@pytest.mark.node_count(2) +def test_loadbalancer(instances: List[harness.Instance]): + instance = instances[0] + tester_instance = instances[1] + + instance_default_ip = get_default_ip(instance) + tester_instance_default_ip = get_default_ip(tester_instance) + + instance_default_cidr = get_default_cidr(instance, instance_default_ip) + + lb_cidr = find_suitable_cidr( + parent_cidr=instance_default_cidr, + excluded_ips=[instance_default_ip, tester_instance_default_ip], + ) + + instance.exec(["k8s", "enable", "loadbalancer", f'--cidrs="{lb_cidr}"']) + + util.stubbornly(retries=3, delay_s=1).on(instance).exec( + [ + "k8s", + "kubectl", + "wait", + "--for=condition=ready", + "pod", + "-n", + "kube-system", + "-l", + "io.cilium/app=operator", + "--timeout", + "180s", + ] + ) + + util.stubbornly(retries=3, delay_s=1).on(instance).exec( + [ + "k8s", + "kubectl", + "wait", + "--for=condition=ready", + "pod", + "-n", + "kube-system", + "-l", + "k8s-app=cilium", + "--timeout", + "180s", + ] + ) + + manifest = MANIFESTS_DIR / "loadbalancer-test.yaml" + instance.exec( + ["k8s", "kubectl", "apply", "-f", "-"], + input=Path(manifest).read_bytes(), + ) + + LOG.info("Waiting for nginx pod to show up...") + util.stubbornly(retries=5, delay_s=10).on(instance).until( + lambda p: "my-nginx" in p.stdout.decode() + ).exec(["k8s", "kubectl", "get", "pod", "-o", "json"]) + LOG.info("Nginx pod showed up.") + + util.stubbornly(retries=3, delay_s=1).on(instance).exec( + [ + "k8s", + "kubectl", + "wait", + "--for=condition=ready", + "pod", + "-l", + "run=my-nginx", + "--timeout", + "180s", + ] + ) + + util.stubbornly(retries=5, delay_s=2).on(instance).until( + lambda p: "my-nginx" in p.stdout.decode() + ).exec(["k8s", "kubectl", "get", "service", "-o", "json"]) + + p = ( + util.stubbornly(retries=5, delay_s=3) + .on(instance) + .until(lambda p: len(p.stdout.decode().replace("'", "")) > 0) + .exec( + [ + "k8s", + "kubectl", + "get", + "service", + "my-nginx", + "-o=jsonpath='{.status.loadBalancer.ingress[0].ip}'", + ], + ) + ) + service_ip = p.stdout.decode().replace("'", "") + + p = tester_instance.exec( + ["curl", service_ip], + capture_output=True, + ) + + assert "Welcome to nginx!" in p.stdout.decode()