From 1b8f3dc728ed8fc4fedb8e95b33091afc8b7fb0c Mon Sep 17 00:00:00 2001 From: Lucian Petrut Date: Tue, 31 Dec 2024 07:47:36 +0000 Subject: [PATCH 1/5] wip: Add ipv6 load balancer tests We're adjusting the load balancer tests and related utils to also cover ipv6. This is currently wip as we're hitting the following Cilium issue: * https://github.com/cilium/cilium/issues/15082 * https://github.com/cilium/cilium/issues/17240 --- tests/integration/tests/test_loadbalancer.py | 29 ++++++++++++--- tests/integration/tests/test_util/util.py | 38 +++++++++++++++----- 2 files changed, 54 insertions(+), 13 deletions(-) diff --git a/tests/integration/tests/test_loadbalancer.py b/tests/integration/tests/test_loadbalancer.py index 7182fce6d..bed45a54c 100644 --- a/tests/integration/tests/test_loadbalancer.py +++ b/tests/integration/tests/test_loadbalancer.py @@ -14,13 +14,34 @@ @pytest.mark.node_count(2) @pytest.mark.tags(tags.PULL_REQUEST) -def test_loadbalancer(instances: List[harness.Instance]): - instance = instances[0] +@pytest.mark.disable_k8s_bootstrapping() +def test_loadbalancer_ipv4(instances: List[harness.Instance]): + _test_loadbalancer(instances, ipv6=False) + + +@pytest.mark.node_count(2) +@pytest.mark.tags(tags.PULL_REQUEST) +@pytest.mark.disable_k8s_bootstrapping() +@pytest.mark.network_type("dualstack") +def test_loadbalancer_ipv6(instances: List[harness.Instance]): + _test_loadbalancer(instances, ipv6=True) + +def _test_loadbalancer(instances: List[harness.Instance], ipv6=False): + instance = instances[0] tester_instance = instances[1] - instance_default_ip = util.get_default_ip(instance) - tester_instance_default_ip = util.get_default_ip(tester_instance) + if ipv6: + bootstrap_config = (MANIFESTS_DIR / "bootstrap-ipv6-only.yaml").read_text() + instance.exec( + ["k8s", "bootstrap", "--file", "-", "--address", "::/0"], + input=str.encode(bootstrap_config), + ) + else: + instance.exec(["k8s", "bootstrap"]) + + instance_default_ip = util.get_default_ip(instance, ipv6=ipv6) + tester_instance_default_ip = util.get_default_ip(tester_instance, ipv6=ipv6) instance_default_cidr = util.get_default_cidr(instance, instance_default_ip) diff --git a/tests/integration/tests/test_util/util.py b/tests/integration/tests/test_util/util.py index 3c98c91ad..1e2c6ddee 100644 --- a/tests/integration/tests/test_util/util.py +++ b/tests/integration/tests/test_util/util.py @@ -357,26 +357,40 @@ def join_cluster(instance: harness.Instance, join_token: str): instance.exec(["k8s", "join-cluster", join_token]) +def is_ipv6(ip: str) -> bool: + addr = ipaddress.ip_address(ip.strip("'")) + return isinstance(addr, ipaddress.IPv6Address) + + def get_default_cidr(instance: harness.Instance, instance_default_ip: str): # ---- # 1: lo inet 127.0.0.1/8 scope host lo ..... # 28: eth0 inet 10.42.254.197/24 metric 100 brd 10.42.254.255 scope global dynamic eth0 .... # ---- # Fetching the cidr for the default interface by matching with instance ip from the output - p = instance.exec(["ip", "-o", "-f", "inet", "addr", "show"], capture_output=True) + addr_family = "-6" if is_ipv6(instance_default_ip) else "-4" + p = instance.exec(["ip", "-o", addr_family, "addr", "show"], capture_output=True) out = p.stdout.decode().split(" ") return [i for i in out if instance_default_ip in i][0] -def get_default_ip(instance: harness.Instance): +def get_default_ip(instance: harness.Instance, ipv6=False): # --- # default via 10.42.254.1 dev eth0 proto dhcp src 10.42.254.197 metric 100 # --- # Fetching the default IP address from the output, e.g. 10.42.254.197 - p = instance.exec( - ["ip", "-o", "-4", "route", "show", "to", "default"], capture_output=True - ) - return p.stdout.decode().split(" ")[8] + addr_family = "-6" if ipv6 else "-4" + if ipv6: + p = instance.exec( + ["ip", "-json", "-6", "addr", "show", "scope", "global"], capture_output=True + ) + addr_json = json.loads(p.stdout.decode()) + return addr_json[0]['addr_info'][0]['local'] + else: + p = instance.exec( + ["ip", "-o", "-4", "route", "show", "to", "default"], capture_output=True + ) + return p.stdout.decode().split(" ")[8] def get_global_unicast_ipv6(instance: harness.Instance, interface="eth0") -> str | None: @@ -519,14 +533,20 @@ def previous_track(snap_version: str) -> str: def find_suitable_cidr(parent_cidr: str, excluded_ips: List[str]): """Find a suitable CIDR for LoadBalancer services""" - net = ipaddress.IPv4Network(parent_cidr, False) + net = ipaddress.ip_network(parent_cidr, False) + ipv6 = isinstance(net, ipaddress.IPv6Network) + if ipv6: + ip_range = 124 + else: + ip_range = 30 # Starting from the first IP address from the parent cidr, # we search for a /30 cidr block(4 total ips, 2 available) # that doesn't contain the excluded ips to avoid collisions - # /30 because this is the smallest CIDR cilium hands out IPs from + # /30 because this is the smallest CIDR cilium hands out IPs from. + # For ipv6, we use a /124 block that contains 16 total ips. for i in range(4, 255, 4): - lb_net = ipaddress.IPv4Network(f"{str(net[0]+i)}/30", False) + lb_net = ipaddress.ip_network(f"{str(net[0]+i)}/{ip_range}", False) contains_excluded = False for excluded in excluded_ips: From 157bf4a3b877f9dc878ac54dd150ba27c2e37c4b Mon Sep 17 00:00:00 2001 From: Lucian Petrut Date: Fri, 3 Jan 2025 08:02:11 +0000 Subject: [PATCH 2/5] Add ipv6 dualstack lb test --- tests/integration/tests/test_loadbalancer.py | 57 +++++++++++++++----- tests/integration/tests/test_util/util.py | 6 +-- 2 files changed, 47 insertions(+), 16 deletions(-) diff --git a/tests/integration/tests/test_loadbalancer.py b/tests/integration/tests/test_loadbalancer.py index bed45a54c..f53d040ba 100644 --- a/tests/integration/tests/test_loadbalancer.py +++ b/tests/integration/tests/test_loadbalancer.py @@ -22,36 +22,67 @@ def test_loadbalancer_ipv4(instances: List[harness.Instance]): @pytest.mark.node_count(2) @pytest.mark.tags(tags.PULL_REQUEST) @pytest.mark.disable_k8s_bootstrapping() -@pytest.mark.network_type("dualstack") -def test_loadbalancer_ipv6(instances: List[harness.Instance]): +def test_loadbalancer_ipv6_only(instances: List[harness.Instance]): + pytest.xfail( + "Cilium ipv6 only unsupported: https://github.com/cilium/cilium/issues/15082" + ) _test_loadbalancer(instances, ipv6=True) -def _test_loadbalancer(instances: List[harness.Instance], ipv6=False): +@pytest.mark.node_count(2) +@pytest.mark.tags(tags.PULL_REQUEST) +@pytest.mark.disable_k8s_bootstrapping() +@pytest.mark.dualstack() +@pytest.mark.network_type("dualstack") +def test_loadbalancer_ipv6_dualstack(instances: List[harness.Instance]): + _test_loadbalancer(instances, ipv6=True, dualstack=True) + + +def _test_loadbalancer(instances: List[harness.Instance], ipv6=False, dualstack=False): instance = instances[0] tester_instance = instances[1] if ipv6: - bootstrap_config = (MANIFESTS_DIR / "bootstrap-ipv6-only.yaml").read_text() + bootstrap_args = [] + if dualstack: + bootstrap_config = (MANIFESTS_DIR / "bootstrap-dualstack.yaml").read_text() + else: + bootstrap_config = (MANIFESTS_DIR / "bootstrap-ipv6-only.yaml").read_text() + bootstrap_args += ["--address", "::/0"] instance.exec( - ["k8s", "bootstrap", "--file", "-", "--address", "::/0"], + ["k8s", "bootstrap", "--file", "-", *bootstrap_args], input=str.encode(bootstrap_config), ) else: instance.exec(["k8s", "bootstrap"]) - instance_default_ip = util.get_default_ip(instance, ipv6=ipv6) - tester_instance_default_ip = util.get_default_ip(tester_instance, ipv6=ipv6) + lb_cidrs = [] - instance_default_cidr = util.get_default_cidr(instance, instance_default_ip) + def get_lb_cidr(ipv6_cidr): + instance_default_ip = util.get_default_ip(instance, ipv6=ipv6_cidr) + tester_instance_default_ip = util.get_default_ip( + tester_instance, ipv6=ipv6_cidr + ) + instance_default_cidr = util.get_default_cidr(instance, instance_default_ip) + lb_cidr = util.find_suitable_cidr( + parent_cidr=instance_default_cidr, + excluded_ips=[instance_default_ip, tester_instance_default_ip], + ) + return lb_cidr - lb_cidr = util.find_suitable_cidr( - parent_cidr=instance_default_cidr, - excluded_ips=[instance_default_ip, tester_instance_default_ip], - ) + if dualstack or not ipv6: + lb_cidrs.append(get_lb_cidr(ipv6_cidr=False)) + if ipv6: + lb_cidrs.append(get_lb_cidr(ipv6_cidr=True)) + lb_cidr_str = ",".join(lb_cidrs) instance.exec( - ["k8s", "set", f"load-balancer.cidrs={lb_cidr}", "load-balancer.l2-mode=true"] + [ + "k8s", + "set", + f"load-balancer.cidrs={lb_cidr_str}", + "load-balancer.l2-mode=true", + ] ) instance.exec(["k8s", "enable", "load-balancer"]) diff --git a/tests/integration/tests/test_util/util.py b/tests/integration/tests/test_util/util.py index 1e2c6ddee..4c3684262 100644 --- a/tests/integration/tests/test_util/util.py +++ b/tests/integration/tests/test_util/util.py @@ -379,13 +379,13 @@ def get_default_ip(instance: harness.Instance, ipv6=False): # default via 10.42.254.1 dev eth0 proto dhcp src 10.42.254.197 metric 100 # --- # Fetching the default IP address from the output, e.g. 10.42.254.197 - addr_family = "-6" if ipv6 else "-4" if ipv6: p = instance.exec( - ["ip", "-json", "-6", "addr", "show", "scope", "global"], capture_output=True + ["ip", "-json", "-6", "addr", "show", "scope", "global"], + capture_output=True, ) addr_json = json.loads(p.stdout.decode()) - return addr_json[0]['addr_info'][0]['local'] + return addr_json[0]["addr_info"][0]["local"] else: p = instance.exec( ["ip", "-o", "-4", "route", "show", "to", "default"], capture_output=True From 48dfc61063d3f91635c6609259610840297b4cae Mon Sep 17 00:00:00 2001 From: Lucian Petrut Date: Wed, 8 Jan 2025 08:54:16 +0000 Subject: [PATCH 3/5] Address PR comments * add a "network type" param to the lb test * move header bump to separate PR * update ipv6 cidr --- tests/integration/tests/test_loadbalancer.py | 38 ++++++++++++-------- tests/integration/tests/test_util/util.py | 6 ++-- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/tests/integration/tests/test_loadbalancer.py b/tests/integration/tests/test_loadbalancer.py index f53d040ba..386e1487b 100644 --- a/tests/integration/tests/test_loadbalancer.py +++ b/tests/integration/tests/test_loadbalancer.py @@ -2,6 +2,7 @@ # Copyright 2025 Canonical, Ltd. # import logging +from enum import Enum from pathlib import Path from typing import List @@ -12,11 +13,17 @@ LOG = logging.getLogger(__name__) +class K8sNetType(Enum): + ipv4 = "ipv4" + ipv6 = "ipv6" + dualstack = "dualstack" + + @pytest.mark.node_count(2) @pytest.mark.tags(tags.PULL_REQUEST) @pytest.mark.disable_k8s_bootstrapping() def test_loadbalancer_ipv4(instances: List[harness.Instance]): - _test_loadbalancer(instances, ipv6=False) + _test_loadbalancer(instances, k8s_net_type=K8sNetType.ipv4) @pytest.mark.node_count(2) @@ -26,7 +33,7 @@ def test_loadbalancer_ipv6_only(instances: List[harness.Instance]): pytest.xfail( "Cilium ipv6 only unsupported: https://github.com/cilium/cilium/issues/15082" ) - _test_loadbalancer(instances, ipv6=True) + _test_loadbalancer(instances, k8s_net_type=K8sNetType.ipv6) @pytest.mark.node_count(2) @@ -35,22 +42,23 @@ def test_loadbalancer_ipv6_only(instances: List[harness.Instance]): @pytest.mark.dualstack() @pytest.mark.network_type("dualstack") def test_loadbalancer_ipv6_dualstack(instances: List[harness.Instance]): - _test_loadbalancer(instances, ipv6=True, dualstack=True) + _test_loadbalancer(instances, k8s_net_type=K8sNetType.dualstack) -def _test_loadbalancer(instances: List[harness.Instance], ipv6=False, dualstack=False): +def _test_loadbalancer(instances: List[harness.Instance], k8s_net_type: K8sNetType): instance = instances[0] tester_instance = instances[1] - if ipv6: - bootstrap_args = [] - if dualstack: - bootstrap_config = (MANIFESTS_DIR / "bootstrap-dualstack.yaml").read_text() - else: - bootstrap_config = (MANIFESTS_DIR / "bootstrap-ipv6-only.yaml").read_text() - bootstrap_args += ["--address", "::/0"] + if k8s_net_type == K8sNetType.ipv6: + bootstrap_config = (MANIFESTS_DIR / "bootstrap-ipv6-only.yaml").read_text() + instance.exec( + ["k8s", "bootstrap", "--file", "-", "--address", "::/0"], + input=str.encode(bootstrap_config), + ) + elif k8s_net_type == K8sNetType.dualstack: + bootstrap_config = (MANIFESTS_DIR / "bootstrap-dualstack.yaml").read_text() instance.exec( - ["k8s", "bootstrap", "--file", "-", *bootstrap_args], + ["k8s", "bootstrap", "--file", "-"], input=str.encode(bootstrap_config), ) else: @@ -58,7 +66,7 @@ def _test_loadbalancer(instances: List[harness.Instance], ipv6=False, dualstack= lb_cidrs = [] - def get_lb_cidr(ipv6_cidr): + def get_lb_cidr(ipv6_cidr: bool): instance_default_ip = util.get_default_ip(instance, ipv6=ipv6_cidr) tester_instance_default_ip = util.get_default_ip( tester_instance, ipv6=ipv6_cidr @@ -70,9 +78,9 @@ def get_lb_cidr(ipv6_cidr): ) return lb_cidr - if dualstack or not ipv6: + if k8s_net_type in (K8sNetType.ipv4, K8sNetType.dualstack): lb_cidrs.append(get_lb_cidr(ipv6_cidr=False)) - if ipv6: + if k8s_net_type in (K8sNetType.ipv6, K8sNetType.dualstack): lb_cidrs.append(get_lb_cidr(ipv6_cidr=True)) lb_cidr_str = ",".join(lb_cidrs) diff --git a/tests/integration/tests/test_util/util.py b/tests/integration/tests/test_util/util.py index 4c3684262..52e6e8a95 100644 --- a/tests/integration/tests/test_util/util.py +++ b/tests/integration/tests/test_util/util.py @@ -358,7 +358,7 @@ def join_cluster(instance: harness.Instance, join_token: str): def is_ipv6(ip: str) -> bool: - addr = ipaddress.ip_address(ip.strip("'")) + addr = ipaddress.ip_address(ip) return isinstance(addr, ipaddress.IPv6Address) @@ -536,7 +536,7 @@ def find_suitable_cidr(parent_cidr: str, excluded_ips: List[str]): net = ipaddress.ip_network(parent_cidr, False) ipv6 = isinstance(net, ipaddress.IPv6Network) if ipv6: - ip_range = 124 + ip_range = 126 else: ip_range = 30 @@ -544,7 +544,7 @@ def find_suitable_cidr(parent_cidr: str, excluded_ips: List[str]): # we search for a /30 cidr block(4 total ips, 2 available) # that doesn't contain the excluded ips to avoid collisions # /30 because this is the smallest CIDR cilium hands out IPs from. - # For ipv6, we use a /124 block that contains 16 total ips. + # For ipv6, we use a /126 block that contains 4 total ips. for i in range(4, 255, 4): lb_net = ipaddress.ip_network(f"{str(net[0]+i)}/{ip_range}", False) From b94ded9923c0d0279e52ff627946e735dea67392 Mon Sep 17 00:00:00 2001 From: Lucian Petrut Date: Fri, 10 Jan 2025 10:57:20 +0000 Subject: [PATCH 4/5] Apply Docker workaround for all lxd bridges, including ipv6 --- .github/actions/install-lxd/action.yaml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/actions/install-lxd/action.yaml b/.github/actions/install-lxd/action.yaml index a24800774..12c96019a 100644 --- a/.github/actions/install-lxd/action.yaml +++ b/.github/actions/install-lxd/action.yaml @@ -32,5 +32,16 @@ runs: - name: Apply Docker iptables workaround shell: bash run: | - sudo iptables -I DOCKER-USER -i lxdbr0 -j ACCEPT - sudo iptables -I DOCKER-USER -o lxdbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + set -x + ip a + ip r + + bridges=('lxdbr0' 'dualstack-br0' 'ipv6-br0') + for i in ${bridges[@]}; do + set +e + sudo iptables -I DOCKER-USER -i $i -j ACCEPT + sudo ip6tables -I DOCKER-USER -i $i -j ACCEPT + sudo iptables -I DOCKER-USER -o $i -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + sudo ip6tables -I DOCKER-USER -o $i -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + set -e + done From e9c0e491bcc067def12af47a70b9b85f46f788df Mon Sep 17 00:00:00 2001 From: Lucian Petrut Date: Mon, 13 Jan 2025 08:16:55 +0000 Subject: [PATCH 5/5] Disable "test_strict_interfaces" The "strict" test jobs are currently disabled, except for "test_strict_interfaces", which is now failing. We'll disable this test as well until we get to fix the "strict" channel, which is currently low-prio. --- tests/integration/tests/test_strict_interfaces.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/tests/test_strict_interfaces.py b/tests/integration/tests/test_strict_interfaces.py index fb722f67f..101e75503 100644 --- a/tests/integration/tests/test_strict_interfaces.py +++ b/tests/integration/tests/test_strict_interfaces.py @@ -17,6 +17,8 @@ ) @pytest.mark.tags(tags.WEEKLY) def test_strict_interfaces(instances: List[harness.Instance], tmp_path): + pytest.xfail("Strict channel tests are currently skipped.") + channels = config.STRICT_INTERFACE_CHANNELS cp = instances[0] current_channel = channels[0]