Skip to content

Commit

Permalink
Add initial support for extra OVN cluster per node for ovn-bgp-agent
Browse files Browse the repository at this point in the history
  • Loading branch information
luis5tb committed Nov 17, 2023
1 parent e327d3a commit c793676
Show file tree
Hide file tree
Showing 22 changed files with 515 additions and 5 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
---
{% set mtu_list = [ctlplane_mtu] %}
{% for network in role_networks %}
{{ mtu_list.append(lookup('vars', networks_lower[network] ~ '_mtu')) }}
{%- endfor %}
{% set min_viable_mtu = mtu_list | max %}
network_config:
- type: interface
name: nic1
mtu: {{ ctlplane_mtu }}
dns_servers: {{ ctlplane_dns_nameservers }}
domain: {{ dns_search_domains }}
use_dhcp: false
addresses:
- ip_netmask: {{ ctlplane_ip }}/{{ ctlplane_subnet_cidr }}
{% for network in role_networks %}
- type: vlan
device: nic1
mtu: {{ lookup('vars', networks_lower[network] ~ '_mtu') }}
vlan_id: {{ lookup('vars', networks_lower[network] ~ '_vlan_id') }}
addresses:
- ip_netmask:
{{ lookup('vars', networks_lower[network] ~ '_ip') }}/{{ lookup('vars', networks_lower[network] ~ '_cidr') }}
routes: {{ lookup('vars', networks_lower[network] ~ '_host_routes') }}
{% endfor %}
- type: ovs_bridge
name: br-provider
use_dhcp: false
- type: ovs_bridge
name: {{ neutron_physical_bridge_name }}
mtu: {{ min_viable_mtu }}
use_dhcp: false
addresses:
- ip_netmask: {{ lookup('vars', 'bgp_net1_ip') }}/30
members:
- type: interface
name: nic2
mtu: {{ min_viable_mtu }}
# force the MAC address of the bridge to this interface
primary: true
- type: ovs_bridge
name: {{ neutron_physical_bridge_name }}-2
mtu: {{ min_viable_mtu }}
use_dhcp: false
addresses:
- ip_netmask: {{ lookup('vars', 'bgp_net2_ip') }}/30
members:
- type: interface
name: nic3
mtu: {{ min_viable_mtu }}
# force the MAC address of the bridge to this interface
primary: true
- type: interface
name: lo
addresses:
- ip_netmask: {{ lookup('vars', 'bgp_main_net_ip') }}/32
- ip_netmask: {{ lookup('vars', 'bgp_main_net6_ip') }}/128
70 changes: 68 additions & 2 deletions roles/edpm_ovn_bgp_agent/defaults/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ edpm_ovn_bgp_agent_debug: true
edpm_ovn_bgp_agent_reconcile_interval: 300
edpm_ovn_bgp_agent_expose_tenant_networks: false
edpm_ovn_bgp_agent_expose_ipv6_gua_tenant_networks: false
edpm_ovn_bgp_agent_driver: ovn_bgp_driver
edpm_ovn_bgp_agent_driver: nb_ovn_bgp_driver
edpm_ovn_bgp_agent_exposing_method: '' # default is 'underlay'
edpm_ovn_bgp_agent_private_key: /etc/pki/tls/private/ovn_bgp_agent.key
edpm_ovn_bgp_agent_certificate: /etc/pki/tls/certs/ovn_bgp_agent.crt
edpm_ovn_bgp_agent_ca_cert: /etc/ipa/ca.crt
Expand All @@ -40,14 +41,22 @@ edpm_ovn_bgp_agent_root_helper_daemon: "sudo ovn-bgp-agent-rootwrap-daemon /etc/
edpm_ovn_bgp_agent_ovsdb_connection: "tcp:127.0.0.1:6640"
edpm_ovn_bgp_agent_ovs_manager: "ptcp:6640:127.0.0.1"
edpm_ovn_bgp_agent_image: "quay.io/podified-antelope-centos9/openstack-ovn-bgp-agent:current-podified"
edpm_ovn_protocol: "{% if edpm_ovn_bgp_agent_internal_tls_enable | bool %}ssl{% else %}tcp{% endif %}"
edpm_ovn_bgp_agent_provider_networks_pool_prefixes: "192.168.0.0/16"


# optional parameters
edpm_ovn_bgp_agent_ovn_sb_connection: ''
edpm_ovn_bgp_agent_ovn_nb_connection: ''
edpm_ovn_bgp_agent_address_scopes: ''
edpm_ovn_bgp_agent_bgp_router_id: ''
edpm_ovn_bgp_agent_evpn_local_ip: ''
edpm_ovn_bgp_agent_evpn_nic: ''
edpm_ovn_bgp_agent_evpn_udp_dstport: '' # 4789

edpm_ovn_nb_dbs: []
edpm_ovn_nb_server_port: 6641

edpm_ovn_bgp_agent_common_volumes:
- /etc/hosts:/etc/hosts:ro
- /etc/localtime:/etc/localtime:ro
Expand All @@ -61,7 +70,10 @@ edpm_ovn_bgp_agent_common_volumes:
- /var/lib/kolla/config_files/ovn_bgp_agent.json:/var/lib/kolla/config_files/config.json:ro
- /var/lib/config-data/ansible-generated/ovn-bgp-agent:/var/lib/kolla/config_files/src:ro
- /run/frr:/run/frr:shared,z
- /run/openvswitch:/run/openvswitch:shared,z
- /run/openvswitch:/run/openvswitch:z
- /var/lib/openvswitch/ovn:/run/ovn:shared,z
- /var/log/containers/openvswitch:/var/log/openvswitch:z
- /var/log/containers/openvswitch:/var/log/ovn:z

edpm_ovn_bgp_agent_tls_volumes:
- /etc/pki/tls/certs/ovn_bgp_agent.crt:/etc/pki/tls/certs/ovn_bgp_agent.crt
Expand All @@ -74,3 +86,57 @@ edpm_ovn_bgp_agent_images_download_delay: 5

# number of retries for download tasks
edpm_ovn_bgp_agent_images_download_retries: 5

# ovn cluster per node parameters
# Enable or disable OVN routing for Datapath acceleration
edpm_ovn_bgp_agent_local_ovn_routing: false

edpm_ovn_bgp_agent_local_ovn_nb_connection: 'unix:/run/ovn/ovnnb_db.sock'
edpm_ovn_bgp_agent_local_ovn_sb_connection: 'unix:/run/ovn/ovnsb_db.sock'
edpm_ovn_bgp_agent_local_ovn_external_nics: []
edpm_ovn_bgp_agent_local_ovn_peer_ips: []

ovn_bgp_agent_external_nics: "{{ edpm_ovn_bgp_agent_local_ovn_external_nics | join(',') }}"
ovn_bgp_agent_peer_ips: "{{ edpm_ovn_bgp_agent_local_ovn_peer_ips | join(',') }}"

edpm_ovn_bgp_agent_local_ovn_nb_db_image: "quay.io/podified-antelope-centos9/openstack-ovn-nb-db-server:current-podified"
edpm_ovn_bgp_agent_local_ovn_sb_db_image: "quay.io/podified-antelope-centos9/openstack-ovn-sb-db-server:current-podified"
edpm_ovn_bgp_agent_local_ovn_northd_image: "quay.io/podified-antelope-centos9/openstack-ovn-northd:current-podified"
edpm_ovn_bgp_agent_local_ovn_controller_image: "quay.io/podified-antelope-centos9/openstack-ovn-controller:current-podified"

# Set external_id data from provided variables
edpm_ovn_bgp_agent_local_ovn_bridge_mappings:
- bgp-openstack:br-provider
- bgp-ex:br-ex
- bgp-ex-2:br-ex-2
edpm_ovn_bgp_agent_local_ovn_ovs_external_ids:
ovn-bridge-bgp: "br-bgp"
ovn-bridge-mappings-bgp: "{{ edpm_ovn_bgp_agent_local_ovn_bridge_mappings | join(',') }}"
ovn-remote-bgp: "unix:/run/ovn/ovnsb_db.sock"
ovn-nb-remote: >-
"{%- set db_addresses = [] -%}{%- for host in edpm_ovn_nb_dbs -%}
{{ db_addresses.append([edpm_ovn_protocol, host, edpm_ovn_nb_server_port] | join(':')) }}{%- endfor -%}
{{ db_addresses | join(',') }}"
edpm_ovn_bgp_agent_local_ovn_cluster_common_volumes:
- /lib/modules:/lib/modules:ro
- /run:/run
- /var/lib/openvswitch/ovn:/run/ovn:shared,z
- /var/log/containers/openvswitch:/var/log/openvswitch:z
- /var/log/containers/openvswitch:/var/log/ovn:z

edpm_ovn_bgp_agent_local_ovn_controller_volumes:
- /var/lib/kolla/config_files/bgp_ovn_controller.json:/var/lib/kolla/config_files/config.json:ro

edpm_ovn_bgp_agent_local_ovn_nb_volumes:
- /var/lib/kolla/config_files/nb_db_server.json:/var/lib/kolla/config_files/config.json:ro

edpm_ovn_bgp_agent_local_ovn_sb_volumes:
- /var/lib/kolla/config_files/sb_db_server.json:/var/lib/kolla/config_files/config.json:ro

edpm_ovn_bgp_agent_local_ovn_northd_volumes:
- /var/lib/kolla/config_files/northd.json:/var/lib/kolla/config_files/config.json:ro

edpm_ovn_bgp_agent_local_ovn_cluster_tls_volumes:
- /etc/pki/tls/certs/:/etc/pki/tls/certs/
- /etc/pki/tls/private/:/etc/pki/tls/private/
12 changes: 12 additions & 0 deletions roles/edpm_ovn_bgp_agent/files/ovn-bgp-agent-cleanup
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash
# Cleanup ovn-bgp-agent OVS bridges. To be called on startup to avoid
# "difficult-to-debug" issues with partially configured resources.

$BGP_BRIDGE=${INT_BRIDGE:-"br-bgp"}

for port in `ovs-vsctl list-ports ${BGP_BRIDGE}`; do
skip_cleanup=`ovs-vsctl --if-exists get Interface $port external_ids:skip_cleanup`
if ! [[ "x$skip_cleanup" == "x\"true\"" ]]; then
ovs-vsctl del-port ${BGP_BRIDGE} $port
fi
done
12 changes: 12 additions & 0 deletions roles/edpm_ovn_bgp_agent/files/ovn-bgp-agent-cleanup.service
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
[Unit]
Description=OVN BGP Agent cleanup on startup
After=openvswitch.service network.target
Before=edpm_ovn_bgp_agent.service
RefuseManualStop=yes

[Service]
Type=oneshot
ExecStart=/usr/libexec/ovn-bgp-agent-cleanup

[Install]
WantedBy=multi-user.target
11 changes: 11 additions & 0 deletions roles/edpm_ovn_bgp_agent/molecule/default/prepare.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,17 @@
- ansible.builtin.include_role:
name: osp.edpm.env_data

# needed due to regression in 1.11.1
# see https://github.com/containers/crun/issues/1338
# fixed with https://github.com/containers/crun/pull/1341
# we should wait to crun > 1.11.2
- name: Downgrade crun for 1.11.1 regression
become: true
ansible.builtin.dnf:
name: crun < 1.11
allow_downgrade: true
state: present

# The openvswitch kernel module needs to be loaded on the host
- name: Install and modprobe openvswitch
shell: |
Expand Down
10 changes: 10 additions & 0 deletions roles/edpm_ovn_bgp_agent/tasks/configure.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,16 @@
# License for the specific language governing permissions and limitations
# under the License.

- name: Ensure the Openvswitch package is installed
ansible.builtin.package:
name: openvswitch
state: present

- name: Ensure the OVS service is running
ansible.builtin.systemd:
name: openvswitch
state: started

- name: Configure OVN BGP agent
ansible.builtin.template:
src: ovn-bgp-agent.conf.j2
Expand Down
33 changes: 33 additions & 0 deletions roles/edpm_ovn_bgp_agent/tasks/configure_ovn.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
---
# Copyright 2023 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

- name: Ensure the Openvswitch package is installed
ansible.builtin.package:
name: openvswitch
state: present

- name: Ensure the OVS service is running
ansible.builtin.systemd:
name: openvswitch
state: started

- name: Configure OVS external_ids
ansible.builtin.shell: >
ovs-vsctl set open . {% for key, value in edpm_ovn_bgp_agent_local_ovn_ovs_external_ids.items() -%} external_ids:{{ key }}={{ value }} {% endfor %}
register: ovn_ovs_external_ids
changed_when: ovn_ovs_external_ids.rc == 0
failed_when: ovn_ovs_external_ids.rc != 0
when: edpm_ovn_bgp_agent_local_ovn_ovs_external_ids | length > 0
15 changes: 15 additions & 0 deletions roles/edpm_ovn_bgp_agent/tasks/download_cache.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,20 @@
---

- name: Download needed container images for in node ovn cluster
containers.podman.podman_image:
name: "{{ item }}"
loop:
- "{{ edpm_ovn_bgp_agent_local_ovn_nb_db_image }}"
- "{{ edpm_ovn_bgp_agent_local_ovn_sb_db_image }}"
- "{{ edpm_ovn_bgp_agent_local_ovn_northd_image }}"
- "{{ edpm_ovn_bgp_agent_local_ovn_controller_image }}"
become: true
register: edpm_ovn_cluster_ovn_bgp_agent_images_download
until: edpm_ovn_cluster_ovn_bgp_agent_images_download.failed == false
retries: "{{ edpm_ovn_bgp_agent_images_download_retries }}"
delay: "{{ edpm_ovn_bgp_agent_images_download_delay }}"
when: edpm_ovn_bgp_agent_local_ovn_routing

- name: Download needed container images
containers.podman.podman_image:
name: "{{ edpm_ovn_bgp_agent_image }}"
Expand Down
53 changes: 53 additions & 0 deletions roles/edpm_ovn_bgp_agent/tasks/install_ovn.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
---
# Copyright 2023 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

- name: Create persistent directories for in-node ovn cluster
ansible.builtin.file:
path: "{{ item.path }}"
state: directory
setype: "{{ item.setype }}"
mode: "{{ item.mode | default(omit) }}"
loop:
- {'path': /var/log/containers/openvswitch, 'setype': container_file_t, 'mode': '0750'}
- {'path': /var/lib/openvswitch/ovn, 'setype': container_file_t}

- name: Enable virt_sandbox_use_netlink for healthcheck
ansible.posix.seboolean:
name: virt_sandbox_use_netlink
persistent: true
state: true
when:
- ansible_facts.selinux is defined
- ansible_facts.selinux.status == "enabled"

- name: Copy in cleanup script
ansible.builtin.copy:
src: ovn-bgp-agent-cleanup
dest: '/usr/libexec/ovn-bgp-agent-cleanup'
force: true
mode: '0755'

- name: Copy in cleanup service
ansible.builtin.copy:
src: ovn-bgp-agent-cleanup.service
dest: '/usr/lib/systemd/system/ovn-bgp-agent-cleanup.service'
force: true
mode: '0644'

- name: Enabling the cleanup service
ansible.builtin.service:
name: ovn-bgp-agent-cleanup
enabled: true
12 changes: 12 additions & 0 deletions roles/edpm_ovn_bgp_agent/tasks/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,23 @@
# License for the specific language governing permissions and limitations
# under the License.

- name: Include host prep tasks for ovn_cluster
ansible.builtin.import_tasks: install_ovn.yml
when: edpm_ovn_bgp_agent_local_ovn_routing

- name: Include host prep tasks
ansible.builtin.import_tasks: install.yml

- name: Configure in node ovn cluster for ovn_bgp_agent
ansible.builtin.import_tasks: configure_ovn.yml
when: edpm_ovn_bgp_agent_local_ovn_routing

- name: Configure ovn_bgp_agent
ansible.builtin.import_tasks: configure.yml

- name: Ensure in node ovn cluster for ovn_bgp_agent is running
ansible.builtin.import_tasks: run_ovn.yml
when: edpm_ovn_bgp_agent_local_ovn_routing

- name: Ensure ovn_bgp_agent is running
ansible.builtin.import_tasks: run.yml
Loading

0 comments on commit c793676

Please sign in to comment.