Refactor libvirt to be host installed

This commit removed installation of of libvirt as a
container and converts to doing a host level install.

The molecule tests are enhanced to allow testing this and
we revert to using ceph config from /etc/ceph.

molecule/test-helpers/verify_systemd_unit.yaml

@@ -6,11 +6,13 @@
       ansible.builtin.stat:
         path: "/etc/systemd/system/{{ item.name }}"
       register: unit_exists
+      when: item.osp_service | default(true)
     - name: Assert systemd unit exists {{ item.name }}
       ansible.builtin.assert:
         that:
           - unit_exists.stat.exists
         fail_msg: "Systemd unit file for {{ item.name }} does not exist"
+      when: item.osp_service | default(true)
     - name: Check if systemd unit is enabled {{ item.name }}
       when: item.enabled | default(true)
       ansible.builtin.command: systemctl is-enabled {{ item.name }}
@@ -34,5 +36,5 @@
       when: item.running | default(true)
       ansible.builtin.assert:
         that:
-          - unit_running.stdout == "active"
+          - unit_running.stdout in "{{ item.active | default(['active']) | list }}"
         fail_msg: "Systemd unit {{ item.name }} is not running"

roles/edpm_libvirt/defaults/main.yml

@@ -20,16 +20,34 @@
 # All variables within this role should have a prefix of "edpm_libvirt"
 
 # seconds between retries for download tasks
-edpm_libvirt_image_download_delay: 5
-
+edpm_libvirt_download_delay: 5
 # number of retries for download tasks
-edpm_libvirt_image_download_retries: 5
-
-edpm_libvirt_image: "quay.io/podified-antelope-centos9/openstack-nova-libvirt:current-podified"
-edpm_libvirt_containers:
-  - "libvirt_virtlogd"
-  - "libvirt_virtnodedevd"
-  - "libvirt_virtproxyd"
-  - "libvirt_virtqemud"
-  - "libvirt_virtsecretd"
+edpm_libvirt_download_retries: 5
+# this sould map to the libvirt
+edpm_libvirt_services:
+  - virtlogd
+  - virtnodedevd
+  - virtproxyd
+  - virtqemud
+  - virtsecretd
+edpm_libvirt_packages:
+  # main libvirt packages
+  - libvirt
+  - libvirt-admin
+  - libvirt-client
+  - libvirt-daemon
+  # qemu pakcages
+  - qemu-kvm
+  - qemu-img
+  # libguestfs is need for config drive and ephermeral disks
+  - libguestfs
+  - libseccomp
+  # swtpm is needed for TPM support
+  - swtpm
+  - swtpm-tools
+  # edk2-ovmf is needed for UEFI support
+  - edk2-ovmf
+  # ceph-common is needed for ceph support to provide the ceph client lib
+  # for qemu and the ceph udev rules
+  - ceph-common
 edpm_libvirt_ceph_path: /var/lib/openstack/config/ceph/

roles/edpm_libvirt/handlers/main.yml

@@ -14,15 +14,15 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-- name: Restart libvirt containers
+- name: Restart libvirt services
   become: true
   ansible.builtin.systemd:
+    name: "{{ item }}.service"
     state: restarted
-    name: "edpm_{{ item }}.service"
-  loop: "{{ edpm_libvirt_containers }}"
+    daemon_reload: true
+  loop: "{{ edpm_libvirt_services }}"
   listen: "Restart libvirt"
 
-
 - name: Restart edpm_libvirt.target
   become: true
   ansible.builtin.systemd:

roles/edpm_libvirt/meta/argument_specs.yml

@@ -11,16 +11,28 @@ argument_specs:
       edpm_libvirt_image_download_retries:
         type: int
         default: 5
-      edpm_libvirt_image:
-        type: str
-        description: The image to use for the libvirt_virt* containers.
-        default: "quay.io/podified-antelope-centos9/openstack-nova-libvirt:current-podified"
-      edpm_libvirt_containers:
+      edpm_libvirt_services:
         type: list
-        description: The list of containers to run.
+        description: The list of libvirt services to run.
         default:
-          - "libvirt_virtlogd"
-          - "libvirt_virtnodedevd"
-          - "libvirt_virtproxyd"
-          - "libvirt_virtqemud"
-          - "libvirt_virtsecretd"
+          - "virtlogd"
+          - "virtnodedevd"
+          - "virtproxyd"
+          - "virtqemud"
+          - "virtsecretd"
+      edpm_libvirt_packages:
+        type: list
+        description: The list of packages to install for libvirt.
+        default:
+          - libvirt
+          - libvirt-admin
+          - libvirt-client
+          - libvirt-daemon
+          - qemu-kvm
+          - qemu-img
+          - libguestfs
+          - libseccomp
+          - swtpm
+          - swtpm-tools
+          - edk2-ovmf
+          - ceph-common

roles/edpm_libvirt/molecule/default/prepare.yml

@@ -14,16 +14,33 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-
-- name: Prepare
-  hosts: all
-  roles:
-    - role: ../../../../molecule/common/test_deps
-    - role: osp.edpm.env_data
-
 - name: Setup DUT
   hosts: all
+  pre_tasks:
+    - name: set basic user fact
+      set_fact:
+        ansible_user: "{{ lookup('env', 'USER') }}"
+      when:
+        - ansible_user is undefined
+    - name: set basic home fact
+      set_fact:
+        ansible_user_dir: "{{ lookup('env', 'HOME') }}"
+      when:
+        - ansible_user_dir is undefined
   tasks:
+    - name: Enable package repos
+      become: true
+      ansible.builtin.shell: |
+        set -euxo pipefail
+        pushd /var/tmp
+        curl -sL https://github.com/openstack-k8s-operators/repo-setup/archive/refs/heads/main.tar.gz | tar -xz
+        pushd repo-setup-main
+        python3 -m venv ./venv
+        PBR_VERSION=0.0.0 ./venv/bin/pip install ./
+        ./venv/bin/repo-setup current-podified -b antelope
+        popd
+        rm -rf repo-setup-main
+
     - name: set /etc/localtime
       become: true
       ansible.builtin.file:
@@ -35,10 +52,6 @@
       ansible.builtin.import_role:
         name: osp.edpm.edpm_timezone
 
-    - name: install podman
-      ansible.builtin.import_role:
-        name: osp.edpm.edpm_podman
-
     - name: Create firewall directory
       become: true
       ansible.builtin.file:

roles/edpm_libvirt/molecule/default/test-helpers/verify_ceph_secret.yaml

@@ -16,8 +16,7 @@
 
 - name: Get cephx key from libvirt_virtsecretd container by passing FSID
   become: true
-  ansible.builtin.command: |
-    podman exec libvirt_virtsecretd bash -c "virsh secret-get-value $FSID"
+  ansible.builtin.command: "virsh secret-get-value $FSID"
   environment:
     FSID: "{{ fsid.stdout }}"
   register: virsh_secret

roles/edpm_libvirt/molecule/default/test-helpers/verify_sockets_exits.yaml

@@ -0,0 +1,19 @@
+- name: Verify libvirt sockets are created with correct permissions {{ item }}
+  block:
+    - name: Check that libvirt sockets exist {{ item }}
+      become: true
+      ansible.builtin.stat:
+        path: "{{ item }}"
+      register: socket_exists
+    - name: Assert that libvirt sockets exist {{ item }}
+      ansible.builtin.assert:
+        that:
+          - socket_exists.stat.exists
+          - socket_exists.stat.issock
+        fail_msg: "libvirt socket {{ item }} does not exist"
+    - name: Assert that libvirt socket is owned by root and has libvirt group {{ item }}
+      ansible.builtin.assert:
+        that:
+          - socket_exists.stat.pw_name == 'root'
+          - socket_exists.stat.gr_name == 'libvirt'
+        fail_msg: "libvirt socket {{ item }} is not owned by root:libvirt"

roles/edpm_libvirt/molecule/default/verify.yml

@@ -13,52 +13,73 @@
         # common directories
         - "/etc/tmpfiles.d/"
         - "/var/lib/openstack"
-        - "/var/lib/openstack/config/containers"
-        - "/var/log/containers"
-        - "/var/log/containers/stdouts"
         # extrenal deps
         - "/var/lib/openstack/config/ceph"
         # libvirt directories
         - "/var/lib/libvirt"
-        - "/var/lib/openstack/config/libvirt"
-        - "/var/log/containers/libvirt"
+        - "/etc/libvirt"
     - name: ensure firewall is configured
       ansible.builtin.include_tasks: "test-helpers/verify_firewall.yaml"
     - name: ensure ceph secret is configured
       ansible.builtin.include_tasks: "test-helpers/verify_ceph_secret.yaml"
     - name: ensure systemd services are defined and functional
       ansible.builtin.include_tasks: "{{test_helper_dir}}/verify_systemd_unit.yaml"
       loop:
-        - { "name": "edpm_libvirt_virtlogd.service" }
-        - { "name": "edpm_libvirt_virtnodedevd.service" }
-        - { "name": "edpm_libvirt_virtproxyd.service" }
-        - { "name": "edpm_libvirt_virtqemud.service" }
-        - { "name": "edpm_libvirt_virtsecretd.service" }
-    - name: ensure podman container exists and are running
-      ansible.builtin.include_tasks: "{{test_helper_dir}}/verify_podman.yaml"
-      loop:
-        - "libvirt_virtlogd"
-        - "libvirt_virtnodedevd"
-        - "libvirt_virtproxyd"
-        - "libvirt_virtqemud"
-        - "libvirt_virtsecretd"
+        # rhel installed services have service units defiend via rpm in
+        # /usr/lib/systemd/system so disable the check for them them in
+        # /etc/systemd/system
+        - { "name": "virtlogd.service", "osp_service": false, "enabled": "indirect" }
+        - { "name": "virtnodedevd.service" , "osp_service": false, "enabled": "enabled", "active": ["active", "inactive"] }
+        - { "name": "virtproxyd.service", "osp_service": false, "enabled": "enabled", "active": ["active", "inactive"] }
+        - { "name": "virtqemud.service", "osp_service": false, "enabled": "enabled", "active": ["active", "inactive"] }
+        - { "name": "virtsecretd.service", "osp_service": false, "enabled": "enabled", "active": ["active", "inactive"] }
     - name: ensure libvirt.target exist and are running
       ansible.builtin.include_tasks: "{{test_helper_dir}}/verify_systemd_unit.yaml"
       loop:
         - { "name": "edpm_libvirt.target", "enabled": "static"}
         - { "name": "edpm_libvirt_guests.service", "running":false }
         - { "name": "virt-guest-shutdown.target", "enabled": "static", "running":false }
-
     - name: Check if user exists
       ansible.builtin.getent:
         database: passwd
         key: libvirt
       register: libvirt_user
-
     - name: Assert that libvirt user is created with kolla uid and gid
       ansible.builtin.assert:
         that:
           # user
           - "libvirt_user.ansible_facts.getent_passwd.libvirt[1] == '42473'"
           # group
           - "libvirt_user.ansible_facts.getent_passwd.libvirt[2] == '42473'"
+    - name: ensure libvirt sockets are enabled
+      ansible.builtin.include_tasks: "{{test_helper_dir}}/verify_systemd_unit.yaml"
+      loop:
+        - { "name": "virtlogd-admin.socket", "osp_service": false }
+        - { "name": "virtlogd.socket", "osp_service": false }
+        - { "name": "virtnodedevd-admin.socket", "osp_service": false }
+        - { "name": "virtnodedevd-ro.socket", "osp_service": false }
+        - { "name": "virtnodedevd.socket", "osp_service": false }
+        - { "name": "virtproxyd-admin.socket", "osp_service": false }
+        - { "name": "virtproxyd-ro.socket", "osp_service": false }
+        - { "name": "virtproxyd.socket", "osp_service": false }
+        - { "name": "virtqemud-admin.socket", "osp_service": false }
+        - { "name": "virtqemud-ro.socket", "osp_service": false }
+        - { "name": "virtqemud.socket", "osp_service": false }
+        - { "name": "virtsecretd-admin.socket", "osp_service": false }
+        - { "name": "virtsecretd-ro.socket", "osp_service": false }
+        - { "name": "virtsecretd.socket", "osp_service": false }
+
+    - name: ensure libvirt socket activation drop-in directories exist
+      ansible.builtin.include_tasks: "test-helpers/verify_sockets_exits.yaml"
+      loop:
+        - /run/libvirt/virtlogd-admin-sock
+        - /run/libvirt/virtlogd-sock
+        - /run/libvirt/virtnodedevd-admin-sock
+        - /run/libvirt/virtnodedevd-sock
+        - /run/libvirt/virtnodedevd-sock-ro
+        - /run/libvirt/virtqemud-sock
+        - /run/libvirt/virtqemud-sock-ro
+        - /run/libvirt/virtqemud-admin-sock
+        - /run/libvirt/virtsecretd-admin-sock
+        - /run/libvirt/virtsecretd-sock
+        - /run/libvirt/virtsecretd-sock-ro

roles/edpm_libvirt/tasks/configure.yml

@@ -13,54 +13,65 @@
     group: "{{ item.group | default(ansible_user) }}"
     mode: "{{ item.mode | default(omit) }}"
   loop:
-    - {"path": "/var/lib/openstack/config/libvirt"}
-    - {"path": "/var/lib/openstack/config/containers"}
-    - {'path': "/var/lib/nova"}
     - {"path": "/etc/tmpfiles.d/", "owner": "root", "group": "root"}
     - {"path": "/var/lib/edpm-config/firewall", "owner": "root", "group": "root"}
 
-- name: Create container config dirs
-  become: true
+- name: Render libvirt config files
   tags:
     - configure
     - libvirt
-  ansible.builtin.file:
-    path: "/var/lib/openstack/config/libvirt/{{ item }}"
-    state: directory
-    setype: "{{ item.setype | default('container_file_t') }}"
-    owner: "{{ item.owner | default(ansible_user) }}"
-    group: "{{ item.group | default(ansible_user) }}"
-    mode: "0755"
-  loop: "{{ edpm_libvirt_containers }}"
+  become: true
+  ansible.builtin.template:
+    src: "{{ item.src }}"
+    dest: "/etc/libvirt/{{ item.dest }}"
+    mode: "0644"
+    # FIXME: update to libvirt user/group
+    owner: "root"
+    group: "root"
+  loop:
+    - {"src": "virtlogd.conf", "dest": "virtlogd.conf"}
+    - {"src": "virtnodedevd.conf", "dest": "virtnodedevd.conf"}
+    - {"src": "virtproxyd.conf", "dest": "virtproxyd.conf"}
+    - {"src": "virtqemud.conf", "dest": "virtqemud.conf"}
+    - {"src": "qemu.conf", "dest": "qemu.conf"}
+    - {"src": "virtsecretd.conf", "dest": "virtsecretd.conf"}
+  notify:
+    - Restart libvirt
 
-- name: Render libvirt container config.json
+- name: Create libvirt socket activation drop-in directories
   tags:
     - configure
     - libvirt
-  ansible.builtin.template:
-    src: "{{ item }}/config.json.j2"
-    dest: "/var/lib/openstack/config/libvirt/{{ item }}/config.json"
-    setype: "container_file_t"
-    mode: "0644"
-  loop: "{{ edpm_libvirt_containers }}"
+  become: true
+  vars:
+    socket_suffix: ".socket.d"
+    socket_types: ['', '-ro', '-admin']
+  ansible.builtin.file:
+    path: "/etc/systemd/system/{{ item.0 }}{{ item.1 }}{{ socket_suffix }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  when: not (item.0 == 'virtlogd' and item.1 == '-ro')
+  loop: "{{ edpm_libvirt_services | product(socket_types) | list }}"
   notify:
     - Restart libvirt
 
-- name: Render libvirt config files
+- name: Render libvirt socket activation drop-in files
   tags:
     - configure
     - libvirt
+  become: true
+  vars:
+    socket_suffix: ".socket.d"
+    socket_types: ['', '-ro', '-admin']
   ansible.builtin.template:
-    src: "{{ item.src }}"
-    dest: "/var/lib/openstack/config/libvirt/{{ item.dest }}"
-    setype: "container_file_t"
+    src: "libvirt-socket.unit.j2"
+    dest: "/etc/systemd/system/{{ item.0 }}{{ item.1 }}{{ socket_suffix }}/override.conf"
     mode: "0644"
-  loop:
-    - {"src": "libvirt_virtlogd/virtlogd.conf", "dest": "libvirt_virtlogd/virtlogd.conf"}
-    - {"src": "libvirt_virtnodedevd/virtnodedevd.conf", "dest": "libvirt_virtnodedevd/virtnodedevd.conf"}
-    - {"src": "libvirt_virtproxyd/virtproxyd.conf", "dest": "libvirt_virtproxyd/virtproxyd.conf"}
-    - {"src": "libvirt_virtqemud/virtqemud.conf", "dest": "libvirt_virtqemud/virtqemud.conf"}
-    - {"src": "libvirt_virtqemud/qemu.conf", "dest": "libvirt_virtqemud/qemu.conf"}
-    - {"src": "libvirt_virtsecretd/virtsecretd.conf", "dest": "libvirt_virtsecretd/virtsecretd.conf"}
+    owner: "root"
+    group: "root"
+  when: not (item.0 == 'virtlogd' and item.1 == '-ro')
+  loop: "{{ edpm_libvirt_services | product(socket_types) | list }}"
   notify:
     - Restart libvirt