Target remote Incus host

README.md

@@ -24,9 +24,11 @@ Init the terraform project:
 tofu init
 ```
 
-Create the VMs for testing:
+If your Incus host needs different values from the default, you may need to copy `terraform.tfvars.example` to `terraform.tfvars` and update the variables.
+
+Create 5 VMs for testing with incus, ovn, and ceph cluster:
 ```
-tofu apply
+tofu apply -target=module.baremetal
 ```
 
 ### Run the Ansible Playbook
@@ -35,10 +37,19 @@ Go to the ansible directory:
 cd ../ansible/
 ```
 
+NOTE: If you need the same version of ansible this was tested with:
+```
+pyenv install 3.13.1
+pipenv --python "3.13.1" install
+pipenv shell
+ansible-galaxy install -r ansible_requirements.yml
+```
+
 Copy the example inventory file:
 ```
 cp hosts.yaml.example hosts.yaml
 ```
+NOTE: If you are connecting to a remote incus host you will need to change the `ansible_incus_remote` variable to match the name of the Incus remote (see: `incus remote list` for a list of remote names to use).
 
 Run the Playbooks:
 ```
@@ -50,6 +61,32 @@ destroy`), you need to make sure to also clear any local state from the
 `data` directory, failure to do so will cause Ceph/OVN to attempt
 connection to the previously deployed systems which will cause the
 deployment to get stuck.
+```
+rm ansible/data/ceph/*
+rm ansible/data/lvmcluster/*
+rm ansible/data/ovn/*
+```
+
+### Test a VM and Contrainer on the new Incus cluster
+
+```
+# Open a shell on one of the Incus cluster nodes
+incus exec server01 bash
+
+# List all instances
+incus list
+
+# Launch a system container
+incus launch images:ubuntu/22.04 ubuntu-container
+
+# Launch a virtual machine
+incus launch images:ubuntu/22.04 ubuntu-vm --vm
+
+# Launch an application container
+incus remote add oci-docker https://docker.io --protocol=oci
+incus launch oci-docker:hello-world --ephemeral --console
+incus launch oci-docker:nginx nginx-app-container
+```
 
 ## Deploying against production systems
 ### Requirements (when using Incus with both Ceph and OVN)

ansible/.gitignore

@@ -1,2 +1,3 @@
 data/*
 hosts.yaml
+Pipfile.lock

ansible/Pipfile

@@ -0,0 +1,14 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+jmespath = "*"
+ansible-core = "==2.18.1"
+
+[dev-packages]
+
+[requires]
+python_version = "3.13"
+python_full_version = "3.13.1"

ansible/ansible_requirements.yml

@@ -0,0 +1,9 @@
+---
+# Install Collections and Roles with Ansible Galaxy
+# ansible-galaxy install -r ansible_requirements.yml
+
+collections:
+  - name: community.crypto
+  - name: community.general
+
+roles:

ansible/books/ceph.yaml

@@ -1,83 +1,4 @@
 ---
-- name: Ceph - Generate cluster keys and maps
-  hosts: all
-  order: shuffle
-  gather_facts: yes
-  gather_subset:
-    - "default_ipv4"
-    - "default_ipv6"
-  vars:
-    task_fsid: "{{ ceph_fsid | default('') }}"
-    task_bootstrap_osd_keyring: ../data/ceph/cluster.{{ task_fsid }}.bootstrap-osd.keyring
-    task_client_admin_keyring: ../data/ceph/cluster.{{ task_fsid }}.client.admin.keyring
-    task_mon_keyring: ../data/ceph/cluster.{{ task_fsid }}.mon.keyring
-    task_mon_map: ../data/ceph/cluster.{{ task_fsid }}.mon.map
-    task_release: "{{ ceph_release | default('squid') }}"
-    task_roles: "{{ ceph_roles | default([]) }}"
-
-    task_release_majors:
-      luminous: 12
-      mimic: 13
-      nautilus: 14
-      octopus: 15
-      pacific: 16
-      quincy: 17
-      reef: 18
-      squid: 19
-  any_errors_fatal: true
-  tasks:
-    - name: Generate mon keyring
-      delegate_to: 127.0.0.1
-      shell:
-        cmd: ceph-authtool --create-keyring {{ task_mon_keyring }} --gen-key -n mon. --cap mon 'allow *'
-        creates: '{{ task_mon_keyring }}'
-      throttle: 1
-      when: 'task_fsid'
-
-    - name: Generate client.admin keyring
-      delegate_to: 127.0.0.1
-      shell:
-        cmd: ceph-authtool --create-keyring {{ task_client_admin_keyring }} --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'
-        creates: '{{ task_client_admin_keyring }}'
-      throttle: 1
-      notify: Add key to client.admin keyring
-      when: 'task_fsid'
-
-    - name: Generate bootstrap-osd keyring
-      delegate_to: 127.0.0.1
-      shell:
-        cmd: ceph-authtool --create-keyring {{ task_bootstrap_osd_keyring }} --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'
-        creates: '{{ task_bootstrap_osd_keyring }}'
-      throttle: 1
-      notify: Add key to bootstrap-osd keyring
-      when: 'task_fsid'
-
-    - name: Generate mon map
-      delegate_to: 127.0.0.1
-      shell:
-        cmd: monmaptool --create{% if task_release_majors[task_release] | default(None) %} --set-min-mon-release={{ task_release_majors[task_release] }}{% endif %} --fsid {{ task_fsid }} {{ task_mon_map }}
-        creates: '{{ task_mon_map }}'
-      throttle: 1
-      notify: Add nodes to mon map
-      when: 'task_fsid'
-
-  handlers:
-    - name: Add key to client.admin keyring
-      delegate_to: 127.0.0.1
-      shell:
-        cmd: ceph-authtool {{ task_mon_keyring }} --import-keyring {{ task_client_admin_keyring }}
-
-    - name: Add key to bootstrap-osd keyring
-      delegate_to: 127.0.0.1
-      shell:
-        cmd: ceph-authtool {{ task_mon_keyring }} --import-keyring {{ task_bootstrap_osd_keyring }}
-
-    - name: Add nodes to mon map
-      delegate_to: 127.0.0.1
-      shell:
-        cmd: monmaptool --add {{ item.name }} {{ item.ip }} {{ task_mon_map }}
-      loop: "{{ lookup('template', '../files/ceph/ceph.monitors.tpl') | from_yaml | default([]) }}"
-
 - name: Ceph - Add package repository
   hosts: all
   order: shuffle
@@ -191,6 +112,171 @@
         state: present
       when: '"rgw" in task_roles'
 
+- name: Ceph - Generate cluster keys and maps
+  hosts: server01
+  gather_facts: yes
+  gather_subset:
+    - "default_ipv4"
+    - "default_ipv6"
+  vars:
+    task_fsid: "{{ ceph_fsid | default('') }}"
+    task_bootstrap_osd_keyring: ../data/ceph/cluster.{{ task_fsid }}.bootstrap-osd.keyring
+    task_client_admin_keyring: ../data/ceph/cluster.{{ task_fsid }}.client.admin.keyring
+    task_mon_keyring: ../data/ceph/cluster.{{ task_fsid }}.mon.keyring
+    task_mon_map: ../data/ceph/cluster.{{ task_fsid }}.mon.map
+    task_release: "{{ ceph_release | default('squid') }}"
+    task_roles: "{{ ceph_roles | default([]) }}"
+
+    task_release_majors:
+      luminous: 12
+      mimic: 13
+      nautilus: 14
+      octopus: 15
+      pacific: 16
+      quincy: 17
+      reef: 18
+      squid: 19
+  any_errors_fatal: true
+  tasks:
+    - name: create /root/generate dir
+      ansible.builtin.file:
+        path: /root/generate
+        owner: root
+        group: root
+        mode: 0700
+        state: directory
+
+    - name: Generate mon keyring (control node stat)
+      delegate_to: 127.0.0.1
+      ansible.builtin.stat:
+        path: '{{ task_mon_keyring }}'
+      register: task_mon_keyring_stat
+    - name: Generate mon keyring (copy to remote)
+      ansible.builtin.copy:
+        src: '{{ task_mon_keyring }}'
+        dest: /root/generate/task_mon_keyring
+      when: task_mon_keyring_stat.stat.exists
+    - name: Generate mon keyring
+      ansible.builtin.shell:
+        cmd: ceph-authtool --create-keyring /root/generate/task_mon_keyring --gen-key -n mon. --cap mon 'allow *'
+        creates: /root/generate/task_mon_keyring
+      throttle: 1
+      when: 'task_fsid'
+    - name: Generate mon keyring (fetch back to control node)
+      ansible.builtin.fetch:
+        src: /root/generate/task_mon_keyring
+        dest: '{{ task_mon_keyring }}'
+        flat: true
+      when: not task_mon_keyring_stat.stat.exists
+
+    - name: Generate client.admin keyring (control node stat)
+      delegate_to: 127.0.0.1
+      ansible.builtin.stat:
+        path: '{{ task_client_admin_keyring }}'
+      register: task_client_admin_keyring_stat
+    - name: Generate client.admin keyring (copy to remote)
+      ansible.builtin.copy:
+        src: '{{ task_client_admin_keyring }}'
+        dest: /root/generate/task_client_admin_keyring
+      when: task_client_admin_keyring_stat.stat.exists
+    - name: Generate client.admin keyring
+      ansible.builtin.shell:
+        cmd: ceph-authtool --create-keyring /root/generate/task_client_admin_keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'
+        creates: /root/generate/task_client_admin_keyring
+      throttle: 1
+      notify: Add key to client.admin keyring
+      when: 'task_fsid'
+    - name: Generate client.admin keyring (fetch back to control node)
+      ansible.builtin.fetch:
+        src: /root/generate/task_client_admin_keyring
+        dest: '{{ task_client_admin_keyring }}'
+        flat: true
+      when: not task_client_admin_keyring_stat.stat.exists
+
+    - name: Generate bootstrap-osd keyring (control node stat)
+      delegate_to: 127.0.0.1
+      ansible.builtin.stat:
+        path: '{{ task_bootstrap_osd_keyring }}'
+      register: task_bootstrap_osd_keyring_stat
+    - name: Generate bootstrap-osd keyring (copy to remote)
+      ansible.builtin.copy:
+        src: '{{ task_bootstrap_osd_keyring }}'
+        dest: /root/generate/task_bootstrap_osd_keyring
+      when: task_bootstrap_osd_keyring_stat.stat.exists
+    - name: Generate bootstrap-osd keyring
+      ansible.builtin.shell:
+        cmd: ceph-authtool --create-keyring /root/generate/task_bootstrap_osd_keyring --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r'
+        creates: /root/generate/task_bootstrap_osd_keyring
+      throttle: 1
+      notify: Add key to bootstrap-osd keyring
+      when: 'task_fsid'
+    - name: Generate bootstrap-osd keyring (fetch back to control node)
+      ansible.builtin.fetch:
+        src: /root/generate/task_bootstrap_osd_keyring
+        dest: '{{ task_bootstrap_osd_keyring }}'
+        flat: true
+      when: not task_bootstrap_osd_keyring_stat.stat.exists
+
+    - name: Generate mon map (control node stat)
+      delegate_to: 127.0.0.1
+      ansible.builtin.stat:
+        path: '{{ task_mon_map }}'
+      register: task_mon_map_stat
+    - name: Generate mon map (copy to remote)
+      ansible.builtin.copy:
+        src: '{{ task_mon_map }}'
+        dest: /root/generate/task_mon_map
+      when: task_mon_map_stat.stat.exists
+    - name: Generate mon map
+      ansible.builtin.shell:
+        cmd: monmaptool --create{% if task_release_majors[task_release] | default(None) %} --set-min-mon-release={{ task_release_majors[task_release] }}{% endif %} --fsid {{ task_fsid }} /root/generate/task_mon_map
+        creates: /root/generate/task_mon_map
+      throttle: 1
+      notify: Add nodes to mon map
+      when: 'task_fsid'
+    - name: Generate mon map (fetch back to control node)
+      ansible.builtin.fetch:
+        src: /root/generate/task_mon_map
+        dest: '{{ task_mon_map }}'
+        flat: true
+      when: not task_mon_map_stat.stat.exists
+
+  handlers:
+    - name: Add key to client.admin keyring (remote)
+      # delegate_to: 127.0.0.1
+      ansible.builtin.shell:
+        cmd: ceph-authtool /root/generate/task_mon_keyring --import-keyring /root/generate/task_client_admin_keyring
+      listen: Add key to client.admin keyring
+    - name: Add key to client.admin keyring (fetch)
+      ansible.builtin.fetch:
+        src: /root/generate/task_mon_keyring
+        dest: '{{ task_mon_keyring }}'
+        flat: true
+      listen: Add key to client.admin keyring
+
+    - name: Add key to bootstrap-osd keyring (remote)
+      ansible.builtin.shell:
+        cmd: ceph-authtool /root/generate/task_mon_keyring --import-keyring /root/generate/task_bootstrap_osd_keyring
+      listen: Add key to bootstrap-osd keyring
+    - name: Add key to bootstrap-osd keyring (fetch)
+      ansible.builtin.fetch:
+        src: /root/generate/task_mon_keyring
+        dest: '{{ task_mon_keyring }}'
+        flat: true
+      listen: Add key to bootstrap-osd keyring
+
+    - name: Add nodes to mon map (remote)
+      ansible.builtin.shell:
+        cmd: monmaptool --add {{ item.name }} {{ item.ip }} /root/generate/task_mon_map
+      loop: "{{ lookup('template', '../files/ceph/ceph.monitors.tpl') | from_yaml | default([]) }}"
+      listen: Add nodes to mon map
+    - name: Add nodes to mon map (fetch)
+      ansible.builtin.fetch:
+        src: /root/generate/task_mon_map
+        dest: '{{ task_mon_map }}'
+        flat: true
+      listen: Add nodes to mon map
+
 - name: Ceph - Set up config and keyrings
   hosts: all
   order: shuffle

ansible/files/ceph/ceph.monitors.tpl

@@ -1,4 +1,4 @@
-{% for host in vars['ansible_play_hosts'] %}
+{% for host in groups['all'] %}
 {% if hostvars[host]['ceph_fsid'] == task_fsid and "mon" in hostvars[host]['ceph_roles'] %}
 - name: "{{ host }}"
   ip: "{{ hostvars[host]['ceph_ip_address'] | default(hostvars[host]['ansible_default_ipv6']['address'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}"

terraform/.gitignore

@@ -2,3 +2,6 @@
 .terraform.lock.hcl
 terraform.tfstate
 terraform.tfstate.backup
+
+*.tfvars
+!*.auto.tfvars

terraform/baremetal-incus/main.tf

@@ -17,9 +17,9 @@ resource "incus_network" "this" {
   description = "Network used to test incus-deploy (OVN uplink)"
 
   config = {
-    "ipv4.address" = "172.31.254.1/24"
+    "ipv4.address" = var.ovn_net_uplink_ipv4_address
     "ipv4.nat"     = "true"
-    "ipv6.address" = "fd00:1e4d:637d:1234::1/64"
+    "ipv6.address" = var.ovn_net_uplink_ipv6_address
     "ipv6.nat"     = "true"
   }
 }
@@ -48,9 +48,12 @@ resource "incus_profile" "this" {
     type = "nic"
     name = "eth0"
 
-    properties = {
-      "network" = "incusbr0"
+    properties = var.nic_method == "network" ? {
+      "network" = var.network_name
       "name"    = "eth0"
+      } : {
+      "nictype" = var.nictype
+      "parent"  = var.nictype_parent
     }
   }