CI: Fetch DevStack and k3s logs (#2527)

This commit makes sure we're fetching and putting the logs of the
DevStack services as well as k3s into the artifacts for every CI job.

tests/ci-csi-cinder-e2e.sh

@@ -110,6 +110,18 @@ ansible-playbook -v \
   tests/playbooks/test-csi-cinder-e2e.yaml
 exit_code=$?
 
+# Fetch logs for debugging purpose
+ansible-playbook -v \
+  --user ${USERNAME} \
+  --private-key ~/.ssh/google_compute_engine \
+  --inventory ${PUBLIC_IP}, \
+  --ssh-common-args "-o StrictHostKeyChecking=no" \
+  tests/playbooks/fetch-logs.yaml
+
+ scp -i ~/.ssh/google_compute_engine \
+   -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
+   -r ${USERNAME}@${PUBLIC_IP}:~/logs $ARTIFACTS/logs/devstack || true
+
 # Fetch cinder-csi tests logs for debugging purpose
 scp -i ~/.ssh/google_compute_engine \
   -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \

tests/ci-csi-manila-e2e.sh

@@ -110,6 +110,18 @@ ansible-playbook -v \
   tests/playbooks/test-csi-manila-e2e.yaml
 exit_code=$?
 
+# Fetch logs for debugging purpose
+ansible-playbook -v \
+  --user ${USERNAME} \
+  --private-key ~/.ssh/google_compute_engine \
+  --inventory ${PUBLIC_IP}, \
+  --ssh-common-args "-o StrictHostKeyChecking=no" \
+  tests/playbooks/fetch-logs.yaml
+
+ scp -i ~/.ssh/google_compute_engine \
+   -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
+   -r ${USERNAME}@${PUBLIC_IP}:~/logs $ARTIFACTS/logs/devstack || true
+
 # Fetch manila-csi tests results
 scp -i ~/.ssh/google_compute_engine \
   -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \

tests/ci-occm-e2e.sh

@@ -113,10 +113,17 @@ ansible-playbook -v \
   -e run_e2e=true
 exit_code=$?
 
-# Fetch devstack logs for debugging purpose
-# scp -i ~/.ssh/google_compute_engine \
-#   -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
-#   -r ${USERNAME}@${PUBLIC_IP}:/opt/stack/logs $ARTIFACTS/logs/devstack || true
+# Fetch logs for debugging purpose
+ansible-playbook -v \
+  --user ${USERNAME} \
+  --private-key ~/.ssh/google_compute_engine \
+  --inventory ${PUBLIC_IP}, \
+  --ssh-common-args "-o StrictHostKeyChecking=no" \
+  tests/playbooks/fetch-logs.yaml
+
+ scp -i ~/.ssh/google_compute_engine \
+   -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
+   -r ${USERNAME}@${PUBLIC_IP}:~/logs $ARTIFACTS/logs/devstack || true
 
 # Fetch octavia amphora image build logs for debugging purpose
 scp -i ~/.ssh/google_compute_engine \

tests/playbooks/fetch-logs.yaml

@@ -0,0 +1,11 @@
+- hosts: all
+  become: true
+  become_method: sudo
+  gather_facts: true
+
+  vars:
+    user: stack
+    devstack_workdir: /home/{{ user }}/devstack
+
+  roles:
+    - role: fetch-logs

tests/playbooks/roles/fetch-logs/README.md

@@ -0,0 +1 @@
+The ansible role gets logs of various services running in the CI for further analysis

tests/playbooks/roles/fetch-logs/defaults/main.yaml

@@ -0,0 +1,2 @@
+---
+master_port_name: "k3s_master"

tests/playbooks/roles/fetch-logs/tasks/main.yaml

@@ -0,0 +1,34 @@
+- name: Get k3s master floating IP
+  shell:
+    executable: /bin/bash
+    cmd: |
+      set +x; source {{ devstack_workdir }}/openrc demo demo > /dev/null
+      openstack floating ip list --port {{ master_port_name }} -c "Floating IP Address" -f value
+  register: fip
+
+- name: Set fact for k3s master floating IP
+  set_fact:
+    k3s_fip: "{{ fip.stdout }}"
+
+- name: Creates directory
+  ansible.builtin.file:
+    path: "/root/logs"
+    state: directory
+
+- name: Fetch k3s logs
+  shell:
+    executable: /bin/bash
+    cmd: |
+      ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i {{ ansible_user_dir }}/.ssh/id_rsa ubuntu@{{ k3s_fip }} sudo journalctl -u k3s.service --no-pager > /root/logs/k3s.log
+
+- name: Fetch DevStack logs
+  shell:
+    executable: /bin/bash
+    cmd: |
+      set +x;
+      units=`systemctl list-units --type service | awk '{ print $1 }' | grep devstack\@`
+      for unit in $units; do
+        filename=${unit#"devstack@"}
+        filename=${filename%".service"}
+        sudo journalctl -u $unit --no-pager > /root/logs/${filename}.log
+      done;