From a59dd8bfee456ad6c336369cdf1852ed21fd2f8f Mon Sep 17 00:00:00 2001
From: Bruno Travouillon <bruno.travouillon@mila.quebec>
Date: Tue, 3 Sep 2024 14:54:43 -0400
Subject: [PATCH] Skip service reload when restart already triggered

Upon changes, some tasks will trigger the reload handler while others
will trigger the restart handler. During initial installation of Slurm
with the role, this can lead to some race condition where reload and
restart will execute too closely one from the other, which can trigger a
failure of the second service.

When a restart handler is notified, there should be no need to trigger
the reload as well. Move the restart handlers before the reload handlers
to make sure those are executed first. Register their result in a
variable. Test if the variable is defined in the reload handler (which
means that the restart handler was executed).

Also remove one of the 'Reload slurmdbd' handler which was defined
twice.
---
 handlers/main.yml | 40 +++++++++++++++++++++++-----------------
 1 file changed, 23 insertions(+), 17 deletions(-)

diff --git a/handlers/main.yml b/handlers/main.yml
index 5dc1e6c..5fa944a 100644
--- a/handlers/main.yml
+++ b/handlers/main.yml
@@ -4,12 +4,6 @@
     name: munge
     state: restarted
 
-- name: Reload slurmdbd
-  ansible.builtin.service:
-    name: "{{ slurmdbd_service_name }}"
-    state: reloaded
-  when: "(slurm_start_services | bool) and ('slurmdbdservers' in group_names or 'dbd' in slurm_roles)"
-
 - name: Restart slurmdbd
   ansible.builtin.systemd:
     name: "{{ slurmdbd_service_name }}"
@@ -18,18 +12,16 @@
     enabled: yes
     daemon_reload: yes
   when: "(slurm_start_services | bool) and ('slurmservers' in group_names or 'controller' in slurm_roles)"
+  register: slurmdbd_restart
 
-- name: Restart slurmdbd
+- name: Reload slurmdbd
   ansible.builtin.service:
     name: "{{ slurmdbd_service_name }}"
-    state: restarted
-  when: "slurm_start_services and ('slurmdbdservers' in group_names or 'dbd' in slurm_roles)"
-
-- name: Reload slurmctld
-  ansible.builtin.service:
-    name: "{{ slurmctld_service_name }}"
     state: reloaded
-  when: "(slurm_start_services | bool) and ('slurmservers' in group_names or 'controller' in slurm_roles)"
+  when:
+    - slurm_start_services | bool
+    - ('slurmdbdservers' in group_names or 'dbd' in slurm_roles)
+    - slurmdbd_restart is not defined
 
 - name: Restart slurmctld
   ansible.builtin.systemd:
@@ -39,15 +31,29 @@
     enabled: yes
     daemon_reload: yes
   when: "(slurm_start_services | bool) and ('slurmservers' in group_names or 'controller' in slurm_roles)"
+  register: slurmctld_restart
 
-- name: Reload slurmd
+- name: Reload slurmctld
   ansible.builtin.service:
-    name: "{{ slurmd_service_name }}"
+    name: "{{ slurmctld_service_name }}"
     state: reloaded
-  when: "(slurm_start_services | bool) and ('slurmexechosts' in group_names or 'exec' in slurm_roles)"
+  when:
+    - slurm_start_services | bool
+    - ('slurmservers' in group_names or 'controller' in slurm_roles)
+    - slurmctld_restart is not defined
 
 - name: Restart slurmd
   ansible.builtin.service:
     name: "{{ slurmd_service_name }}"
     state: restarted
   when: "(slurm_start_services | bool) and ('slurmexechosts' in group_names or 'exec' in slurm_roles)"
+  register: slurmd_restart
+
+- name: Reload slurmd
+  ansible.builtin.service:
+    name: "{{ slurmd_service_name }}"
+    state: reloaded
+  when:
+    - slurm_start_services | bool
+    - ('slurmexechosts' in group_names or 'exec' in slurm_roles)
+    - slurmd_restart is not defined