From a3f6b8adf5656c2ce95867b4ed0f6e9310024c61 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= <mail@nh2.me>
Date: Sat, 27 Jan 2018 19:49:06 +0100
Subject: [PATCH] Don't use port open check to determine if reboot completed.
 Fixes #856.

The old approach, waiting for the machine to not having an open
port, and then waiting for it to be open again, was insufficient,
because of the race condition that the machine rebooted so quickly
that the port was immediately open again without nixops noticing
that it went down. I experienced this on a Hetzner cloud server.

The new approach checks the `last reboot` on the remote side
to change, which is not racy.
---
 nixops/backends/__init__.py | 33 +++++++++++++++++++++++++--------
 1 file changed, 25 insertions(+), 8 deletions(-)

diff --git a/nixops/backends/__init__.py b/nixops/backends/__init__.py
index 8c30d63c4..578ea5f3a 100644
--- a/nixops/backends/__init__.py
+++ b/nixops/backends/__init__.py
@@ -170,7 +170,7 @@ def backup(self, defn, backup_id):
         """Make backup of persistent disks, if possible."""
         self.warn("don't know how to make backup of disks for machine ‘{0}’".format(self.name))
 
-    def reboot(self, hard=False):
+    def reboot(self, hard=False, reset=True):
         """Reboot this machine."""
         self.log("rebooting...")
         if self.state == self.RESCUE:
@@ -182,16 +182,33 @@ def reboot(self, hard=False):
             reboot_command = "systemctl reboot"
         self.run_command(reboot_command, check=False)
         self.state = self.STARTING
-        self.ssh.reset()
+        if reset:
+            self.ssh.reset()
 
     def reboot_sync(self, hard=False):
         """Reboot this machine and wait until it's up again."""
-        self.reboot(hard=hard)
-        self.log_start("waiting for the machine to finish rebooting...")
-        nixops.util.wait_for_tcp_port(self.get_ssh_name(), self.ssh_port, open=False, callback=lambda: self.log_continue("."))
-        self.log_continue("[down]")
-        nixops.util.wait_for_tcp_port(self.get_ssh_name(), self.ssh_port, callback=lambda: self.log_continue("."))
-        self.log_end("[up]")
+
+        # To check when the machine has finished rebooting in a race-free
+        # manner, we compare the output of `last reboot` before and after
+        # the reboot. Once the output has changed, the reboot is done.
+        def get_last_reboot_output():
+            return self.run_command('last reboot --time-format iso | head -n1', capture_stdout=True).rstrip()
+
+        pre_reboot_last_reboot_output = get_last_reboot_output()
+
+        # We set reset=False so that we can continue running `last reboot`
+        # remotely; when the reboot happens, our SSH connection will be reset
+        # by the remote side instead.
+        self.reboot(hard=hard, reset=False)
+
+        self.log_start("waiting for reboot to complete...")
+        while True:
+            last_reboot_output = get_last_reboot_output()
+            if last_reboot_output != pre_reboot_last_reboot_output:
+                break
+            self.log_continue(".")
+        self.log_end("done.")
+
         self.state = self.UP
         self.ssh_pinged = True
         self._ssh_pinged_this_time = True