Add retries to SSH iperf on server

Extend show tech logs gathered on server VMs Fixes #310 Signed-off-by: Pau Capdevila <[email protected]>
githedgehog · Jan 24, 2025 · 445d5e7 · 445d5e7
1 parent 5b806be
commit 445d5e7
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 5 deletions.
diff --git a/pkg/hhfab/show-tech/server.sh b/pkg/hhfab/show-tech/server.sh
@@ -69,4 +69,7 @@ journalctl -k >> "$OUTPUT_FILE" 2>/dev/null
 echo -e "\n=== Kernel Network Logs ===" >> "$OUTPUT_FILE"
 dmesg | grep -i "network\|bond\|vlan" >> "$OUTPUT_FILE"
 
+echo -e "\n=== SSH Logs ===" >> "$OUTPUT_FILE"
+journalctl -u sshd >> "$OUTPUT_FILE" 2>/dev/null
+
 echo "Diagnostics collected to $OUTPUT_FILE"
diff --git a/pkg/hhfab/testing.go b/pkg/hhfab/testing.go
@@ -1201,12 +1201,20 @@ func checkIPerf(ctx context.Context, opts TestConnectivityOpts, iperfs *semaphor
 	g, ctx := errgroup.WithContext(ctx)
 
 	g.Go(func() error {
-		out, err := toSSH.RunContext(ctx, fmt.Sprintf("toolbox -q timeout -v %d iperf3 -s -1", opts.IPerfsSeconds+25))
-		if err != nil {
-			return fmt.Errorf("running iperf server: %w: %s", err, string(out))
+		maxRetries := 3
+		var lastErr error
+		for attempt := 1; attempt <= maxRetries; attempt++ {
+			slog.Debug("Starting iperf3 server", "server", to, "attempt", attempt)
+			out, err := toSSH.RunContext(ctx, fmt.Sprintf("toolbox -q timeout -v %d iperf3 -s -1", opts.IPerfsSeconds+25))
+			if err == nil {
+				slog.Debug("iperf3 server started successfully", "server", to)
+				return nil
+			}
+			slog.Warn("iperf3 server failed", "server", to, "attempt", attempt, "error", err)
+			lastErr = fmt.Errorf("running iperf server: %w: %s", err, string(out))
+			time.Sleep(2 * time.Second) // Backoff between retries
 		}
-
-		return nil
+		return fmt.Errorf("iperf3 server failed after %d attempts: %w", maxRetries, lastErr)
 	})
 
 	g.Go(func() error {