From ff92232d27cce6445c2c388ddeeeec66ceaf67f0 Mon Sep 17 00:00:00 2001
From: Pau Capdevila <pau@githedgehog.com>
Date: Sat, 1 Feb 2025 10:01:17 +0000
Subject: [PATCH] Add error handling for remote serial console

Fixes #358

Signed-off-by: Pau Capdevila <pau@githedgehog.com>
---
 pkg/hhfab/vlabhelpers.go            | 28 +++++++++++++++++++---------
 pkg/hhfab/vlabhelpers_reinstall.exp |  4 ++++
 2 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/pkg/hhfab/vlabhelpers.go b/pkg/hhfab/vlabhelpers.go
index dfadea2c..e61893c5 100644
--- a/pkg/hhfab/vlabhelpers.go
+++ b/pkg/hhfab/vlabhelpers.go
@@ -342,21 +342,31 @@ func (c *Config) VLABSwitchReinstall(ctx context.Context, opts SwitchReinstallOp
 			cmd.Stderr = logutil.NewSink(ctx, slog.Debug, sw.Name+": ")
 
 			if err := cmd.Run(); err != nil {
-				mu.Lock()
-				defer mu.Unlock()
-
 				var exitErr *exec.ExitError
 				if errors.As(err, &exitErr) {
-					errs = append(errs, wrapError(sw.Name, exitErr.ExitCode()))
+					if exitErr.ExitCode() == 1 || exitErr.ExitCode() == 255 {
+						mu.Lock()
+						errs = append(errs, wrapError(sw.Name, exitErr.ExitCode()))
+						mu.Unlock()
+						slog.Error("Failed to reinstall switch", "name", sw.Name, "error", err)
+
+						return
+					}
 				} else if errors.Is(ctx.Err(), context.DeadlineExceeded) {
+					mu.Lock()
 					errs = append(errs, fmt.Errorf("%s: timeout (context deadline exceeded)", sw.Name)) //nolint:goerr113
-				} else {
-					errs = append(errs, fmt.Errorf("%s: %w", sw.Name, err)) //nolint:goerr113
-				}
+					mu.Unlock()
+					slog.Error("Failed to reinstall switch", "name", sw.Name, "error", err)
 
-				slog.Error("Failed to reinstall switch", "name", sw.Name, "error", err)
+					return
+				} else {
+					mu.Lock()
+					errs = append(errs, fmt.Errorf("%s: %w", sw.Name, err))
+					mu.Unlock()
+					slog.Error("Failed to reinstall switch", "name", sw.Name, "error", err)
 
-				return
+					return
+				}
 			}
 
 			if opts.WaitReady {
diff --git a/pkg/hhfab/vlabhelpers_reinstall.exp b/pkg/hhfab/vlabhelpers_reinstall.exp
index d3e42a26..62fcfe2e 100755
--- a/pkg/hhfab/vlabhelpers_reinstall.exp
+++ b/pkg/hhfab/vlabhelpers_reinstall.exp
@@ -103,6 +103,10 @@ if {$spawn_result != 0} {
 }
 
 expect {
+	-re "This connection is in use.*" {
+		log_message "ERR" $SW_NAME "Serial port is in use by another session"
+		exit $ERROR_CONSOLE
+	}
 	-ex "Type the hot key to suspend the connection: <CTRL>Z" {
 		send "\r"
 	}