From 0262a03e230a232a407542fbc12ba56de084a8e6 Mon Sep 17 00:00:00 2001
From: Pau Capdevila <pau@githedgehog.com>
Date: Sat, 1 Feb 2025 10:01:17 +0000
Subject: [PATCH] Add error handling for remote serial console

Fixes #358

Signed-off-by: Pau Capdevila <pau@githedgehog.com>
---
 pkg/hhfab/vlabhelpers.go            | 28 ++++++++++++++++++----------
 pkg/hhfab/vlabhelpers_reinstall.exp |  4 ++++
 2 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/pkg/hhfab/vlabhelpers.go b/pkg/hhfab/vlabhelpers.go
index dfadea2cd..d3bd2b5d6 100644
--- a/pkg/hhfab/vlabhelpers.go
+++ b/pkg/hhfab/vlabhelpers.go
@@ -342,21 +342,29 @@ func (c *Config) VLABSwitchReinstall(ctx context.Context, opts SwitchReinstallOp
 			cmd.Stderr = logutil.NewSink(ctx, slog.Debug, sw.Name+": ")
 
 			if err := cmd.Run(); err != nil {
-				mu.Lock()
-				defer mu.Unlock()
-
 				var exitErr *exec.ExitError
 				if errors.As(err, &exitErr) {
-					errs = append(errs, wrapError(sw.Name, exitErr.ExitCode()))
+					// Handle both exit code 1 and 255 as critical errors
+					if exitErr.ExitCode() == 1 || exitErr.ExitCode() == 255 {
+						mu.Lock()
+						errs = append(errs, wrapError(sw.Name, exitErr.ExitCode()))
+						mu.Unlock()
+						slog.Error("Failed to reinstall switch", "name", sw.Name, "error", err)
+						return
+					}
 				} else if errors.Is(ctx.Err(), context.DeadlineExceeded) {
-					errs = append(errs, fmt.Errorf("%s: timeout (context deadline exceeded)", sw.Name)) //nolint:goerr113
+					mu.Lock()
+					errs = append(errs, fmt.Errorf("%s: timeout (context deadline exceeded)", sw.Name))
+					mu.Unlock()
+					slog.Error("Failed to reinstall switch", "name", sw.Name, "error", err)
+					return
 				} else {
-					errs = append(errs, fmt.Errorf("%s: %w", sw.Name, err)) //nolint:goerr113
+					mu.Lock()
+					errs = append(errs, fmt.Errorf("%s: %w", sw.Name, err))
+					mu.Unlock()
+					slog.Error("Failed to reinstall switch", "name", sw.Name, "error", err)
+					return
 				}
-
-				slog.Error("Failed to reinstall switch", "name", sw.Name, "error", err)
-
-				return
 			}
 
 			if opts.WaitReady {
diff --git a/pkg/hhfab/vlabhelpers_reinstall.exp b/pkg/hhfab/vlabhelpers_reinstall.exp
index d3e42a263..62fcfe2ef 100755
--- a/pkg/hhfab/vlabhelpers_reinstall.exp
+++ b/pkg/hhfab/vlabhelpers_reinstall.exp
@@ -103,6 +103,10 @@ if {$spawn_result != 0} {
 }
 
 expect {
+	-re "This connection is in use.*" {
+		log_message "ERR" $SW_NAME "Serial port is in use by another session"
+		exit $ERROR_CONSOLE
+	}
 	-ex "Type the hot key to suspend the connection: <CTRL>Z" {
 		send "\r"
 	}