Add error handling for remote serial console

Fixes #358

Signed-off-by: Pau Capdevila <pau@githedgehog.com>

pkg/hhfab/vlabhelpers.go

@@ -342,21 +342,29 @@ func (c *Config) VLABSwitchReinstall(ctx context.Context, opts SwitchReinstallOp
 			cmd.Stderr = logutil.NewSink(ctx, slog.Debug, sw.Name+": ")
 
 			if err := cmd.Run(); err != nil {
-				mu.Lock()
-				defer mu.Unlock()
-
 				var exitErr *exec.ExitError
 				if errors.As(err, &exitErr) {
-					errs = append(errs, wrapError(sw.Name, exitErr.ExitCode()))
+					// Handle both exit code 1 and 255 as critical errors
+					if exitErr.ExitCode() == 1 || exitErr.ExitCode() == 255 {
+						mu.Lock()
+						errs = append(errs, wrapError(sw.Name, exitErr.ExitCode()))
+						mu.Unlock()
+						slog.Error("Failed to reinstall switch", "name", sw.Name, "error", err)
+						return
+					}
 				} else if errors.Is(ctx.Err(), context.DeadlineExceeded) {
-					errs = append(errs, fmt.Errorf("%s: timeout (context deadline exceeded)", sw.Name)) //nolint:goerr113
+					mu.Lock()
+					errs = append(errs, fmt.Errorf("%s: timeout (context deadline exceeded)", sw.Name))
+					mu.Unlock()
+					slog.Error("Failed to reinstall switch", "name", sw.Name, "error", err)
+					return
 				} else {
-					errs = append(errs, fmt.Errorf("%s: %w", sw.Name, err)) //nolint:goerr113
+					mu.Lock()
+					errs = append(errs, fmt.Errorf("%s: %w", sw.Name, err))
+					mu.Unlock()
+					slog.Error("Failed to reinstall switch", "name", sw.Name, "error", err)
+					return
 				}
-
-				slog.Error("Failed to reinstall switch", "name", sw.Name, "error", err)
-
-				return
 			}
 
 			if opts.WaitReady {

pkg/hhfab/vlabhelpers_reinstall.exp

@@ -103,6 +103,10 @@ if {$spawn_result != 0} {
 }
 
 expect {
+	-re "This connection is in use.*" {
+		log_message "ERR" $SW_NAME "Serial port is in use by another session"
+		exit $ERROR_CONSOLE
+	}
 	-ex "Type the hot key to suspend the connection: <CTRL>Z" {
 		send "\r"
 	}