ENG-2740 Use elapsed time for SSH retry timeout

src/plugins/aws/ssh.ts

@@ -22,11 +22,10 @@
   AwsSshRoleRequest,
 } from "./types";
 
-/** Maximum number of attempts to start an SSH session
- *
- * Each attempt consumes ~ 1 s.
+/**
+ * It can take up to 1 minute for access to propagate on AWS, so set the time limit to 10 minutes.
  */
-const MAX_SSH_RETRIES = 30;
+const TIME_LIMIT_MS = 10 * 60 * 1000;
 
 /** The name of the SessionManager port forwarding document. This document is managed by AWS.  */
 const START_SSH_SESSION_DOCUMENT_NAME = "AWS-StartSSHSession";
@@ -84,7 +83,7 @@
 
   friendlyName: "AWS",
 
   maxRetries: MAX_SSH_RETRIES,
 
   preTestAccessPropagationArgs: () => undefined,
 
@@ -130,6 +129,8 @@
           access: "idc",
         };
   },
+
+  timeLimit: TIME_LIMIT_MS,
 
   toCliRequest: async (request) => ({ ...request, cliLocalData: undefined }),
 

src/plugins/google/ssh.ts

@@ -14,11 +14,10 @@
 import { importSshKey } from "./ssh-key";
 import { GcpSshPermissionSpec, GcpSshRequest } from "./types";
 
-/** Maximum number of attempts to start an SSH session
- *
- * The length of each attempt varies based on the type of error from a few seconds to < 1s
+/**
+ * It typically takes < 1 minute for access to propagate on GCP, so set the time limit to 2 minutes.
  */
-const MAX_SSH_RETRIES = 120;
+const TIME_LIMIT_MS = 2 * 60 * 1000;
 
 /**
  * There are 7 cases of unprovisioned access in Google Cloud.
@@ -74,7 +73,7 @@
 
   loginRequiredPattern: /You do not currently have an active account selected/,
 
   maxRetries: MAX_SSH_RETRIES,
 
   preTestAccessPropagationArgs: (cmdArgs) => {
     if (isSudoCommand(cmdArgs)) {
@@ -117,6 +116,8 @@
       type: "gcloud",
     };
   },
+
+  timeLimit: TIME_LIMIT_MS,
 
   unprovisionedAccessPatterns,
 

src/plugins/ssh/index.ts

@@ -107,7 +107,7 @@ type SpawnSshNodeOptions = {
   credential?: AwsCredentials;
   command: string;
   args: string[];
-  attemptsRemaining: number;
+  endTime: number;
   abortController?: AbortController;
   detached?: boolean;
   stdio: [StdioNull, StdioNull, StdioPipe];
@@ -127,14 +127,11 @@ async function spawnSshNode(
   return new Promise((resolve, reject) => {
     const provider = SSH_PROVIDERS[options.provider];
 
-    const attemptsRemaining = options.attemptsRemaining;
     if (options.debug) {
       const gerund = options.isAccessPropagationPreTest
         ? "Pre-testing"
         : "Trying";
-      print2(
-        `Waiting for access to propagate. ${gerund} SSH session... (remaining attempts: ${attemptsRemaining})`
-      );
+      print2(`Waiting for access to propagate. ${gerund} SSH session...)`);
     }
 
     const child = spawnChildProcess(
@@ -156,23 +153,17 @@ async function spawnSshNode(
       // In the case of ephemeral AccessDenied exceptions due to unpropagated
       // permissions, continually retry access until success
       if (!isAccessPropagated()) {
-        if (attemptsRemaining <= 0) {
+        if (options.endTime < Date.now()) {
           reject(
-            `Access did not propagate through ${provider.friendlyName} before max retry attempts were exceeded. Please contact support@p0.dev for assistance.`
+            `Access did not propagate through ${provider.friendlyName} in time. Please contact support@p0.dev for assistance.`
           );
           return;
         }
 
         delay(RETRY_DELAY_MS)
-          .then(() =>
-            spawnSshNode({
-              ...options,
-              attemptsRemaining: attemptsRemaining - 1,
-            })
-          )
+          .then(() => spawnSshNode(options))
           .then((code) => resolve(code))
           .catch(reject);
-
         return;
       } else if (isLoginException()) {
         reject(
@@ -335,7 +326,7 @@ const preTestAccessPropagationIfNeeded = async <
       stdio: ["inherit", "inherit", "pipe"],
       debug: cmdArgs.debug,
       provider: request.type,
-      attemptsRemaining: sshProvider.maxRetries,
+      endTime: Date.now() + sshProvider.timeLimit,
       isAccessPropagationPreTest: true,
     });
   }
@@ -398,6 +389,6 @@ export const sshOrScp = async (args: {
     stdio: ["inherit", "inherit", "pipe"],
     debug: cmdArgs.debug,
     provider: request.type,
-    attemptsRemaining: sshProvider.maxRetries,
+    endTime: Date.now() + sshProvider.timeLimit,
   });
 };

src/types/ssh.ts

@@ -80,6 +80,8 @@ export type SshProvider<
 
   /** Unwraps this provider's types */
   requestToSsh: (request: CliPermissionSpec<PR, O>) => SR;
+
+  timeLimit: number;
 
   /** Regex matches for error strings indicating that the provider has not yet fully provisioned node acces */
   unprovisionedAccessPatterns: readonly {