fix: handle docker container startup race condition in local provisioner

cargo-shuttle/src/provisioner_server.rs

@@ -244,8 +244,15 @@ impl LocalProvisioner {
     }
 
     async fn wait_for_ready(&self, container_name: &str, is_ready_cmd: Vec<String>) -> Result<()> {
-        loop {
-            trace!("waiting for '{container_name}' to be ready for connections");
+        const MAX_RETRIES: u32 = 60;
+        const RETRY_DELAY_MS: u64 = 500;
+
+        for attempt in 0..MAX_RETRIES {
+            trace!(
+                "waiting for '{container_name}' to be ready for connections (attempt {}/{})",
+                attempt + 1,
+                MAX_RETRIES
+            );
 
             let config = CreateExecOptions {
                 cmd: Some(is_ready_cmd.clone()),
@@ -254,32 +261,64 @@ impl LocalProvisioner {
                 ..Default::default()
             };
 
-            let CreateExecResults { id } = self
-                .docker
-                .create_exec(container_name, config)
-                .await
-                .expect("failed to create exec to check if container is ready");
+            let exec_result = self.docker.create_exec(container_name, config).await;
+
+            let exec_id = match exec_result {
+                Ok(CreateExecResults { id }) => id,
+                Err(bollard::errors::Error::DockerResponseServerError {
+                    status_code: 409,
+                    message,
+                }) => {
+                    trace!("container '{container_name}' not ready yet: {message}");
+
+                    match self.docker.inspect_container(container_name, None).await {
+                        Ok(container) => {
+                            let state =
+                                container.state.as_ref().context("container has no state")?;
+
+                            if !state.running.unwrap_or(false) {
+                                bail!("container '{container_name}' has stopped unexpectedly with status: {:?}, exit code: {:?}", 
+                                      state.status, state.exit_code);
+                            }
+
+                            sleep(Duration::from_millis(RETRY_DELAY_MS)).await;
+                            continue;
+                        }
+                        Err(e) => {
+                            bail!("failed to inspect container '{container_name}': {e}");
+                        }
+                    }
+                }
+                Err(e) => {
+                    bail!("failed to create exec to check if container '{container_name}' is ready: {e}");
+                }
+            };
 
             let ready_result = self
                 .docker
-                .start_exec(&id, None)
+                .start_exec(&exec_id, None)
                 .await
-                .expect("failed to execute ready command");
+                .context("failed to execute ready command")?;
 
             if let bollard::exec::StartExecResults::Attached { mut output, .. } = ready_result {
                 while let Some(line) = output.next().await {
                     trace!("line: {:?}", line);
 
                     if let bollard::container::LogOutput::StdOut { .. } =
-                        line.expect("output to have a log line")
+                        line.context("output to have a log line")?
                     {
                         return Ok(());
                     }
                 }
             }
 
-            sleep(Duration::from_millis(500)).await;
+            sleep(Duration::from_millis(RETRY_DELAY_MS)).await;
         }
+
+        bail!(
+            "container '{container_name}' did not become ready within {} seconds",
+            (MAX_RETRIES as u64 * RETRY_DELAY_MS) / 1000
+        )
     }
 
     async fn pull_image(&self, image: &str) -> Result<(), String> {