Debug issues w/ default GPU test

PiperOrigin-RevId: 670750194

.buildkite/pipeline.yaml

@@ -185,7 +185,7 @@ steps:
     agents:
       queue: cos-canary-gpu
   - <<: *common
-    <<: *source_test_continuous
+    <<: *source_test
     label: ":male_supervillain: COS GPU Tests (Default)"
     commands:
       - make cos-gpu-all-tests

Makefile

@@ -289,9 +289,9 @@ gpu-smoke-tests: gpu-smoke-images $(RUNTIME_BIN)
 .PHONY: gpu-smoke-tests
 
 cos-gpu-smoke-tests: gpu-smoke-images $(RUNTIME_BIN)
-	@$(call sudo,test/gpu:smoke_test,--runtime=runc -test.v --cos-gpu $(ARGS))
 	@$(call install_runtime,$(RUNTIME),--nvproxy=true)
 	@$(call sudo,test/gpu:smoke_test,--runtime=$(RUNTIME) -test.v --cos-gpu $(ARGS))
+	@$(call sudo,test/gpu:smoke_test,--runtime=runc -test.v --cos-gpu $(ARGS))
 .PHONY: cos-gpu-smoke-tests
 
 # Images needed for GPU tests.

pkg/test/dockerutil/container.go

@@ -205,17 +205,18 @@ func (c *Container) SpawnProcess(ctx context.Context, r RunOpts, args ...string)
 // Run is analogous to 'docker run'.
 func (c *Container) Run(ctx context.Context, r RunOpts, args ...string) (string, error) {
 	if err := c.create(ctx, r.Image, c.config(r, args), c.hostConfig(r), nil); err != nil {
-		return "", err
+		return "", fmt.Errorf("failed to create container: %w", err)
 	}
 
 	if err := c.Start(ctx); err != nil {
 		logs, _ := c.Logs(ctx)
-		return logs, err
+		return logs, fmt.Errorf("failed to start container: %w", err)
 	}
 
 	if err := c.Wait(ctx); err != nil {
+		info, ierr := c.client.ContainerInspect(ctx, c.id)
 		logs, _ := c.Logs(ctx)
-		return logs, err
+		return logs, fmt.Errorf("failed to wait for container: %w info: %+v info_err: %v", err, info.ContainerJSONBase.State, ierr)
 	}
 
 	return c.Logs(ctx)
@@ -265,7 +266,7 @@ func (c *Container) create(ctx context.Context, profileImage string, conf *conta
 	}
 	cont, err := c.client.ContainerCreate(ctx, conf, hostconf, nil, nil, c.Name)
 	if err != nil {
-		return err
+		return fmt.Errorf("ContainerCreate failed: %v warnings: %v", err, cont.Warnings)
 	}
 	c.id = cont.ID
 	return nil