Debug issues w/ default GPU test

PiperOrigin-RevId: 670750194

.buildkite/pipeline.yaml

@@ -184,6 +184,13 @@ steps:
       - make cos-gpu-all-tests
     agents:
       queue: cos-canary-gpu
+  - <<: *common
+    <<: *source_test
+    label: ":male_supervillain: COS GPU Tests (Default)"
+    commands:
+      - make cos-gpu-all-tests
+    agents:
+      queue: cos-canary-gpu-default
   - <<: *common
     <<: *source_test_continuous
     label: ":screwdriver: All GPU Drivers Test"

Makefile

@@ -284,15 +284,15 @@ gpu-smoke-images: load-basic_cuda-vector-add load-gpu_cuda-tests
 .PHONY: gpu-smoke-images
 
 gpu-smoke-tests: gpu-smoke-images $(RUNTIME_BIN)
-	@$(call sudo,test/gpu:smoke_test,--runtime=runc -test.v $(ARGS))
 	@$(call install_runtime,$(RUNTIME),--nvproxy=true --nvproxy-docker=true)
 	@$(call sudo,test/gpu:smoke_test,--runtime=$(RUNTIME) -test.v $(ARGS))
+	@$(call sudo,test/gpu:smoke_test,--runtime=runc -test.v $(ARGS))
 .PHONY: gpu-smoke-tests
 
 cos-gpu-smoke-tests: gpu-smoke-images $(RUNTIME_BIN)
-	@$(call sudo,test/gpu:smoke_test,--runtime=runc -test.v --cos-gpu $(ARGS))
 	@$(call install_runtime,$(RUNTIME),--nvproxy=true)
 	@$(call sudo,test/gpu:smoke_test,--runtime=$(RUNTIME) -test.v --cos-gpu $(ARGS))
+	@$(call sudo,test/gpu:smoke_test,--runtime=runc -test.v --cos-gpu $(ARGS))
 .PHONY: cos-gpu-smoke-tests
 
 # Images needed for GPU tests.

pkg/test/dockerutil/container.go

@@ -225,12 +225,13 @@ func (c *Container) Run(ctx context.Context, r RunOpts, args ...string) (string,
 
 	if err := c.Start(ctx); err != nil {
 		logs, _ := c.Logs(ctx)
-		return logs, err
+		return logs, fmt.Errorf("failed to start container: %w", err)
 	}
 
 	if err := c.Wait(ctx); err != nil {
+		info, ierr := c.client.ContainerInspect(ctx, c.id)
 		logs, _ := c.Logs(ctx)
-		return logs, err
+		return logs, fmt.Errorf("failed to wait for container: %w info: %+v info_err: %v", err, info.ContainerJSONBase.State, ierr)
 	}
 
 	return c.Logs(ctx)
@@ -288,7 +289,7 @@ func (c *Container) create(ctx context.Context, profileImage string, conf *conta
 	}
 	cont, err := c.client.ContainerCreate(ctx, conf, hostconf, nil, nil, c.Name)
 	if err != nil {
-		return err
+		return fmt.Errorf("ContainerCreate failed: %v warnings: %v", err, cont.Warnings)
 	}
 	c.id = cont.ID
 	return nil