test

test/integration/e2e/client_utils.go

@@ -387,9 +387,9 @@ func getInstance(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWrit
 	return &getInstancesResponse.Payload, nil
 }
 
-func deleteInstance(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWriter, instanceID string) error {
+func deleteInstance(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWriter, instanceID string, forceRemove bool) error {
 	return apiCli.Instances.DeleteInstance(
-		clientInstances.NewDeleteInstanceParams().WithInstanceName(instanceID),
+		clientInstances.NewDeleteInstanceParams().WithInstanceName(instanceID).WithForceRemove(&forceRemove),
 		apiAuthToken)
 }
 

test/integration/e2e/e2e.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"time"
 
+	commonParams "github.com/cloudbase/garm-provider-common/params"
 	"github.com/cloudbase/garm/params"
 )
 
@@ -72,13 +73,30 @@ func GracefulCleanup() {
 			panic(err)
 		}
 		for _, instance := range poolInstances {
-			if err := deleteInstance(cli, authToken, instance.Name); err != nil {
+			if err := deleteInstance(cli, authToken, instance.Name, false); err != nil {
 				panic(err)
 			}
 			log.Printf("Instance %s deletion initiated", instance.Name)
 		}
 	}
 
+	// wait for all instances to be deleted with forceRemove
+	for _, pool := range pools {
+		poolInstances, err := listPoolInstances(cli, authToken, pool.ID)
+		if err != nil {
+			panic(err)
+		}
+		for _, instance := range poolInstances {
+			if instance.Status == commonParams.InstancePendingDelete {
+				err := deleteInstance(cli, authToken, instance.Name, true)
+				if err != nil {
+					panic(err)
+				}
+			}
+			log.Printf("Instance %s deletion with forceRemove initiated", instance.Name)
+		}
+	}
+
 	// wait for all instances to be deleted
 	for _, pool := range pools {
 		if err := waitPoolNoInstances(pool.ID, 3*time.Minute); err != nil {

test/integration/e2e/instances.go

@@ -67,37 +67,41 @@ func waitInstanceToBeRemoved(name string, timeout time.Duration) error {
 	return fmt.Errorf("instance %s was not removed within the timeout", name)
 }
 
-func waitPoolRunningIdleInstances(poolID string, timeout time.Duration) error {
+func waitPoolRunningIdleAndPendingInstances(poolID string, timeout time.Duration) error {
 	var timeWaited time.Duration = 0
 
 	pool, err := getPool(cli, authToken, poolID)
 	if err != nil {
 		return err
 	}
 
-	log.Printf("Waiting for pool %s to have all instances as idle running", poolID)
+	log.Printf("Waiting for pool %s to have all instances as idle or pending running", poolID)
 	for timeWaited < timeout {
 		poolInstances, err := listPoolInstances(cli, authToken, poolID)
 		if err != nil {
 			return err
 		}
 
 		runningIdleCount := 0
+		pendingCount := 0
 		for _, instance := range poolInstances {
-			if instance.Status == commonParams.InstanceRunning && instance.RunnerStatus == params.RunnerIdle {
-				runningIdleCount++
+			if instance.Status == commonParams.InstanceRunning && (instance.RunnerStatus == params.RunnerIdle || instance.RunnerStatus == params.RunnerPending) {
+				if instance.RunnerStatus == params.RunnerIdle {
+					runningIdleCount++
+				} else {
+					pendingCount++
+				}
 			}
 		}
 
-		log.Printf("Pool min idle runners: %d, pool instances: %d, current pool running idle instances: %d", pool.MinIdleRunners, len(poolInstances), runningIdleCount)
-		if runningIdleCount == int(pool.MinIdleRunners) && runningIdleCount == len(poolInstances) {
+		log.Printf("Pool min idle runners: %d, pool instances: %d, current pool running idle instances: %d, current pool pending instances: %d", pool.MinIdleRunners, len(poolInstances), runningIdleCount, pendingCount)
+		if runningIdleCount+pendingCount == int(pool.MinIdleRunners) && runningIdleCount+pendingCount == len(poolInstances) {
 			return nil
 		}
 		time.Sleep(5 * time.Second)
 		timeWaited += 5 * time.Second
 	}
-
 	_ = dumpPoolInstancesDetails(pool.ID)
 
-	return fmt.Errorf("timeout waiting for pool %s to have all idle instances running", poolID)
+	return fmt.Errorf("timeout waiting for pool %s to have all idle or pending instances running", poolID)
 }

test/integration/e2e/jobs.go

@@ -46,8 +46,8 @@ func ValidateJobLifecycle(label string) {
 		panic(err)
 	}
 
-	// wait for GARM to rebuild the pool running idle instances
-	err = waitPoolRunningIdleInstances(instance.PoolID, 6*time.Minute)
+	// wait for GARM to rebuild the pool running idle or pending instances
+	err = waitPoolRunningIdleAndPendingInstances(instance.PoolID, 6*time.Minute)
 	if err != nil {
 		panic(err)
 	}

test/integration/e2e/organizations.go

@@ -100,7 +100,7 @@ func WaitOrgRunningIdleInstances(orgID string, timeout time.Duration) {
 		panic(err)
 	}
 	for _, pool := range orgPools {
-		err := waitPoolRunningIdleInstances(pool.ID, timeout)
+		err := waitPoolRunningIdleAndPendingInstances(pool.ID, timeout)
 		if err != nil {
 			_ = dumpOrgInstancesDetails(orgID)
 			panic(err)

test/integration/e2e/repositories.go

@@ -101,7 +101,7 @@ func WaitRepoRunningIdleInstances(repoID string, timeout time.Duration) {
 		panic(err)
 	}
 	for _, pool := range repoPools {
-		err := waitPoolRunningIdleInstances(pool.ID, timeout)
+		err := waitPoolRunningIdleAndPendingInstances(pool.ID, timeout)
 		if err != nil {
 			_ = dumpRepoInstancesDetails(repoID)
 			panic(err)

test/integration/main.go

@@ -34,6 +34,17 @@ var (
 		Tags:           []string{"repo-runner"},
 		Enabled:        true,
 	}
+	repoPoolParams2 = params.CreatePoolParams{
+		MaxRunners:     2,
+		MinIdleRunners: 0,
+		Flavor:         "default",
+		Image:          "ubuntu:22.04",
+		OSType:         commonParams.Linux,
+		OSArch:         commonParams.Amd64,
+		ProviderName:   "test_external",
+		Tags:           []string{"repo-runner"},
+		Enabled:        true,
+	}
 
 	orgName          = os.Getenv("ORG_NAME")
 	orgWebhookSecret = os.Getenv("ORG_WEBHOOK_SECRET")
@@ -101,6 +112,12 @@ func main() {
 	repoPool = e2e.CreateRepoPool(repo.ID, repoPoolParams)
 	_ = e2e.UpdateRepoPool(repo.ID, repoPool.ID, repoPoolParams.MaxRunners, 1)
 
+	/////////////////////////////
+	// Test external provider ///
+	/////////////////////////////
+	repoPool2 := e2e.CreateRepoPool(repo.ID, repoPoolParams2)
+	_ = e2e.UpdateRepoPool(repo.ID, repoPool2.ID, repoPoolParams2.MaxRunners, 1)
+
 	///////////////////
 	// organizations //
 	///////////////////