test

test/integration/e2e/client_utils.go

@@ -387,9 +387,9 @@ func getInstance(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWrit
 	return &getInstancesResponse.Payload, nil
 }
 
-func deleteInstance(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWriter, instanceID string) error {
+func deleteInstance(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWriter, instanceID string, forceRemove bool) error {
 	return apiCli.Instances.DeleteInstance(
-		clientInstances.NewDeleteInstanceParams().WithInstanceName(instanceID),
+		clientInstances.NewDeleteInstanceParams().WithInstanceName(instanceID).WithForceRemove(&forceRemove),
 		apiAuthToken)
 }
 

test/integration/e2e/e2e.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"time"
 
+	commonParams "github.com/cloudbase/garm-provider-common/params"
 	"github.com/cloudbase/garm/params"
 )
 
@@ -72,7 +73,8 @@ func GracefulCleanup() {
 			panic(err)
 		}
 		for _, instance := range poolInstances {
-			if err := deleteInstance(cli, authToken, instance.Name); err != nil {
+			forceRemove := instance.Status == commonParams.InstancePendingDelete
+			if err := deleteInstance(cli, authToken, instance.Name, forceRemove); err != nil {
 				panic(err)
 			}
 			log.Printf("Instance %s deletion initiated", instance.Name)

test/integration/e2e/instances.go

@@ -67,30 +67,35 @@ func waitInstanceToBeRemoved(name string, timeout time.Duration) error {
 	return fmt.Errorf("instance %s was not removed within the timeout", name)
 }
 
-func waitPoolRunningIdleInstances(poolID string, timeout time.Duration) error {
+func waitPoolRunningIdleAndPendingInstances(poolID string, timeout time.Duration) error {
 	var timeWaited time.Duration = 0
 
 	pool, err := getPool(cli, authToken, poolID)
 	if err != nil {
 		return err
 	}
 
-	log.Printf("Waiting for pool %s to have all instances as idle running", poolID)
+	log.Printf("Waiting for pool %s to have all instances as idle/pending running", poolID)
 	for timeWaited < timeout {
 		poolInstances, err := listPoolInstances(cli, authToken, poolID)
 		if err != nil {
 			return err
 		}
 
 		runningIdleCount := 0
+		pendingCount := 0
 		for _, instance := range poolInstances {
-			if instance.Status == commonParams.InstanceRunning && instance.RunnerStatus == params.RunnerIdle {
-				runningIdleCount++
+			if instance.Status == commonParams.InstanceRunning && (instance.RunnerStatus == params.RunnerIdle || instance.RunnerStatus == params.RunnerPending) {
+				if instance.RunnerStatus == params.RunnerIdle {
+					runningIdleCount++
+				} else {
+					pendingCount++
+				}
 			}
 		}
 
-		log.Printf("Pool min idle runners: %d, pool instances: %d, current pool running idle instances: %d", pool.MinIdleRunners, len(poolInstances), runningIdleCount)
-		if runningIdleCount == int(pool.MinIdleRunners) && runningIdleCount == len(poolInstances) {
+		log.Printf("Pool min idle runners: %d, pool instances: %d, current pool running idle instances: %d, current pool pending instances: %d", pool.MinIdleRunners, len(poolInstances), runningIdleCount, pendingCount)
+		if runningIdleCount+pendingCount == int(pool.MinIdleRunners) && runningIdleCount+pendingCount == len(poolInstances) {
 			return nil
 		}
 		time.Sleep(5 * time.Second)
@@ -99,5 +104,5 @@ func waitPoolRunningIdleInstances(poolID string, timeout time.Duration) error {
 
 	_ = dumpPoolInstancesDetails(pool.ID)
 
-	return fmt.Errorf("timeout waiting for pool %s to have all idle instances running", poolID)
+	return fmt.Errorf("timeout waiting for pool %s to have all instances as idle/pending running", poolID)
 }

test/integration/e2e/jobs.go

@@ -47,7 +47,7 @@ func ValidateJobLifecycle(label string) {
 	}
 
 	// wait for GARM to rebuild the pool running idle instances
-	err = waitPoolRunningIdleInstances(instance.PoolID, 6*time.Minute)
+	err = waitPoolRunningIdleAndPendingInstances(instance.PoolID, 6*time.Minute)
 	if err != nil {
 		panic(err)
 	}

test/integration/e2e/organizations.go

@@ -94,13 +94,13 @@ func DeleteOrgPool(orgID, orgPoolID string) {
 	}
 }
 
-func WaitOrgRunningIdleInstances(orgID string, timeout time.Duration) {
+func WaitOrgRunningIdleAndPendingInstances(orgID string, timeout time.Duration) {
 	orgPools, err := listOrgPools(cli, authToken, orgID)
 	if err != nil {
 		panic(err)
 	}
 	for _, pool := range orgPools {
-		err := waitPoolRunningIdleInstances(pool.ID, timeout)
+		err := waitPoolRunningIdleAndPendingInstances(pool.ID, timeout)
 		if err != nil {
 			_ = dumpOrgInstancesDetails(orgID)
 			panic(err)

test/integration/e2e/repositories.go

@@ -95,13 +95,13 @@ func DeleteRepoPool(repoID, repoPoolID string) {
 	}
 }
 
-func WaitRepoRunningIdleInstances(repoID string, timeout time.Duration) {
+func WaitRepoRunningIdleAndPendingInstances(repoID string, timeout time.Duration) {
 	repoPools, err := listRepoPools(cli, authToken, repoID)
 	if err != nil {
 		panic(err)
 	}
 	for _, pool := range repoPools {
-		err := waitPoolRunningIdleInstances(pool.ID, timeout)
+		err := waitPoolRunningIdleAndPendingInstances(pool.ID, timeout)
 		if err != nil {
 			_ = dumpRepoInstancesDetails(repoID)
 			panic(err)

test/integration/main.go

@@ -34,6 +34,17 @@ var (
 		Tags:           []string{"repo-runner"},
 		Enabled:        true,
 	}
+	repoPoolParams2 = params.CreatePoolParams{
+		MaxRunners:     2,
+		MinIdleRunners: 0,
+		Flavor:         "default",
+		Image:          "ubuntu:22.04",
+		OSType:         commonParams.Linux,
+		OSArch:         commonParams.Amd64,
+		ProviderName:   "test_external",
+		Tags:           []string{"repo-runner"},
+		Enabled:        true,
+	}
 
 	orgName          = os.Getenv("ORG_NAME")
 	orgWebhookSecret = os.Getenv("ORG_WEBHOOK_SECRET")
@@ -101,6 +112,12 @@ func main() {
 	repoPool = e2e.CreateRepoPool(repo.ID, repoPoolParams)
 	_ = e2e.UpdateRepoPool(repo.ID, repoPool.ID, repoPoolParams.MaxRunners, 1)
 
+	/////////////////////////////
+	// Test external provider ///
+	/////////////////////////////
+	repoPool2 := e2e.CreateRepoPool(repo.ID, repoPoolParams2)
+	_ = e2e.UpdateRepoPool(repo.ID, repoPool2.ID, repoPoolParams2.MaxRunners, 1)
+
 	///////////////////
 	// organizations //
 	///////////////////
@@ -123,8 +140,8 @@ func main() {
 	///////////////
 	// instances //
 	///////////////
-	e2e.WaitRepoRunningIdleInstances(repo.ID, 6*time.Minute)
-	e2e.WaitOrgRunningIdleInstances(org.ID, 6*time.Minute)
+	e2e.WaitRepoRunningIdleAndPendingInstances(repo.ID, 6*time.Minute)
+	e2e.WaitOrgRunningIdleAndPendingInstances(org.ID, 6*time.Minute)
 
 	//////////
 	// jobs //