test2

test/integration/e2e/instances.go

@@ -84,37 +84,38 @@ func waitInstanceToBeRemoved(name string, timeout time.Duration) error {
 	return fmt.Errorf("instance %s was not removed within the timeout", name)
 }
 
-func waitPoolRunningIdleInstances(poolID string, timeout time.Duration) error {
+func WaitPoolInstances(poolID string, status commonParams.InstanceStatus, runnerStatus params.RunnerStatus, timeout time.Duration) error {
 	var timeWaited time.Duration = 0
 
 	pool, err := getPool(cli, authToken, poolID)
 	if err != nil {
 		return err
 	}
 
-	log.Printf("Waiting for pool %s to have all instances as idle running", poolID)
+	log.Printf("Waiting for pool %s instances to reach status %s and runner status %s", poolID, status, runnerStatus)
 	for timeWaited < timeout {
 		poolInstances, err := listPoolInstances(cli, authToken, poolID)
 		if err != nil {
 			return err
 		}
 
-		runningIdleCount := 0
+		instancesCount := 0
 		for _, instance := range poolInstances {
-			if instance.Status == commonParams.InstanceRunning && instance.RunnerStatus == params.RunnerIdle {
-				runningIdleCount++
+			log.Printf("Instance %s status %s and runner status %s", instance.Name, instance.Status, instance.RunnerStatus)
+			if instance.Status == status && instance.RunnerStatus == runnerStatus {
+				instancesCount++
 			}
 		}
 
-		log.Printf("Pool min idle runners: %d, pool instances: %d, current pool running idle instances: %d", pool.MinIdleRunners, len(poolInstances), runningIdleCount)
-		if runningIdleCount == int(pool.MinIdleRunners) && runningIdleCount == len(poolInstances) {
+		log.Printf("Pool %s instances count: %d", poolID, instancesCount)
+		if instancesCount == int(pool.MinIdleRunners) && instancesCount == len(poolInstances) {
 			return nil
 		}
-		time.Sleep(5 * time.Second)
-		timeWaited += 5 * time.Second
-	}
 
+			time.Sleep(5 * time.Second)
+			timeWaited += 5 * time.Second
+		}
 	_ = dumpPoolInstancesDetails(pool.ID)
 
-	return fmt.Errorf("timeout waiting for pool %s to have all idle instances running", poolID)
+	return fmt.Errorf("timeout waiting for pool %s instances to reach status %s and runner status %s", poolID, status, runnerStatus)
 }

test/integration/e2e/jobs.go

@@ -47,7 +47,7 @@ func ValidateJobLifecycle(label string) {
 	}
 
 	// wait for GARM to rebuild the pool running idle instances
-	err = waitPoolRunningIdleInstances(instance.PoolID, 6*time.Minute)
+	err = WaitPoolInstances(instance.PoolID, commonParams.InstanceRunning, params.RunnerIdle, 5*time.Minute)
 	if err != nil {
 		panic(err)
 	}

test/integration/e2e/organizations.go

@@ -4,6 +4,7 @@ import (
 	"log"
 	"time"
 
+	commonParams "github.com/cloudbase/garm-provider-common/params"
 	"github.com/cloudbase/garm/params"
 )
 
@@ -100,7 +101,7 @@ func WaitOrgRunningIdleInstances(orgID string, timeout time.Duration) {
 		panic(err)
 	}
 	for _, pool := range orgPools {
-		err := waitPoolRunningIdleInstances(pool.ID, timeout)
+		err := WaitPoolInstances(pool.ID, commonParams.InstanceRunning, params.RunnerIdle, timeout)
 		if err != nil {
 			_ = dumpOrgInstancesDetails(orgID)
 			panic(err)

test/integration/e2e/repositories.go

@@ -4,6 +4,7 @@ import (
 	"log"
 	"time"
 
+	commonParams "github.com/cloudbase/garm-provider-common/params"
 	"github.com/cloudbase/garm/params"
 )
 
@@ -110,7 +111,7 @@ func WaitRepoRunningIdleInstances(repoID string, timeout time.Duration) {
 		panic(err)
 	}
 	for _, pool := range repoPools {
-		err := waitPoolRunningIdleInstances(pool.ID, timeout)
+		err := WaitPoolInstances(pool.ID, commonParams.InstanceRunning, params.RunnerIdle, timeout)
 		if err != nil {
 			_ = dumpRepoInstancesDetails(repoID)
 			panic(err)

test/integration/main.go

@@ -117,9 +117,12 @@ func main() {
 	/////////////////////////////
 	repoPool2 := e2e.CreateRepoPool(repo.ID, repoPoolParams2)
 	_ = e2e.UpdateRepoPool(repo.ID, repoPool2.ID, repoPoolParams2.MaxRunners, 1)
+	e2e.WaitPoolInstances(repoPool2.ID, commonParams.InstanceRunning, params.RunnerPending, 1*time.Minute)
 	e2e.DisableRepoPool(repo.ID, repoPool2.ID)
 	e2e.DeleteInstance(repoPool2.Instances[0].Name, false)
+	e2e.WaitPoolInstances(repoPool2.ID, commonParams.InstancePendingDelete, params.RunnerPending, 1*time.Minute)
 	e2e.DeleteInstance(repoPool2.Instances[0].Name, true)
+	e2e.WaitPoolInstances(repoPool2.ID, commonParams.InstancePendingForceDelete, params.RunnerPending, 1*time.Minute)
 
 	///////////////////
 	// organizations //