Add integration tests for test external provider

Signed-off-by: Mihaela Balutoiu <mbalutoiu@cloudbasesolutions.com>

test/integration/config/config.toml

@@ -50,6 +50,14 @@ description = "Local LXD installation"
         protocol = "simplestreams"
         skip_verify = false
 
+[[provider]]
+name = "test_external"
+description = "external test provider"
+provider_type = "external"
+  [provider.external]
+  config_file = "/etc/garm/test-provider/config"
+  provider_executable = "/etc/garm/test-provider/garm-external-provider"
+
 [[github]]
 name = "${CREDENTIALS_NAME}"
 description = "GARM GitHub OAuth token"

test/integration/e2e/client_utils.go

@@ -387,9 +387,9 @@ func getInstance(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWrit
 	return &getInstancesResponse.Payload, nil
 }
 
-func deleteInstance(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWriter, instanceID string) error {
+func deleteInstance(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWriter, instanceID string, forceRemove bool) error {
 	return apiCli.Instances.DeleteInstance(
-		clientInstances.NewDeleteInstanceParams().WithInstanceName(instanceID),
+		clientInstances.NewDeleteInstanceParams().WithInstanceName(instanceID).WithForceRemove(&forceRemove),
 		apiAuthToken)
 }
 

test/integration/e2e/e2e.go

@@ -72,7 +72,7 @@ func GracefulCleanup() {
 			panic(err)
 		}
 		for _, instance := range poolInstances {
-			if err := deleteInstance(cli, authToken, instance.Name); err != nil {
+			if err := deleteInstance(cli, authToken, instance.Name, false); err != nil {
 				panic(err)
 			}
 			log.Printf("Instance %s deletion initiated", instance.Name)

test/integration/e2e/instances.go

@@ -33,7 +33,14 @@ func waitInstanceStatus(name string, status commonParams.InstanceStatus, runnerS
 	return nil, fmt.Errorf("timeout waiting for instance %s status to reach status %s and runner status %s", name, status, runnerStatus)
 }
 
-func waitInstanceToBeRemoved(name string, timeout time.Duration) error {
+func DeleteInstance(name string, forceRemove bool) {
+	if err := deleteInstance(cli, authToken, name, forceRemove); err != nil {
+		panic(err)
+	}
+	log.Printf("Instance %s deletion initiated", name)
+}
+
+func WaitInstanceToBeRemoved(name string, timeout time.Duration) error {
 	var timeWaited time.Duration = 0
 	var instance *params.Instance
 
@@ -67,30 +74,30 @@ func waitInstanceToBeRemoved(name string, timeout time.Duration) error {
 	return fmt.Errorf("instance %s was not removed within the timeout", name)
 }
 
-func waitPoolRunningIdleInstances(poolID string, timeout time.Duration) error {
+func WaitPoolInstances(poolID string, status commonParams.InstanceStatus, runnerStatus params.RunnerStatus, timeout time.Duration) error {
 	var timeWaited time.Duration = 0
 
 	pool, err := getPool(cli, authToken, poolID)
 	if err != nil {
 		return err
 	}
 
-	log.Printf("Waiting for pool %s to have all instances as idle running", poolID)
+	log.Printf("Waiting for pool %s instances to reach status: %s and runner status: %s", poolID, status, runnerStatus)
 	for timeWaited < timeout {
 		poolInstances, err := listPoolInstances(cli, authToken, poolID)
 		if err != nil {
 			return err
 		}
 
-		runningIdleCount := 0
+		instancesCount := 0
 		for _, instance := range poolInstances {
-			if instance.Status == commonParams.InstanceRunning && instance.RunnerStatus == params.RunnerIdle {
-				runningIdleCount++
+			if instance.Status == status && instance.RunnerStatus == runnerStatus {
+				instancesCount++
 			}
 		}
 
-		log.Printf("Pool min idle runners: %d, pool instances: %d, current pool running idle instances: %d", pool.MinIdleRunners, len(poolInstances), runningIdleCount)
-		if runningIdleCount == int(pool.MinIdleRunners) && runningIdleCount == len(poolInstances) {
+		log.Printf("Pool %s instance reached status: %s and runner status: %s: %d/%d", poolID, status, runnerStatus, instancesCount, len(poolInstances))
+		if instancesCount == int(pool.MinIdleRunners) && instancesCount == len(poolInstances) {
 			return nil
 		}
 		time.Sleep(5 * time.Second)
@@ -99,5 +106,5 @@ func waitPoolRunningIdleInstances(poolID string, timeout time.Duration) error {
 
 	_ = dumpPoolInstancesDetails(pool.ID)
 
-	return fmt.Errorf("timeout waiting for pool %s to have all idle instances running", poolID)
+	return fmt.Errorf("timeout waiting for pool %s instances to reach status: %s and runner status: %s", poolID, status, runnerStatus)
 }

test/integration/e2e/jobs.go

@@ -41,13 +41,13 @@ func ValidateJobLifecycle(label string) {
 	}
 
 	// wait for instance to be removed
-	err = waitInstanceToBeRemoved(instance.Name, 5*time.Minute)
+	err = WaitInstanceToBeRemoved(instance.Name, 5*time.Minute)
 	if err != nil {
 		panic(err)
 	}
 
 	// wait for GARM to rebuild the pool running idle instances
-	err = waitPoolRunningIdleInstances(instance.PoolID, 6*time.Minute)
+	err = WaitPoolInstances(instance.PoolID, commonParams.InstanceRunning, params.RunnerIdle, 5*time.Minute)
 	if err != nil {
 		panic(err)
 	}

test/integration/e2e/organizations.go

@@ -4,6 +4,7 @@ import (
 	"log"
 	"time"
 
+	commonParams "github.com/cloudbase/garm-provider-common/params"
 	"github.com/cloudbase/garm/params"
 )
 
@@ -100,7 +101,7 @@ func WaitOrgRunningIdleInstances(orgID string, timeout time.Duration) {
 		panic(err)
 	}
 	for _, pool := range orgPools {
-		err := waitPoolRunningIdleInstances(pool.ID, timeout)
+		err := WaitPoolInstances(pool.ID, commonParams.InstanceRunning, params.RunnerIdle, timeout)
 		if err != nil {
 			_ = dumpOrgInstancesDetails(orgID)
 			panic(err)

test/integration/e2e/repositories.go

@@ -4,6 +4,7 @@ import (
 	"log"
 	"time"
 
+	commonParams "github.com/cloudbase/garm-provider-common/params"
 	"github.com/cloudbase/garm/params"
 )
 
@@ -95,13 +96,22 @@ func DeleteRepoPool(repoID, repoPoolID string) {
 	}
 }
 
+func DisableRepoPool(repoID, repoPoolID string) {
+	log.Printf("Disable repo %s pool %s", repoID, repoPoolID)
+	enabled := false
+	poolParams := params.UpdatePoolParams{Enabled: &enabled}
+	if _, err := updateRepoPool(cli, authToken, repoID, repoPoolID, poolParams); err != nil {
+		panic(err)
+	}
+}
+
 func WaitRepoRunningIdleInstances(repoID string, timeout time.Duration) {
 	repoPools, err := listRepoPools(cli, authToken, repoID)
 	if err != nil {
 		panic(err)
 	}
 	for _, pool := range repoPools {
-		err := waitPoolRunningIdleInstances(pool.ID, timeout)
+		err := WaitPoolInstances(pool.ID, commonParams.InstanceRunning, params.RunnerIdle, timeout)
 		if err != nil {
 			_ = dumpRepoInstancesDetails(repoID)
 			panic(err)

test/integration/main.go

@@ -34,6 +34,17 @@ var (
 		Tags:           []string{"repo-runner"},
 		Enabled:        true,
 	}
+	repoPoolParams2 = params.CreatePoolParams{
+		MaxRunners:     2,
+		MinIdleRunners: 0,
+		Flavor:         "default",
+		Image:          "ubuntu:22.04",
+		OSType:         commonParams.Linux,
+		OSArch:         commonParams.Amd64,
+		ProviderName:   "test_external",
+		Tags:           []string{"repo-runner-2"},
+		Enabled:        true,
+	}
 
 	orgName          = os.Getenv("ORG_NAME")
 	orgWebhookSecret = os.Getenv("ORG_WEBHOOK_SECRET")
@@ -101,6 +112,20 @@ func main() {
 	repoPool = e2e.CreateRepoPool(repo.ID, repoPoolParams)
 	_ = e2e.UpdateRepoPool(repo.ID, repoPool.ID, repoPoolParams.MaxRunners, 1)
 
+	/////////////////////////////
+	// Test external provider ///
+	/////////////////////////////
+	repoPool2 := e2e.CreateRepoPool(repo.ID, repoPoolParams2)
+	_ = e2e.UpdateRepoPool(repo.ID, repoPool2.ID, repoPoolParams2.MaxRunners, 1)
+	e2e.WaitPoolInstances(repoPool2.ID, commonParams.InstanceRunning, params.RunnerPending, 1*time.Minute)
+	repoPool2 = e2e.GetRepoPool(repo.ID, repoPool2.ID)
+	e2e.DisableRepoPool(repo.ID, repoPool2.ID)
+	e2e.DeleteInstance(repoPool2.Instances[0].Name, false)
+	e2e.WaitPoolInstances(repoPool2.ID, commonParams.InstancePendingDelete, params.RunnerPending, 1*time.Minute)
+	e2e.DeleteInstance(repoPool2.Instances[0].Name, true) // delete instance with forceRemove
+	e2e.WaitInstanceToBeRemoved(repoPool2.Instances[0].Name, 1*time.Minute)
+	e2e.DeleteRepoPool(repo.ID, repoPool2.ID)
+
 	///////////////////
 	// organizations //
 	///////////////////

test/integration/provider/garm-external-provider

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+set -e
+set -o pipefail
+
+if [ ! -t 0 ]
+then
+    INPUT=$(cat -)
+fi
+
+if [ -z "$GARM_PROVIDER_CONFIG_FILE" ]
+then
+    echo "no config file specified in env"
+    exit 1
+fi
+
+source "$GARM_PROVIDER_CONFIG_FILE"
+
+function CreateInstance() {
+    if [ -z "$INPUT" ]; then
+        echo "expected build params in stdin"
+        exit 1
+    fi
+
+    jq -rnc '{"provider_id": "test-provider-id", "name": "test-instance-name", "os_type": "linux", "os_name": "ubuntu", "os_version": "20.04", "os_arch": "x86_64", "status": "running"}'
+}
+
+case "$GARM_COMMAND" in
+    "CreateInstance")
+        CreateInstance
+        ;;
+    "DeleteInstance")
+        echo "RemoveAllInstances not implemented"
+        exit 1
+        ;;
+    "GetInstance")
+        echo "Get instance with id: ${GARM_INSTANCE_ID}"
+        ;;
+    "ListInstances")
+        echo "List instances with pool id: ${GARM_POOL_ID}"
+        ;;
+    "StartInstance")
+        echo "Start instance: ${GARM_INSTANCE_NAME} with id: ${GARM_INSTANCE_ID}"
+        ;;
+    "StopInstance")
+        echo "Stop instance: ${GARM_INSTANCE_NAME} with id: ${GARM_INSTANCE_ID}"
+        ;;
+    "RemoveAllInstances")
+        echo "RemoveAllInstances not implemented"
+        exit 1
+        ;;
+    *)
+        echo "Invalid GARM provider command: \"$GARM_COMMAND\""
+        exit 1
+        ;;
+esac

test/integration/scripts/setup-garm.sh

@@ -5,6 +5,7 @@ DIR="$(dirname $0)"
 BINARIES_DIR="$PWD/bin"
 CONTRIB_DIR="$PWD/contrib"
 CONFIG_DIR="$PWD/test/integration/config"
+CONFIG_DIR_PROV="$PWD/test/integration/provider"
 
 if [[ ! -f $BINARIES_DIR/garm ]] || [[ ! -f $BINARIES_DIR/garm-cli ]]; then
     echo "ERROR: Please build GARM binaries first"
@@ -46,6 +47,10 @@ sudo mkdir -p /etc/garm
 cat $CONFIG_DIR/config.toml | envsubst | sudo tee /etc/garm/config.toml
 sudo chown -R garm:garm /etc/garm
 
+sudo mkdir /etc/garm/test-provider
+sudo touch $CONFIG_DIR_PROV/config
+sudo cp $CONFIG_DIR_PROV/* /etc/garm/test-provider  
+
 sudo mv $BINARIES_DIR/* /usr/local/bin/
 sudo cp $CONTRIB_DIR/garm.service /etc/systemd/system/garm.service