wip

.github/workflows/integration-tests.yml

@@ -64,8 +64,8 @@ jobs:
           go run ./test/integration/main.go 2>&1 | tee /artifacts-logs/e2e.log
         env:
           GARM_BASE_URL: ${{ steps.ngrok.outputs.tunnel-url }}
-          ORG_NAME: gsamfira
-          REPO_NAME: garm-testing
+          ORG_NAME: test-garm-org
+          REPO_NAME: test-garm-repo
           CREDENTIALS_NAME: test-garm-creds
           WORKFLOW_FILE_NAME: test.yml
           GH_TOKEN: ${{ secrets.GH_OAUTH_TOKEN }}
@@ -94,6 +94,6 @@ jobs:
           go run ./test/integration/gh_cleanup/main.go
         env:
           GARM_BASE_URL: ${{ steps.ngrok.outputs.tunnel-url }}
-          ORG_NAME: gsamfira
-          REPO_NAME: garm-testing
+          ORG_NAME: test-garm-org
+          REPO_NAME: test-garm-repo
           GH_TOKEN: ${{ secrets.GH_OAUTH_TOKEN }}

test/integration/config/config.toml

@@ -50,6 +50,14 @@ description = "Local LXD installation"
         protocol = "simplestreams"
         skip_verify = false
 
+[[provider]]
+name = "test_external"
+description = "external test provider"
+provider_type = "external"
+  [provider.external]
+  config_file = "/etc/garm/test-provider/config"
+  provider_executable = "/etc/garm/test-provider/garm-external-provider"
+
 [[github]]
 name = "${CREDENTIALS_NAME}"
 description = "GARM GitHub OAuth token"

test/integration/e2e/client_utils.go

@@ -387,9 +387,9 @@ func getInstance(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWrit
 	return &getInstancesResponse.Payload, nil
 }
 
-func deleteInstance(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWriter, instanceID string) error {
+func deleteInstance(apiCli *client.GarmAPI, apiAuthToken runtime.ClientAuthInfoWriter, instanceID string, forceRemove bool) error {
 	return apiCli.Instances.DeleteInstance(
-		clientInstances.NewDeleteInstanceParams().WithInstanceName(instanceID),
+		clientInstances.NewDeleteInstanceParams().WithInstanceName(instanceID).WithForceRemove(&forceRemove),
 		apiAuthToken)
 }
 

test/integration/e2e/e2e.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"time"
 
+	commonParams "github.com/cloudbase/garm-provider-common/params"
 	"github.com/cloudbase/garm/params"
 )
 
@@ -72,13 +73,30 @@ func GracefulCleanup() {
 			panic(err)
 		}
 		for _, instance := range poolInstances {
-			if err := deleteInstance(cli, authToken, instance.Name); err != nil {
+			if err := deleteInstance(cli, authToken, instance.Name, false); err != nil {
 				panic(err)
 			}
 			log.Printf("Instance %s deletion initiated", instance.Name)
 		}
 	}
 
+	// wait for all instances to be deleted with forceRemove
+	for _, pool := range pools {
+		poolInstances, err := listPoolInstances(cli, authToken, pool.ID)
+		if err != nil {
+			panic(err)
+		}
+		for _, instance := range poolInstances {
+			if instance.Status == commonParams.InstancePendingDelete {
+				err := deleteInstance(cli, authToken, instance.Name, true)
+				if err != nil {
+					panic(err)
+				}
+			}
+			log.Printf("Instance %s deletion with forceRemove initiated", instance.Name)
+		}
+	}
+
 	// wait for all instances to be deleted
 	for _, pool := range pools {
 		if err := waitPoolNoInstances(pool.ID, 3*time.Minute); err != nil {

test/integration/e2e/instances.go

@@ -67,37 +67,41 @@ func waitInstanceToBeRemoved(name string, timeout time.Duration) error {
 	return fmt.Errorf("instance %s was not removed within the timeout", name)
 }
 
-func waitPoolRunningIdleInstances(poolID string, timeout time.Duration) error {
+func waitPoolRunningIdleAndPendingInstances(poolID string, timeout time.Duration) error {
 	var timeWaited time.Duration = 0
 
 	pool, err := getPool(cli, authToken, poolID)
 	if err != nil {
 		return err
 	}
 
-	log.Printf("Waiting for pool %s to have all instances as idle running", poolID)
+	log.Printf("Waiting for pool %s to have all instances as idle or pending running", poolID)
 	for timeWaited < timeout {
 		poolInstances, err := listPoolInstances(cli, authToken, poolID)
 		if err != nil {
 			return err
 		}
 
 		runningIdleCount := 0
+		pendingCount := 0
 		for _, instance := range poolInstances {
-			if instance.Status == commonParams.InstanceRunning && instance.RunnerStatus == params.RunnerIdle {
-				runningIdleCount++
+			if instance.Status == commonParams.InstanceRunning && (instance.RunnerStatus == params.RunnerIdle || instance.RunnerStatus == params.RunnerPending) {
+				if instance.RunnerStatus == params.RunnerIdle {
+					runningIdleCount++
+				} else {
+					pendingCount++
+				}
 			}
 		}
 
-		log.Printf("Pool min idle runners: %d, pool instances: %d, current pool running idle instances: %d", pool.MinIdleRunners, len(poolInstances), runningIdleCount)
-		if runningIdleCount == int(pool.MinIdleRunners) && runningIdleCount == len(poolInstances) {
+		log.Printf("Pool min idle runners: %d, pool instances: %d, current pool running idle instances: %d, current pool pending instances: %d", pool.MinIdleRunners, len(poolInstances), runningIdleCount, pendingCount)
+		if runningIdleCount+pendingCount == int(pool.MinIdleRunners) && runningIdleCount+pendingCount == len(poolInstances) {
 			return nil
 		}
 		time.Sleep(5 * time.Second)
 		timeWaited += 5 * time.Second
 	}
-
 	_ = dumpPoolInstancesDetails(pool.ID)
 
-	return fmt.Errorf("timeout waiting for pool %s to have all idle instances running", poolID)
+	return fmt.Errorf("timeout waiting for pool %s to have all idle or pending instances running", poolID)
 }

test/integration/e2e/jobs.go

@@ -46,8 +46,8 @@ func ValidateJobLifecycle(label string) {
 		panic(err)
 	}
 
-	// wait for GARM to rebuild the pool running idle instances
-	err = waitPoolRunningIdleInstances(instance.PoolID, 6*time.Minute)
+	// wait for GARM to rebuild the pool running idle or pending instances
+	err = waitPoolRunningIdleAndPendingInstances(instance.PoolID, 6*time.Minute)
 	if err != nil {
 		panic(err)
 	}

test/integration/e2e/organizations.go

@@ -100,7 +100,7 @@ func WaitOrgRunningIdleInstances(orgID string, timeout time.Duration) {
 		panic(err)
 	}
 	for _, pool := range orgPools {
-		err := waitPoolRunningIdleInstances(pool.ID, timeout)
+		err := waitPoolRunningIdleAndPendingInstances(pool.ID, timeout)
 		if err != nil {
 			_ = dumpOrgInstancesDetails(orgID)
 			panic(err)

test/integration/e2e/repositories.go

@@ -101,7 +101,7 @@ func WaitRepoRunningIdleInstances(repoID string, timeout time.Duration) {
 		panic(err)
 	}
 	for _, pool := range repoPools {
-		err := waitPoolRunningIdleInstances(pool.ID, timeout)
+		err := waitPoolRunningIdleAndPendingInstances(pool.ID, timeout)
 		if err != nil {
 			_ = dumpRepoInstancesDetails(repoID)
 			panic(err)

test/integration/main.go

@@ -34,6 +34,17 @@ var (
 		Tags:           []string{"repo-runner"},
 		Enabled:        true,
 	}
+	repoPoolParams2 = params.CreatePoolParams{
+		MaxRunners:     2,
+		MinIdleRunners: 0,
+		Flavor:         "default",
+		Image:          "ubuntu:22.04",
+		OSType:         commonParams.Linux,
+		OSArch:         commonParams.Amd64,
+		ProviderName:   "test_external",
+		Tags:           []string{"repo-runner-2"},
+		Enabled:        true,
+	}
 
 	orgName          = os.Getenv("ORG_NAME")
 	orgWebhookSecret = os.Getenv("ORG_WEBHOOK_SECRET")
@@ -101,6 +112,12 @@ func main() {
 	repoPool = e2e.CreateRepoPool(repo.ID, repoPoolParams)
 	_ = e2e.UpdateRepoPool(repo.ID, repoPool.ID, repoPoolParams.MaxRunners, 1)
 
+	/////////////////////////////
+	// Test external provider ///
+	/////////////////////////////
+	repoPool2 := e2e.CreateRepoPool(repo.ID, repoPoolParams2)
+	_ = e2e.UpdateRepoPool(repo.ID, repoPool2.ID, repoPoolParams2.MaxRunners, 1)
+
 	///////////////////
 	// organizations //
 	///////////////////

test/integration/provider/garm-external-provider

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+set -e
+set -o pipefail
+
+if [ ! -t 0 ]
+then
+    INPUT=$(cat -)
+fi
+
+if [ -z "$GARM_PROVIDER_CONFIG_FILE" ]
+then
+    echo "no config file specified in env"
+    exit 1
+fi
+
+source "$GARM_PROVIDER_CONFIG_FILE"
+
+function CreateInstance() {
+    if [ -z "$INPUT" ]; then
+        echo "expected build params in stdin"
+        exit 1
+    fi
+
+    jq -rnc '{"provider_id": "test-provider-id", "name": "test-instance-name", "os_type": "linux", "os_name": "ubuntu", "os_version": "20.04", "os_arch": "x86_64", "status": "running"}'
+}
+
+case "$GARM_COMMAND" in
+    "CreateInstance")
+        CreateInstance
+        ;;
+    "DeleteInstance")
+        echo "RemoveAllInstances not implemented"
+        exit 1
+        ;;
+    "GetInstance")
+        echo "Get instance with id: ${GARM_INSTANCE_ID}"
+        ;;
+    "ListInstances")
+        echo "List instances with pool id: ${GARM_POOL_ID}"
+        ;;
+    "StartInstance")
+        echo "Start instance: ${GARM_INSTANCE_NAME} with id: ${GARM_INSTANCE_ID}"
+        ;;
+    "StopInstance")
+        echo "Stop instance: ${GARM_INSTANCE_NAME} with id: ${GARM_INSTANCE_ID}"
+        ;;
+    "RemoveAllInstances")
+        echo "RemoveAllInstances not implemented"
+        exit 1
+        ;;
+    *)
+        echo "Invalid GARM provider command: \"$GARM_COMMAND\""
+        exit 1
+        ;;
+esac

test/integration/scripts/setup-garm.sh

@@ -5,6 +5,7 @@ DIR="$(dirname $0)"
 BINARIES_DIR="$PWD/bin"
 CONTRIB_DIR="$PWD/contrib"
 CONFIG_DIR="$PWD/test/integration/config"
+CONFIG_DIR_PROV="$PWD/test/integration/provider"
 
 if [[ ! -f $BINARIES_DIR/garm ]] || [[ ! -f $BINARIES_DIR/garm-cli ]]; then
     echo "ERROR: Please build GARM binaries first"
@@ -46,6 +47,10 @@ sudo mkdir -p /etc/garm
 cat $CONFIG_DIR/config.toml | envsubst | sudo tee /etc/garm/config.toml
 sudo chown -R garm:garm /etc/garm
 
+sudo mkdir -p /etc/garm/test-provider
+sudo touch $CONFIG_DIR_PROV/config
+sudo cp $CONFIG_DIR_PROV/* /etc/garm/test-provider  
+
 sudo mv $BINARIES_DIR/* /usr/local/bin/
 sudo cp $CONTRIB_DIR/garm.service /etc/systemd/system/garm.service