build: add integration tests (#281)

* ci: add integration tests

* fix: typo

* fix: set environment globally

* fix: wait for services

* fix: chain healthcheck

* fix: volumes issue?

* fix: secrets

* fix: admin address

* fix: typo

* fix: update nitro-node

* fix: chain-clean

* fix: stack environment variables

* fix: integration test doppler token

* fix: typo

.github/workflows/test.yml

@@ -0,0 +1,45 @@
+name: Tests
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  run-integration-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install Doppler CLI
+        uses: dopplerhq/cli-action@v1
+
+      - name: Install golang
+        uses: actions/setup-go@v5
+
+      - name: Install node
+        uses: actions/setup-node@v4
+
+      - name: Build Docker Images
+        run: docker compose -f docker/docker-compose.dev.yml build
+
+      - name: Initialize chain
+        env:
+          ADMIN_ADDRESS: ${{ secrets.INTEGRATION_TEST_ADMIN_ADDRESS }}
+        run: |
+          ./stack chain-clean
+          docker compose -f ./docker/docker-compose.dev.yml up chain -d
+          ./stack chain-boot
+
+      - name: Run stack
+        env:
+          ADMIN_ADDRESS: ${{ secrets.INTEGRATION_TEST_ADMIN_ADDRESS }}
+          DOPPLER_TOKEN_JOB_CREATOR: ${{ secrets.INTEGRATION_TEST_DOPPLER_TOKEN_JOB_CREATOR }}
+          DOPPLER_TOKEN_SOLVER: ${{ secrets.INTEGRATION_TEST_DOPPLER_TOKEN_SOLVER }}
+          WEB3_PRIVATE_KEY: ${{ secrets.INTEGRATION_TEST_WEB3_PRIVATE_KEY }}
+        run: docker compose -f ./docker/docker-compose.dev.yml up -d
+
+      - name: Run tests
+        env:
+          DOPPLER_TOKEN: ${{ secrets.INTEGRATION_TESTS_DOPPLER_TOKEN }}
+        run: ./stack integration-tests

docker/docker-compose.dev.yml

@@ -1,7 +1,7 @@
 name: lilypad
 services:
   chain:
-    image: offchainlabs/nitro-node:v2.3.4-b4cc111
+    image: offchainlabs/nitro-node:v3.1.1-beta.2-6073359
     container_name: chain
     ports:
       - 8547:8547
@@ -30,9 +30,9 @@ services:
         "0.0.0.0",
       ]
     volumes:
-      - ../data/chain:/home/user/.arbitrum
+      - chain-data:/home/user/.arbitrum
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8548/"]
+      test: ["CMD", "curl", "-f", "http://localhost:8547/"]
       interval: 30s
       timeout: 10s
       retries: 5
@@ -89,9 +89,12 @@ services:
     extra_hosts:
       - "localhost:host-gateway"
     volumes:
-      - ../data/bacalhau:/tmp/lilypad/data
+      - bacalhau-data:/tmp/lilypad/data
       - /var/run/docker.sock:/var/run/docker.sock
     environment:
       - WEB3_PRIVATE_KEY
       - BACALHAU_SERVE_IPFS_PATH=/tmp/lilypad/data/ipfs
       - LOG_LEVEL=debug
+volumes:
+  chain-data:
+  bacalhau-data:

stack

@@ -309,9 +309,8 @@ function unit-tests() {
   npx hardhat test --network hardhat
 }
 
-# this assumes chain is running
-# and chain-fund-admin
-# and chain-boot
+# this assumes stack is running
+# see LOCAL_DEVELOPMENT.md
 function integration-tests() {
   cd test
   doppler run --preserve-env -p integration-tests -c dev -- go test -v -count 1 .

test/integration_test.go

@@ -12,9 +12,7 @@ import (
 	"github.com/lilypad-tech/lilypad/pkg/jobcreator"
 	"github.com/lilypad-tech/lilypad/pkg/mediator"
 	optionsfactory "github.com/lilypad-tech/lilypad/pkg/options"
-	"github.com/lilypad-tech/lilypad/pkg/resourceprovider"
 	"github.com/lilypad-tech/lilypad/pkg/solver"
-	solvermemorystore "github.com/lilypad-tech/lilypad/pkg/solver/store/memory"
 	"github.com/lilypad-tech/lilypad/pkg/system"
 	"github.com/lilypad-tech/lilypad/pkg/web3"
 	"github.com/stretchr/testify/assert"
@@ -25,58 +23,6 @@ type testOptions struct {
 	executor         noop.NoopExecutorOptions
 }
 
-func getSolver(t *testing.T, options testOptions) (*solver.Solver, error) {
-	solverOptions := optionsfactory.NewSolverOptions()
-	solverOptions.Web3.PrivateKey = os.Getenv("SOLVER_PRIVATE_KEY")
-	solverOptions.Server.Port = 8080
-	solverOptions.Server.URL = "http://localhost:8080"
-
-	// test that the solver private key is defined
-	if solverOptions.Web3.PrivateKey == "" {
-		return nil, fmt.Errorf("SOLVER_PRIVATE_KEY is not defined")
-	}
-
-	web3SDK, err := web3.NewContractSDK(solverOptions.Web3)
-	if err != nil {
-		return nil, err
-	}
-
-	solverStore, err := solvermemorystore.NewSolverStoreMemory()
-	if err != nil {
-		return nil, err
-	}
-
-	return solver.NewSolver(solverOptions, solverStore, web3SDK)
-}
-
-func getResourceProvider(
-	t *testing.T,
-	systemContext *system.CommandContext,
-	options testOptions,
-) (*resourceprovider.ResourceProvider, error) {
-	resourceProviderOptions := optionsfactory.NewResourceProviderOptions()
-	resourceProviderOptions.Web3.PrivateKey = os.Getenv("RESOURCE_PROVIDER_PRIVATE_KEY")
-	if resourceProviderOptions.Web3.PrivateKey == "" {
-		return nil, fmt.Errorf("RESOURCE_PROVIDER_PRIVATE_KEY is not defined")
-	}
-	resourceProviderOptions, err := optionsfactory.ProcessResourceProviderOptions(resourceProviderOptions, "dev")
-	if err != nil {
-		return nil, err
-	}
-
-	web3SDK, err := web3.NewContractSDK(resourceProviderOptions.Web3)
-	if err != nil {
-		return nil, err
-	}
-
-	executor, err := noop.NewNoopExecutor(options.executor)
-	if err != nil {
-		return nil, err
-	}
-
-	return resourceprovider.NewResourceProvider(resourceProviderOptions, web3SDK, executor)
-}
-
 func getMediator(
 	t *testing.T,
 	systemContext *system.CommandContext,
@@ -131,25 +77,10 @@ func testStackWithOptions(
 	commandCtx *system.CommandContext,
 	options testOptions,
 ) (*jobcreator.RunJobResults, error) {
-
-	solver, err := getSolver(t, options)
-	if err != nil {
-		return nil, err
-	}
-
-	solver.Start(commandCtx.Ctx, commandCtx.Cm)
-
 	// give the solver server a chance to boot before we get all the websockets
 	// up and trying to connect to it
 	time.Sleep(100 * time.Millisecond)
 
-	resourceProvider, err := getResourceProvider(t, commandCtx, options)
-	if err != nil {
-		return nil, err
-	}
-
-	resourceProvider.Start(commandCtx.Ctx, commandCtx.Cm)
-
 	mediator, err := getMediator(t, commandCtx, options)
 	if err != nil {
 		return nil, err
@@ -187,7 +118,7 @@ func TestNoModeration(t *testing.T) {
 	})
 
 	assert.NoError(t, err, "there was an error running the job")
-	assert.Equal(t, "123", result.Result.DataID, "the data ID was correct")
+	assert.Equal(t, "QmbCi3yoKzckff24rUJML1ZesVb35cd2LUNMiMYksEGkWv", result.Result.DataID, "the data ID was correct")
 
 	localPath := solver.GetDownloadsFilePath(result.Result.DealID)