From fdcbd95219eff97bd09d5c8a708a194b09169ae9 Mon Sep 17 00:00:00 2001 From: Narb <29411347+narbs91@users.noreply.github.com> Date: Tue, 27 Aug 2024 16:14:04 -0700 Subject: [PATCH] refactor: remove doppler depedency from local development (#291) * Removes Doppler depedency on local start up - Created a .env.local file with all the variables needed to run Lilypad locally without the need of a Doppler account. We agreed in chat that the WEB3_PRIVATE_KEY (and realted keys) is ok to have on the repo in this instance as these are all derived from Hardhat and already public on the repo - Updated the Stack file to remove the Doppler secrets injection and instead introduecd a new method that loads and sets the env variables from .en.local file - updated .gitignore to allow the .env.local file to be uploaded * Enables running app with docker locally w/o doppler * renames .env.local to .local.dev * makes WEB3_PRIVATE_KEY take on values from .local.dev * Refactors test.yml to use stack commands * Updates LOCAL_DEVELOPMENT file with notes on .local.dev * adds troubleshooting section to local dev readme * updates job creator key * increase sleep time to 20 seconds for integration test * Adds troubleshooting guide to local development guide * Adds comments to .local.dev file * Removes API_HOST env var from docker compose * Adds volume clean up step in compose-init * Refactors solver Dockerfile --------- Co-authored-by: Alvin Reyes --- .github/workflows/test.yml | 18 ++--------- .local.dev | 29 +++++++++++++++++ LOCAL_DEVELOPMENT.md | 32 ++++++++++++++++++- docker/docker-compose.dev.yml | 11 +++++-- docker/job-creator/Dockerfile | 6 +++- docker/solver/Dockerfile | 8 +++-- stack | 59 ++++++++++++++++++++++++++++------- test/integration_test.go | 2 +- 8 files changed, 130 insertions(+), 35 deletions(-) create mode 100644 .local.dev diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ce3cae87..d1e8c054 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,25 +21,13 @@ jobs: uses: actions/setup-node@v4 - name: Build Docker Images - run: docker compose -f docker/docker-compose.dev.yml build + run: ./stack compose-build - name: Initialize chain - env: - ADMIN_ADDRESS: ${{ secrets.INTEGRATION_TEST_ADMIN_ADDRESS }} - run: | - ./stack chain-clean - docker compose -f ./docker/docker-compose.dev.yml up chain -d - ./stack chain-boot + run: ./stack compose-init - name: Run stack - env: - ADMIN_ADDRESS: ${{ secrets.INTEGRATION_TEST_ADMIN_ADDRESS }} - DOPPLER_TOKEN_JOB_CREATOR: ${{ secrets.INTEGRATION_TEST_DOPPLER_TOKEN_JOB_CREATOR }} - DOPPLER_TOKEN_SOLVER: ${{ secrets.INTEGRATION_TEST_DOPPLER_TOKEN_SOLVER }} - WEB3_PRIVATE_KEY: ${{ secrets.INTEGRATION_TEST_WEB3_PRIVATE_KEY }} - run: docker compose -f ./docker/docker-compose.dev.yml up -d + run: ./stack compose-up -d - name: Run tests - env: - DOPPLER_TOKEN: ${{ secrets.INTEGRATION_TESTS_DOPPLER_TOKEN }} run: ./stack integration-tests diff --git a/.local.dev b/.local.dev new file mode 100644 index 00000000..43fc8c23 --- /dev/null +++ b/.local.dev @@ -0,0 +1,29 @@ +# Running the Lilypad application locally depends on the `.local.dev` file for secrets injection. +# This file contains a series of private keys (with no funds on them) that are ONLY meant to be used +# for testing this app locally. You are free to replace these keys with your own if you wish; however, +# be warned that the `.local.dev` file is not included in the `.gitignore`. Therefore, you must be +# vigilant not to commit this file in your PRs. We are not responsible for lost funds as a result of +# posting your private keys in your commits/PRs. Unless you have a very good reason to do so, +# leave the `.local.dev` file unchanged. + +API_HOST=http://localhost:8002/ +WEB3_RPC_URL=ws://localhost:8548 +SERVER_PORT=8080 +SERVER_URL=http://localhost:8080 +DIRECTORY_ADDRESS=0x976EA74026E726554dB657fA54763abd0C3a0aa9 +JOB_CREATOR_ADDRESS=0x9965507D1a55bcC2695C58ba16FB37d819B0A4dc +JOB_CREATOR_PRIVATE_KEY=0x5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a +MEDIATOR_PRIVATE_KEY=0x7c852118294e51e653712a81e05800f419141751be58f605c371e15141b007a6 +RESOURCE_PROVIDER_PRIVATE_KEY=0x47e179ec197488593b187f80a00eb0da91f1b9d0b13f8733639f19c30a34926a +SERVICE_SOLVER=0x3C44CdDdB6a900fa2b585dd299e03d12FA4293BC +SOLVER_PRIVATE_KEY=0x5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a +RUN_PRIVATE_KEY=b3994e7660abe5f65f729bb64163c6cd6b7d0b1a8c67881a7346e3e8c7f026f5 +WEB3_CHAIN_ID=412346 +WEB3_CONTROLLER_ADDRESS=0xa85233C63b9Ee964Add6F2cffe00Fd84eb32338f +WEB3_JOBCREATOR_ADDRESS_=0x9A9f2CCfdE556A7E9Ff0848998Aa4a0CFD8863AE +WEB3_MEDIATOR_ADDRESS_=0x0B306BF915C4d645ff596e518fAf3F9669b97016 +WEB3_PAYMENTS_ADDRESS=0x2279B7A0a67DB372996a5FaB50D91eAA73d2eBe6 +WEB3_POW_ADDRESS_=0x4ed7c70F96B99c776995fB64377f0d4aB3B0e1C1 +WEB3_STORAGE_ADDRESS_=0xB7f8BC63BbcaD18155201308C8f3540b07f84F5e +WEB3_TOKEN_ADDRESS_=0xa513E6E4b8f2a923D98304ec87F64353C4D5C853 +WEB3_USERS_ADDRESS=0x0DCd1Bf9A1b36cE34237eEaFef220932846BCD82 \ No newline at end of file diff --git a/LOCAL_DEVELOPMENT.md b/LOCAL_DEVELOPMENT.md index 33e84a66..bf9792a5 100644 --- a/LOCAL_DEVELOPMENT.md +++ b/LOCAL_DEVELOPMENT.md @@ -5,10 +5,11 @@ - [Golang](https://go.dev/doc/install) - [node](https://nodejs.org/en/download/package-manager) - [Docker](https://docs.docker.com/engine/install/) -- [Doppler](https://docs.doppler.com/docs/install-cli) ## Getting started +Running the Lilypad application locally depends on the `.local.dev` file for secrets injection. In this file are a series of private keys (with no funds on them) that are ONLY meant to be used for testing this app locally. You are free to replace these keys with your own if you wish; however, be warned that the `.local.dev` file is not included in the `.gitignore` so you must be vigilant to not commit this file in your PRs. We are not responsible for lost funds as a result of you posting your private keys on your commits/prs. So unless you have a very good reason to do so, leave the `.local.dev` file unchanged. + A minimal local Lilypad network consists of the following pieces of infrastructure: - One blockchain node @@ -114,3 +115,32 @@ There are two commands that can be used to run existing tests: `./stack unit-tes ## Notes on tooling Things should work right out-of-the-box, no extra configuration should be needed as Doppler provides the environment variables that are required with the current setup. + +## Troubleshooting + +In this section we'll address some common problems you might face when trying to boot up Lilypad locally + +### Chain-boot Related issues + +If you try and run `./stack compose-init` or `./stack chain-boot` and get the following error +```bash +ProviderError: failed with 51333200 gas: insufficient funds for gas * price + value: address 0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266 have 9318991353400000000 want 10000000000000000000 + at HttpProvider.request (/Users/nshahnazarian/Development/git/lilypad/hardhat/node_modules/hardhat/src/internal/core/providers/http.ts:88:21) + at processTicksAndRejections (node:internal/process/task_queues:95:5) + at async HardhatEthersProvider.estimateGas (/Users/nshahnazarian/Development/git/lilypad/hardhat/node_modules/@nomicfoundation/hardhat-ethers/src/internal/hardhat-ethers-provider.ts:237:27) + at async Wallet.populateTransaction (/Users/nshahnazarian/Development/git/lilypad/hardhat/node_modules/ethers/src.ts/providers/abstract-signer.ts:105:28) + at async Wallet.sendTransaction (/Users/nshahnazarian/Development/git/lilypad/hardhat/node_modules/ethers/src.ts/providers/abstract-signer.ts:232:21) + at async transferEther (/Users/nshahnazarian/Development/git/lilypad/hardhat/utils/web3.ts:61:14) + at async /Users/nshahnazarian/Development/git/lilypad/hardhat/scripts/fund-services-ether.ts:15:5 +``` +This can be addressed by doing the following: +- Open your Docker Desktop app, go to `Volumes` and delete `lilypad_chain-data` as there might be stale data in the volume not allowing you to properly execute all the transactions `chain-boot` executes + +### Issues running onchain cowsay + +If you find that you have issues with the Job Creator not picking up your `run-cowsay-onchain` command while running the Lilypad stack through Docker, do the following: +1. Stop the Docker stack by pressing ctrl+c +2. Run the following command to clean up your Docker environment: `./stack compose-down && docker system prune -a` +3. Open your Docker Desktop app, go to `Volumes` and delete `lilypad_chain-data` as there might be stale data in the volume not allowing you to properly execute all the transactions +4. Re-run your Docker stack using: `./stack compose-build && ./stack compose-init && ./stack compose-up` +5. Re-run the onchain cowsay job: `./stack run-cowsay-onchain` \ No newline at end of file diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index e8c968e9..6bc10feb 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -49,7 +49,11 @@ services: extra_hosts: - "localhost:host-gateway" environment: - - DOPPLER_TOKEN=${DOPPLER_TOKEN_SOLVER} + - WEB3_PRIVATE_KEY=${SOLVER_PRIVATE_KEY} + - LOG_LEVEL=debug + - SERVER_PORT=${SERVER_PORT} + - SERVER_URL=${SERVER_URL} + - WEB3_RPC_URL=${WEB3_RPC_URL} ports: - 8080:8080 healthcheck: @@ -73,7 +77,8 @@ services: extra_hosts: - "localhost:host-gateway" environment: - - DOPPLER_TOKEN=${DOPPLER_TOKEN_JOB_CREATOR} + - WEB3_PRIVATE_KEY=${JOB_CREATOR_PRIVATE_KEY} + - LOG_LEVEL=debug resource-provider: image: ghcr.io/lilypad-tech/resource-provider container_name: resource-provider @@ -93,7 +98,7 @@ services: - bacalhau-data:/tmp/lilypad/data - /var/run/docker.sock:/var/run/docker.sock environment: - - WEB3_PRIVATE_KEY + - WEB3_PRIVATE_KEY=${RESOURCE_PROVIDER_PRIVATE_KEY} - BACALHAU_SERVE_IPFS_PATH=/tmp/lilypad/data/ipfs - LOG_LEVEL=debug volumes: diff --git a/docker/job-creator/Dockerfile b/docker/job-creator/Dockerfile index eb3db633..c2e69408 100644 --- a/docker/job-creator/Dockerfile +++ b/docker/job-creator/Dockerfile @@ -12,7 +12,11 @@ RUN (curl -Ls --tlsv1.2 --proto "=https" --retry 3 https://cli.doppler.com/insta RUN touch run RUN echo "#!/bin/bash" >> run -RUN echo "doppler run -- lilypad jobcreator --network $network" >> run +RUN if [ "${network}" = "dev" ]; then \ + echo "lilypad jobcreator --network $network" >> run; \ + else \ + echo "doppler run -- lilypad jobcreator --network $network" >> run; \ + fi RUN chmod +x run CMD ["/bin/bash", "./run"] diff --git a/docker/solver/Dockerfile b/docker/solver/Dockerfile index 09026e86..73c6e088 100644 --- a/docker/solver/Dockerfile +++ b/docker/solver/Dockerfile @@ -19,11 +19,15 @@ RUN echo "#!/bin/bash" >> run FROM base AS expose-cloudflare RUN curl -L --output cloudflared.deb https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-${arch}.deb RUN dpkg -i cloudflared.deb -RUN echo "doppler run --command \"cloudflared tunnel run & lilypad solver --network $network\"" >> run +RUN echo "doppler run --command \"cloudflared tunnel run & lilypad solver --network ${network}\"" >> run FROM base AS expose-local EXPOSE 8080 -RUN echo "doppler run -- lilypad solver --network $network" >> run +RUN if [ "${network}" = "dev" ]; then \ + echo "lilypad solver --network $network" >> run; \ + else \ + echo "doppler run -- lilypad solver --network $network" >> run; \ + fi FROM expose-$expose_via AS final RUN chmod +x run diff --git a/stack b/stack index a8c1b41a..c7e3331b 100755 --- a/stack +++ b/stack @@ -10,25 +10,28 @@ OS_ARCH=$(uname -m | awk '{if ($0 ~ /arm64|aarch64/) print "arm64"; else if ($0 ############################################################################ function compose-env() { export ADMIN_ADDRESS=${@:-"0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266"} - export DOPPLER_TOKEN_SOLVER="$(doppler configs tokens create -p solver -c dev docker --max-age 10m --plain)" - export DOPPLER_TOKEN_JOB_CREATOR="$(doppler configs tokens create -p job-creator -c dev docker --max-age 10m --plain)" } function compose-init() { compose-env chain-clean + if docker volume ls -q | grep -q "^lilypad_chain-data$"; then + docker volume rm lilypad_chain-data + fi docker compose -f ./docker/docker-compose.dev.yml up chain -d chain-boot } function compose-build() { + load-local-env compose-env docker compose -f ./docker/docker-compose.dev.yml build "$@" } function compose-up() { + load-local-env compose-env - doppler run -p resource-provider -c dev -- docker compose -f ./docker/docker-compose.dev.yml up "$@" + docker compose -f ./docker/docker-compose.dev.yml up "$@" } function compose-down() { @@ -36,6 +39,19 @@ function compose-down() { docker compose -f ./docker/docker-compose.dev.yml down } + +############################################################################ +# Load env variables from .local.dev +############################################################################ +function load-local-env() { + while IFS= read -r line || [ -n "$line" ]; do + # Skip lines that are empty, start with a #, or are just whitespace + if [[ -n "$line" && ! "$line" =~ ^[[:space:]]*# ]]; then + export $line + fi + done < .local.dev +} + ############################################################################ # chain ############################################################################ @@ -176,17 +192,22 @@ function print-local-dev-env() { } function run-cowsay-onchain() { + load-local-env cd hardhat - doppler run --preserve-env -p run-cowsay-onchain -c dev -- npx hardhat run scripts/run-cowsay-onchain.ts + npx hardhat run scripts/run-cowsay-onchain.ts } ############################################################################ # solver +# +# Note: The presence of the WEB3_PRIVATE_KEY here is only necessary for local development. You are advised not to import this key into a wallet nor use it for anything other for testing Lilypad locally ############################################################################ function solver() { - echo "- Reminder to do doppler setup to project->solver and config->dev" - doppler run --preserve-env -p solver -c dev -- go run . solver --network dev + load-local-env + export WEB3_PRIVATE_KEY=${SOLVER_PRIVATE_KEY} + export LOG_LEVEL=debug + go run . solver --network dev } function solver-docker-build() { @@ -210,11 +231,15 @@ function solver-docker-run() { ############################################################################ # job creator +# +# Note: The presence of the WEB3_PRIVATE_KEY here is only necessary for local development. You are advised not to import this key into a wallet nor use it for anything other for testing Lilypad locally ############################################################################ function job-creator() { - echo "- Reminder to do doppler setup to project->job-creator and config->dev" - doppler run --preserve-env -p job-creator -c dev -- go run . jobcreator --network dev + load-local-env + export WEB3_PRIVATE_KEY=${JOB_CREATOR_PRIVATE_KEY} + export LOG_LEVEL=debug + go run . jobcreator --network dev } function job-creator-docker-build() { @@ -236,11 +261,15 @@ function job-creator-docker-run() { ############################################################################ # resource provider +# +# Note: The presence of the WEB3_PRIVATE_KEY here is only necessary for local development. You are advised not to import this key into a wallet nor use it for anything other for testing Lilypad locally ############################################################################ function resource-provider() { - echo "- Reminder to do doppler setup to project->resource-provider and config->dev" - doppler run --preserve-env -p resource-provider -c dev -- go run . resource-provider "$@" --network dev --disable-pow true + load-local-env + export WEB3_PRIVATE_KEY=${RESOURCE_PROVIDER_PRIVATE_KEY} + export LOG_LEVEL=debug + go run . resource-provider "$@" --network dev --disable-pow true } function resource-provider-docker-build() { @@ -312,15 +341,21 @@ function unit-tests() { # this assumes stack is running # see LOCAL_DEVELOPMENT.md function integration-tests() { + load-local-env cd test - doppler run --preserve-env -p integration-tests -c dev -- go test -v -count 1 . + go test -v -count 1 . } ############################################################################ # run +# +# Note: The presence of the WEB3_PRIVATE_KEY here is only necessary for local development. You are advised not to import this key into a wallet nor use it for anything other for testing Lilypad locally ############################################################################ function run() { - doppler run --preserve-env -p run -c dev -- go run . run --network dev "$@" + load-local-env + export WEB3_PRIVATE_KEY=${RUN_PRIVATE_KEY} + export LOG_LEVEL=info + go run . run --network dev "$@" } eval "$@" diff --git a/test/integration_test.go b/test/integration_test.go index 60fe66a3..ee3bb025 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -79,7 +79,7 @@ func testStackWithOptions( ) (*jobcreator.RunJobResults, error) { // give the solver server a chance to boot before we get all the websockets // up and trying to connect to it - time.Sleep(100 * time.Millisecond) + time.Sleep(20000 * time.Millisecond) mediator, err := getMediator(t, commandCtx, options) if err != nil {