From 878c4fc4b1ca844b3c940a1d34acf7b4c3fb6ff4 Mon Sep 17 00:00:00 2001 From: Flouse <1297478+Flouse@users.noreply.github.com> Date: Sat, 14 Oct 2023 16:55:21 +0000 Subject: [PATCH 1/9] CI: add MANUAL_BUILD tests --- .github/workflows/godwoken-tests.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/godwoken-tests.yml b/.github/workflows/godwoken-tests.yml index 183eb67..f4c3072 100644 --- a/.github/workflows/godwoken-tests.yml +++ b/.github/workflows/godwoken-tests.yml @@ -9,8 +9,22 @@ on: jobs: godwoken-tests: + strategy: + fail-fast: false + matrix: + MANUAL_BUILD_GODWOKEN: ["true", "false"] + MANUAL_BUILD_WEB3: ["true", "false"] + MANUAL_BUILD_WEB3_INDEXER: ["true", "false"] + MANUAL_BUILD_POLYJUICE: ["true", "false"] + MANUAL_BUILD_SCRIPTS: ["true", "false"] + uses: godwokenrises/godwoken-tests/.github/workflows/reusable-integration-test-v1.yml@develop with: extra_github_env: | GODWOKEN_KICKER_REPO=${{ github.repository }} GODWOKEN_KICKER_REF=${{ github.ref }} + MANUAL_BUILD_GODWOKEN=${{ matrix.MANUAL_BUILD_GODWOKEN }} + MANUAL_BUILD_WEB3=${{ matrix.MANUAL_BUILD_WEB3 }} + MANUAL_BUILD_WEB3_INDEXER=${{ matrix.MANUAL_BUILD_WEB3_INDEXER }} + MANUAL_BUILD_POLYJUICE=${{ matrix.MANUAL_BUILD_POLYJUICE }} + MANUAL_BUILD_SCRIPTS=${{ matrix.MANUAL_BUILD_SCRIPTS }} From 70e459ba396c3defcf101af84d6e2ab97dd94e24 Mon Sep 17 00:00:00 2001 From: Flouse <1297478+Flouse@users.noreply.github.com> Date: Sat, 14 Oct 2023 17:09:19 +0000 Subject: [PATCH 2/9] Fix the default MANUAL_BUILD refs --- kicker | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/kicker b/kicker index 140c978..327d8bd 100755 --- a/kicker +++ b/kicker @@ -415,25 +415,37 @@ function withdraw_v0_to_v1() { # GODWOKEN_GIT_CHECKOUT=develop \ # ./kicker manual-build function manual_build() { - DOCKER_MANUAL_BUILD_IMAGE_NAME=${DOCKER_MANUAL_BUILD_IMAGE_NAME:-"retricsu/godwoken-manual-build"} - DOCKER_MANUAL_BUILD_IMAGE_TAG=${DOCKER_MANUAL_BUILD_IMAGE_TAG:-"node18"} - DOCKER_MANUAL_BUILD_IMAGE="${DOCKER_MANUAL_BUILD_IMAGE:-"$DOCKER_MANUAL_BUILD_IMAGE_NAME:$DOCKER_MANUAL_BUILD_IMAGE_TAG"}" - + WEB3_GIT_URL=${WEB3_GIT_URL:-"https://github.com/godwokenrises/godwoken"} + WEB3_GIT_CHECKOUT=${WEB3_GIT_CHECKOUT:-"develop"} echo "MANUAL_BUILD_WEB3 = \"$MANUAL_BUILD_WEB3\"" echo "WEB3_GIT_URL = \"$WEB3_GIT_URL\"" echo "WEB3_GIT_CHECKOUT = \"$WEB3_GIT_CHECKOUT\"" echo "MANUAL_BUILD_WEB3_INDEXER = \"$MANUAL_BUILD_WEB3_INDEXER\"" + + GODWOKEN_GIT_URL=${GODWOKEN_GIT_URL:-"https://github.com/godwokenrises/godwoken"} + GODWOKEN_GIT_CHECKOUT=${GODWOKEN_GIT_CHECKOUT:-"develop"} echo "MANUAL_BUILD_GODWOKEN = \"$MANUAL_BUILD_GODWOKEN\"" echo "GODWOKEN_GIT_URL = \"$GODWOKEN_GIT_URL\"" echo "GODWOKEN_GIT_CHECKOUT = \"$GODWOKEN_GIT_CHECKOUT\"" + + SCRIPTS_GIT_URL=${SCRIPTS_GIT_URL:-"https://github.com/godwokenrises/godwoken"} + SCRIPTS_GIT_CHECKOUT=${SCRIPTS_GIT_CHECKOUT:-"develop"} echo "MANUAL_BUILD_SCRIPTS = \"$MANUAL_BUILD_SCRIPTS\"" echo "SCRIPTS_GIT_URL = \"$SCRIPTS_GIT_URL\"" echo "SCRIPTS_GIT_CHECKOUT = \"$SCRIPTS_GIT_CHECKOUT\"" + echo "OMNI_LOCK_GIT_URL = \"$OMNI_LOCK_GIT_URL\"" echo "OMNI_LOCK_GIT_CHECKOUT = \"$OMNI_LOCK_GIT_CHECKOUT\"" + + POLYJUICE_GIT_URL=${POLYJUICE_GIT_URL:-"https://github.com/godwokenrises/godwoken"} + POLYJUICE_GIT_CHECKOUT=${POLYJUICE_GIT_CHECKOUT:-"develop"} echo "MANUAL_BUILD_POLYJUICE = \"$MANUAL_BUILD_POLYJUICE\"" echo "POLYJUICE_GIT_URL = \"$POLYJUICE_GIT_URL\"" echo "POLYJUICE_GIT_CHECKOUT = \"$POLYJUICE_GIT_CHECKOUT\"" + + DOCKER_MANUAL_BUILD_IMAGE_NAME=${DOCKER_MANUAL_BUILD_IMAGE_NAME:-"retricsu/godwoken-manual-build"} + DOCKER_MANUAL_BUILD_IMAGE_TAG=${DOCKER_MANUAL_BUILD_IMAGE_TAG:-"node18"} + DOCKER_MANUAL_BUILD_IMAGE="${DOCKER_MANUAL_BUILD_IMAGE:-"$DOCKER_MANUAL_BUILD_IMAGE_NAME:$DOCKER_MANUAL_BUILD_IMAGE_TAG"}" echo "DOCKER_MANUAL_BUILD_IMAGE = \"$DOCKER_MANUAL_BUILD_IMAGE\"" echo From 4927aa7d02d8d3d5bc3558053475c67e052d701c Mon Sep 17 00:00:00 2001 From: Flouse <1297478+Flouse@users.noreply.github.com> Date: Sat, 14 Oct 2023 17:23:55 +0000 Subject: [PATCH 3/9] fixup! Fix the default MANUAL_BUILD refs --- kicker | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kicker b/kicker index 327d8bd..bbb84af 100755 --- a/kicker +++ b/kicker @@ -563,7 +563,7 @@ function manual_build() { dstdir=$WORKSPACE/docker/manual-artifacts/polyjuice/ # Download repo - prepare_repo godwoken-polyjuice "$POLYJUICE_GIT_URL" "$POLYJUICE_GIT_CHECKOUT" + prepare_repo godwoken "$POLYJUICE_GIT_URL" "$POLYJUICE_GIT_CHECKOUT" cd $srcdir erun make all-via-docker @@ -580,7 +580,7 @@ function manual_build() { dstdir=$WORKSPACE/docker/manual-artifacts/scripts/ # Download repo - prepare_repo godwoken-scripts "$SCRIPTS_GIT_URL" "$SCRIPTS_GIT_CHECKOUT" + prepare_repo godwoken "$SCRIPTS_GIT_URL" "$SCRIPTS_GIT_CHECKOUT" # Install capsule # TODO: use cpasule from godwoken-manual-build image From 32738174032935174cdfa349e3962b6597ecb9d1 Mon Sep 17 00:00:00 2001 From: Flouse <1297478+Flouse@users.noreply.github.com> Date: Sun, 15 Oct 2023 00:50:20 +0000 Subject: [PATCH 4/9] Fix DOCKER_MANUAL_BUILD_IMAGE --- kicker | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kicker b/kicker index bbb84af..ec70f6a 100755 --- a/kicker +++ b/kicker @@ -506,7 +506,7 @@ function manual_build() { --volume $srcdir:/app \ --volume $WORKSPACE/packages/.rustup:/root/.rustup \ --workdir /app/web3 \ - retricsu/godwoken-manual-build:ckb2021 cargo build --locked --release + $DOCKER_MANUAL_BUILD_IMAGE cargo build --locked --release # Copy the built artifacts to `docker/manual-artifacts/gw-web3-indexer` # @@ -542,7 +542,7 @@ function manual_build() { --volume $srcdir:/app \ --volume $WORKSPACE/packages/.rustup:/root/.rustup \ --workdir /app \ - retricsu/godwoken-manual-build:ckb2021 \ + $DOCKER_MANUAL_BUILD_IMAGE \ cargo build --locked --release --features gw-config/no-builtin # Copy the built artifacts to `docker/manual-artifacts/` From 66b1a1a2d92f0fa8fb903fe9f60e7ba8bcebe3e7 Mon Sep 17 00:00:00 2001 From: Flouse <1297478+Flouse@users.noreply.github.com> Date: Sun, 15 Oct 2023 02:32:41 +0000 Subject: [PATCH 5/9] Fix function manual_build --- docker/docker-compose.yml | 2 ++ kicker | 38 +++++++++++++++++++++++--------------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index b7f50d9..407ce96 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -21,6 +21,7 @@ services: volumes: - ./redis/data:/data + # TODO: update CKB version ckb: image: nervos/ckb:v0.103.0 user: root @@ -31,6 +32,7 @@ services: - ./layer1/ckb:/var/lib/ckb command: [ "run", "-C", "/var/lib/ckb" ] + # TODO: update CKB version ckb-miner: init: true image: nervos/ckb:v0.103.0 diff --git a/kicker b/kicker index ec70f6a..a9e0c5e 100755 --- a/kicker +++ b/kicker @@ -449,16 +449,19 @@ function manual_build() { echo "DOCKER_MANUAL_BUILD_IMAGE = \"$DOCKER_MANUAL_BUILD_IMAGE\"" echo + info "Start building..." | tee /tmp/kicker.log + du -hd6 $WORKSPACE | egrep G | tee --append /tmp/kicker.log + if [ "$MANUAL_BUILD_WEB3" = "true" ]; then info "Start building godwoken-web3" # godwoken-web3 is managed in the Godwoken monorepo. # See https://github.com/godwokenrises/godwoken/tree/develop/web3 - srcdir=$WORKSPACE/packages/godwoken-web3/web3 + srcdir=$WORKSPACE/packages/godwoken/web3 dstdir=$WORKSPACE/docker/manual-artifacts/godwoken-web3 # Download repo - prepare_repo godwoken-web3 "$WEB3_GIT_URL" "$WEB3_GIT_CHECKOUT" + prepare_repo godwoken "$WEB3_GIT_URL" "$WEB3_GIT_CHECKOUT" # Yarn install via docker (assumes the "web3" docker-compose service installed yarn) @@ -490,34 +493,34 @@ function manual_build() { if [ "$MANUAL_BUILD_WEB3_INDEXER" = "true" ]; then info "Start building godwoken-web3-indexer" - srcdir=$WORKSPACE/packages/godwoken-web3 + srcdir=$WORKSPACE/packages/godwoken dstdir=$WORKSPACE/docker/manual-artifacts # Download repo - prepare_repo godwoken-web3 "$WEB3_GIT_URL" "$WEB3_GIT_CHECKOUT" + prepare_repo godwoken "$WEB3_GIT_URL" "$WEB3_GIT_CHECKOUT" # Cargo fetch Rust dependencies (in order to access network via # host network). The docker image must have installed cargo, molecule # and rustfmt. - erun "cd $srcdir/web3 && CARGO_HOME=$srcdir/web3/.cargo cargo fetch --locked && cd -" - erun docker run \ - --rm \ - --env CARGO_HOME=/app/.cargo \ + erun "cd $srcdir/web3 && CARGO_HOME=$WORKSPACE/packages/.cargo cargo fetch --locked && cd -" + + erun docker run --rm \ + --workdir /app/web3 \ --volume $srcdir:/app \ --volume $WORKSPACE/packages/.rustup:/root/.rustup \ - --workdir /app/web3 \ - $DOCKER_MANUAL_BUILD_IMAGE cargo build --locked --release + --env CARGO_HOME=$WORKSPACE/packages/.cargo \ + $DOCKER_MANUAL_BUILD_IMAGE cargo build --locked # Copy the built artifacts to `docker/manual-artifacts/gw-web3-indexer` # # More: ./docker/manual-web3-indexer.compose.yml erun mkdir -p $dstdir - erun cp $srcdir/web3/target/release/gw-web3-indexer $dstdir + erun cp $srcdir/web3/target/debug/gw-web3-indexer $dstdir # Remove Rust target to avoid `out of disk` issue in CI - sudo rm -rf $srcdir/web3/target/release + sudo rm -rf $srcdir/target else - info "skip building godwoken-web3-indexer(gw-web3-indexer)" + info "skip building godwoken-web3-indexer (gw-web3-indexer)" fi if [ "$MANUAL_BUILD_GODWOKEN" = "true" ]; then @@ -538,11 +541,11 @@ function manual_build() { # to `cargo build` arguments. erun docker run \ --rm \ - --env CARGO_HOME=/app/.cargo \ + --env CARGO_HOME=$WORKSPACE/packages/.cargo \ --volume $srcdir:/app \ --volume $WORKSPACE/packages/.rustup:/root/.rustup \ --workdir /app \ - $DOCKER_MANUAL_BUILD_IMAGE \ + godwoken \ cargo build --locked --release --features gw-config/no-builtin # Copy the built artifacts to `docker/manual-artifacts/` @@ -607,6 +610,11 @@ function manual_build() { else info "skip building Scripts" fi + + docker rmi $DOCKER_MANUAL_BUILD_IMAGE + + info "End building." | tee --append /tmp/kicker.log + du -hd6 $WORKSPACE | egrep G | tee --append /tmp/kicker.log } function prepare_repo() { From b6e8201b16fc67cad8de2f21d28459f17ecdc426 Mon Sep 17 00:00:00 2001 From: Flouse <1297478+Flouse@users.noreply.github.com> Date: Sun, 15 Oct 2023 03:30:11 +0000 Subject: [PATCH 6/9] Fix `kicker manual_build` Diskspace is limited in GitHub Actions, so we need to manually clean the middle artifacts --- kicker | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/kicker b/kicker index a9e0c5e..8ed106a 100755 --- a/kicker +++ b/kicker @@ -2,6 +2,16 @@ set -o errexit +# FIXME +GITHUB_ACTIONS_CLEAN_MID_ARTIFACTS=true +# https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables +if [ "$CI" = "true" ] && [ -n "$GITHUB_RUN_ID" ]; then + echo "This script is running in GitHub Actions. GITHUB_RUN_ID: $GITHUB_RUN_ID" + + # Diskspace is limited in GitHub Actions, so we need to manually clean the middle artifacts + GITHUB_ACTIONS_CLEAN_MID_ARTIFACTS=true +fi + EXECUTABLE=$0 WORKSPACE="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" @@ -486,6 +496,10 @@ function manual_build() { $srcdir/node_modules \ $srcdir/yarn.lock \ $dstdir + + if [[ "$GITHUB_ACTIONS_CLEAN_MID_ARTIFACTS" = "true" ]]; then + erun sudo rm -rf $srcdir/node_modules + fi else info "skip building godwoken-web3" fi @@ -518,7 +532,9 @@ function manual_build() { erun cp $srcdir/web3/target/debug/gw-web3-indexer $dstdir # Remove Rust target to avoid `out of disk` issue in CI - sudo rm -rf $srcdir/target + if [[ "$GITHUB_ACTIONS_CLEAN_MID_ARTIFACTS" = "true" ]]; then + sudo rm -rf $srcdir/web3/target + fi else info "skip building godwoken-web3-indexer (gw-web3-indexer)" fi @@ -534,29 +550,32 @@ function manual_build() { # Cargo fetch Rust dependencies (in order to access network via # host network). + erun "cd $srcdir && CARGO_HOME=$WORKSPACE/packages/.cargo cargo fetch --locked && cd -" + # Note: # 1. The docker image must have installed cargo and molecule # 2. The builtin consensus config and binaries are not required on a # Godwoken devnet, so `--features gw-config/no-builtin` is appended # to `cargo build` arguments. - erun docker run \ - --rm \ + erun docker run --rm \ --env CARGO_HOME=$WORKSPACE/packages/.cargo \ --volume $srcdir:/app \ --volume $WORKSPACE/packages/.rustup:/root/.rustup \ --workdir /app \ - godwoken \ - cargo build --locked --release --features gw-config/no-builtin + $DOCKER_MANUAL_BUILD_IMAGE \ + cargo build --locked --features gw-config/no-builtin # Copy the built artifacts to `docker/manual-artifacts/` # # More: ./docker/manual-godwoken.compose.yml erun mkdir -p $dstdir - erun cp $srcdir/target/release/godwoken $dstdir - erun cp $srcdir/target/release/gw-tools $dstdir + erun cp $srcdir/target/debug/godwoken $dstdir + erun cp $srcdir/target/debug/gw-tools $dstdir # Remove Rust target to avoid `out of disk` issue in CI - sudo rm -rf $srcdir/target/release + if [[ "$GITHUB_ACTIONS_CLEAN_MID_ARTIFACTS" = "true" ]]; then + sudo rm -rf $srcdir/target + fi else info "skip building Godwoken" fi @@ -611,7 +630,12 @@ function manual_build() { info "skip building Scripts" fi - docker rmi $DOCKER_MANUAL_BUILD_IMAGE + if [[ "$GITHUB_ACTIONS_CLEAN_MID_ARTIFACTS" = "true" ]]; then + docker images --format '{{.Repository}}:{{.Tag}}' \ + | grep '$DOCKER_MANUAL_BUILD_IMAGE' | xargs docker rmi \ + && echo "Image removed." \ + || echo "Image does not exist." + fi info "End building." | tee --append /tmp/kicker.log du -hd6 $WORKSPACE | egrep G | tee --append /tmp/kicker.log From fdc117b8aade56d65cd7c16ef65e8c4446add8ef Mon Sep 17 00:00:00 2001 From: Flouse <1297478+Flouse@users.noreply.github.com> Date: Sun, 15 Oct 2023 03:34:25 +0000 Subject: [PATCH 7/9] fixup! Fix `kicker manual_build` --- kicker | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kicker b/kicker index 8ed106a..0d456ce 100755 --- a/kicker +++ b/kicker @@ -2,8 +2,7 @@ set -o errexit -# FIXME -GITHUB_ACTIONS_CLEAN_MID_ARTIFACTS=true +GITHUB_ACTIONS_CLEAN_MID_ARTIFACTS=false # https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables if [ "$CI" = "true" ] && [ -n "$GITHUB_RUN_ID" ]; then echo "This script is running in GitHub Actions. GITHUB_RUN_ID: $GITHUB_RUN_ID" From df3036227d96496c0f69005ba58efb02cce887b4 Mon Sep 17 00:00:00 2001 From: Flouse <1297478+Flouse@users.noreply.github.com> Date: Sun, 15 Oct 2023 04:32:06 +0000 Subject: [PATCH 8/9] Fix MANUAL_BUILD_SCRIPTS error: Incompatible capsule version 0.10.1, this project requires a version that's compatible with 0.7.0 --- kicker | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/kicker b/kicker index 0d456ce..16cc095 100755 --- a/kicker +++ b/kicker @@ -521,7 +521,7 @@ function manual_build() { --workdir /app/web3 \ --volume $srcdir:/app \ --volume $WORKSPACE/packages/.rustup:/root/.rustup \ - --env CARGO_HOME=$WORKSPACE/packages/.cargo \ + --volume $WORKSPACE/packages/.cargo:/root/.cargo \ $DOCKER_MANUAL_BUILD_IMAGE cargo build --locked # Copy the built artifacts to `docker/manual-artifacts/gw-web3-indexer` @@ -557,10 +557,10 @@ function manual_build() { # Godwoken devnet, so `--features gw-config/no-builtin` is appended # to `cargo build` arguments. erun docker run --rm \ - --env CARGO_HOME=$WORKSPACE/packages/.cargo \ + --workdir /app \ --volume $srcdir:/app \ --volume $WORKSPACE/packages/.rustup:/root/.rustup \ - --workdir /app \ + --volume $WORKSPACE/packages/.cargo:/root/.cargo \ $DOCKER_MANUAL_BUILD_IMAGE \ cargo build --locked --features gw-config/no-builtin @@ -604,10 +604,11 @@ function manual_build() { prepare_repo godwoken "$SCRIPTS_GIT_URL" "$SCRIPTS_GIT_CHECKOUT" # Install capsule - # TODO: use cpasule from godwoken-manual-build image - if [ -z "$(command -v capsule)" ]; then - erun cargo install ckb-capsule - fi + CAPSULE_VERSION=0.7.0 + export CARGO_HOME=$WORKSPACE/packages/.cargo + (which capsule && test "$(capsule --version)" = "Capsule $CAPSULE_VERSION") \ + || erun cargo install ckb-capsule --version $CAPSULE_VERSION --force + $WORKSPACE/packages/.cargo/bin/capsule --version erun cd $srcdir/c \&\& erun make all-via-docker erun cd $srcdir \&\& capsule build --release --debug-output @@ -620,8 +621,7 @@ function manual_build() { # Copy the prebuild omni-lock to `docker/manual-artifacts/` erun $DOCKER_COMPOSE -f $WORKSPACE/docker/docker-compose.yml run \ - --rm \ - --no-deps \ + --rm --no-deps \ --volume=$dstdir:/godwoken-scripts \ --entrypoint "\"bash -c 'cp /scripts/godwoken-scripts/omni_lock /godwoken-scripts/omni_lock'\"" \ godwoken From 5456705e3b041362fe0aa413f4ac7de456cb45a2 Mon Sep 17 00:00:00 2001 From: Flouse <1297478+Flouse@users.noreply.github.com> Date: Sun, 15 Oct 2023 05:28:34 +0000 Subject: [PATCH 9/9] fixup! Fix MANUAL_BUILD_SCRIPTS --- kicker | 1 + 1 file changed, 1 insertion(+) diff --git a/kicker b/kicker index 16cc095..bafdf38 100755 --- a/kicker +++ b/kicker @@ -604,6 +604,7 @@ function manual_build() { prepare_repo godwoken "$SCRIPTS_GIT_URL" "$SCRIPTS_GIT_CHECKOUT" # Install capsule + # Fixme: error: Incompatible capsule version 0.10.1, this project requires a version that's compatible with 0.7.0 CAPSULE_VERSION=0.7.0 export CARGO_HOME=$WORKSPACE/packages/.cargo (which capsule && test "$(capsule --version)" = "Capsule $CAPSULE_VERSION") \