diff --git a/.github/workflows/godwoken-tests.yml b/.github/workflows/godwoken-tests.yml index 183eb67..f4c3072 100644 --- a/.github/workflows/godwoken-tests.yml +++ b/.github/workflows/godwoken-tests.yml @@ -9,8 +9,22 @@ on: jobs: godwoken-tests: + strategy: + fail-fast: false + matrix: + MANUAL_BUILD_GODWOKEN: ["true", "false"] + MANUAL_BUILD_WEB3: ["true", "false"] + MANUAL_BUILD_WEB3_INDEXER: ["true", "false"] + MANUAL_BUILD_POLYJUICE: ["true", "false"] + MANUAL_BUILD_SCRIPTS: ["true", "false"] + uses: godwokenrises/godwoken-tests/.github/workflows/reusable-integration-test-v1.yml@develop with: extra_github_env: | GODWOKEN_KICKER_REPO=${{ github.repository }} GODWOKEN_KICKER_REF=${{ github.ref }} + MANUAL_BUILD_GODWOKEN=${{ matrix.MANUAL_BUILD_GODWOKEN }} + MANUAL_BUILD_WEB3=${{ matrix.MANUAL_BUILD_WEB3 }} + MANUAL_BUILD_WEB3_INDEXER=${{ matrix.MANUAL_BUILD_WEB3_INDEXER }} + MANUAL_BUILD_POLYJUICE=${{ matrix.MANUAL_BUILD_POLYJUICE }} + MANUAL_BUILD_SCRIPTS=${{ matrix.MANUAL_BUILD_SCRIPTS }} diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index b7f50d9..407ce96 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -21,6 +21,7 @@ services: volumes: - ./redis/data:/data + # TODO: update CKB version ckb: image: nervos/ckb:v0.103.0 user: root @@ -31,6 +32,7 @@ services: - ./layer1/ckb:/var/lib/ckb command: [ "run", "-C", "/var/lib/ckb" ] + # TODO: update CKB version ckb-miner: init: true image: nervos/ckb:v0.103.0 diff --git a/kicker b/kicker index 140c978..bafdf38 100755 --- a/kicker +++ b/kicker @@ -2,6 +2,15 @@ set -o errexit +GITHUB_ACTIONS_CLEAN_MID_ARTIFACTS=false +# https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables +if [ "$CI" = "true" ] && [ -n "$GITHUB_RUN_ID" ]; then + echo "This script is running in GitHub Actions. GITHUB_RUN_ID: $GITHUB_RUN_ID" + + # Diskspace is limited in GitHub Actions, so we need to manually clean the middle artifacts + GITHUB_ACTIONS_CLEAN_MID_ARTIFACTS=true +fi + EXECUTABLE=$0 WORKSPACE="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" @@ -415,38 +424,53 @@ function withdraw_v0_to_v1() { # GODWOKEN_GIT_CHECKOUT=develop \ # ./kicker manual-build function manual_build() { - DOCKER_MANUAL_BUILD_IMAGE_NAME=${DOCKER_MANUAL_BUILD_IMAGE_NAME:-"retricsu/godwoken-manual-build"} - DOCKER_MANUAL_BUILD_IMAGE_TAG=${DOCKER_MANUAL_BUILD_IMAGE_TAG:-"node18"} - DOCKER_MANUAL_BUILD_IMAGE="${DOCKER_MANUAL_BUILD_IMAGE:-"$DOCKER_MANUAL_BUILD_IMAGE_NAME:$DOCKER_MANUAL_BUILD_IMAGE_TAG"}" - + WEB3_GIT_URL=${WEB3_GIT_URL:-"https://github.com/godwokenrises/godwoken"} + WEB3_GIT_CHECKOUT=${WEB3_GIT_CHECKOUT:-"develop"} echo "MANUAL_BUILD_WEB3 = \"$MANUAL_BUILD_WEB3\"" echo "WEB3_GIT_URL = \"$WEB3_GIT_URL\"" echo "WEB3_GIT_CHECKOUT = \"$WEB3_GIT_CHECKOUT\"" echo "MANUAL_BUILD_WEB3_INDEXER = \"$MANUAL_BUILD_WEB3_INDEXER\"" + + GODWOKEN_GIT_URL=${GODWOKEN_GIT_URL:-"https://github.com/godwokenrises/godwoken"} + GODWOKEN_GIT_CHECKOUT=${GODWOKEN_GIT_CHECKOUT:-"develop"} echo "MANUAL_BUILD_GODWOKEN = \"$MANUAL_BUILD_GODWOKEN\"" echo "GODWOKEN_GIT_URL = \"$GODWOKEN_GIT_URL\"" echo "GODWOKEN_GIT_CHECKOUT = \"$GODWOKEN_GIT_CHECKOUT\"" + + SCRIPTS_GIT_URL=${SCRIPTS_GIT_URL:-"https://github.com/godwokenrises/godwoken"} + SCRIPTS_GIT_CHECKOUT=${SCRIPTS_GIT_CHECKOUT:-"develop"} echo "MANUAL_BUILD_SCRIPTS = \"$MANUAL_BUILD_SCRIPTS\"" echo "SCRIPTS_GIT_URL = \"$SCRIPTS_GIT_URL\"" echo "SCRIPTS_GIT_CHECKOUT = \"$SCRIPTS_GIT_CHECKOUT\"" + echo "OMNI_LOCK_GIT_URL = \"$OMNI_LOCK_GIT_URL\"" echo "OMNI_LOCK_GIT_CHECKOUT = \"$OMNI_LOCK_GIT_CHECKOUT\"" + + POLYJUICE_GIT_URL=${POLYJUICE_GIT_URL:-"https://github.com/godwokenrises/godwoken"} + POLYJUICE_GIT_CHECKOUT=${POLYJUICE_GIT_CHECKOUT:-"develop"} echo "MANUAL_BUILD_POLYJUICE = \"$MANUAL_BUILD_POLYJUICE\"" echo "POLYJUICE_GIT_URL = \"$POLYJUICE_GIT_URL\"" echo "POLYJUICE_GIT_CHECKOUT = \"$POLYJUICE_GIT_CHECKOUT\"" + + DOCKER_MANUAL_BUILD_IMAGE_NAME=${DOCKER_MANUAL_BUILD_IMAGE_NAME:-"retricsu/godwoken-manual-build"} + DOCKER_MANUAL_BUILD_IMAGE_TAG=${DOCKER_MANUAL_BUILD_IMAGE_TAG:-"node18"} + DOCKER_MANUAL_BUILD_IMAGE="${DOCKER_MANUAL_BUILD_IMAGE:-"$DOCKER_MANUAL_BUILD_IMAGE_NAME:$DOCKER_MANUAL_BUILD_IMAGE_TAG"}" echo "DOCKER_MANUAL_BUILD_IMAGE = \"$DOCKER_MANUAL_BUILD_IMAGE\"" echo + info "Start building..." | tee /tmp/kicker.log + du -hd6 $WORKSPACE | egrep G | tee --append /tmp/kicker.log + if [ "$MANUAL_BUILD_WEB3" = "true" ]; then info "Start building godwoken-web3" # godwoken-web3 is managed in the Godwoken monorepo. # See https://github.com/godwokenrises/godwoken/tree/develop/web3 - srcdir=$WORKSPACE/packages/godwoken-web3/web3 + srcdir=$WORKSPACE/packages/godwoken/web3 dstdir=$WORKSPACE/docker/manual-artifacts/godwoken-web3 # Download repo - prepare_repo godwoken-web3 "$WEB3_GIT_URL" "$WEB3_GIT_CHECKOUT" + prepare_repo godwoken "$WEB3_GIT_URL" "$WEB3_GIT_CHECKOUT" # Yarn install via docker (assumes the "web3" docker-compose service installed yarn) @@ -471,6 +495,10 @@ function manual_build() { $srcdir/node_modules \ $srcdir/yarn.lock \ $dstdir + + if [[ "$GITHUB_ACTIONS_CLEAN_MID_ARTIFACTS" = "true" ]]; then + erun sudo rm -rf $srcdir/node_modules + fi else info "skip building godwoken-web3" fi @@ -478,34 +506,36 @@ function manual_build() { if [ "$MANUAL_BUILD_WEB3_INDEXER" = "true" ]; then info "Start building godwoken-web3-indexer" - srcdir=$WORKSPACE/packages/godwoken-web3 + srcdir=$WORKSPACE/packages/godwoken dstdir=$WORKSPACE/docker/manual-artifacts # Download repo - prepare_repo godwoken-web3 "$WEB3_GIT_URL" "$WEB3_GIT_CHECKOUT" + prepare_repo godwoken "$WEB3_GIT_URL" "$WEB3_GIT_CHECKOUT" # Cargo fetch Rust dependencies (in order to access network via # host network). The docker image must have installed cargo, molecule # and rustfmt. - erun "cd $srcdir/web3 && CARGO_HOME=$srcdir/web3/.cargo cargo fetch --locked && cd -" - erun docker run \ - --rm \ - --env CARGO_HOME=/app/.cargo \ + erun "cd $srcdir/web3 && CARGO_HOME=$WORKSPACE/packages/.cargo cargo fetch --locked && cd -" + + erun docker run --rm \ + --workdir /app/web3 \ --volume $srcdir:/app \ --volume $WORKSPACE/packages/.rustup:/root/.rustup \ - --workdir /app/web3 \ - retricsu/godwoken-manual-build:ckb2021 cargo build --locked --release + --volume $WORKSPACE/packages/.cargo:/root/.cargo \ + $DOCKER_MANUAL_BUILD_IMAGE cargo build --locked # Copy the built artifacts to `docker/manual-artifacts/gw-web3-indexer` # # More: ./docker/manual-web3-indexer.compose.yml erun mkdir -p $dstdir - erun cp $srcdir/web3/target/release/gw-web3-indexer $dstdir + erun cp $srcdir/web3/target/debug/gw-web3-indexer $dstdir # Remove Rust target to avoid `out of disk` issue in CI - sudo rm -rf $srcdir/web3/target/release + if [[ "$GITHUB_ACTIONS_CLEAN_MID_ARTIFACTS" = "true" ]]; then + sudo rm -rf $srcdir/web3/target + fi else - info "skip building godwoken-web3-indexer(gw-web3-indexer)" + info "skip building godwoken-web3-indexer (gw-web3-indexer)" fi if [ "$MANUAL_BUILD_GODWOKEN" = "true" ]; then @@ -519,29 +549,32 @@ function manual_build() { # Cargo fetch Rust dependencies (in order to access network via # host network). + erun "cd $srcdir && CARGO_HOME=$WORKSPACE/packages/.cargo cargo fetch --locked && cd -" + # Note: # 1. The docker image must have installed cargo and molecule # 2. The builtin consensus config and binaries are not required on a # Godwoken devnet, so `--features gw-config/no-builtin` is appended # to `cargo build` arguments. - erun docker run \ - --rm \ - --env CARGO_HOME=/app/.cargo \ + erun docker run --rm \ + --workdir /app \ --volume $srcdir:/app \ --volume $WORKSPACE/packages/.rustup:/root/.rustup \ - --workdir /app \ - retricsu/godwoken-manual-build:ckb2021 \ - cargo build --locked --release --features gw-config/no-builtin + --volume $WORKSPACE/packages/.cargo:/root/.cargo \ + $DOCKER_MANUAL_BUILD_IMAGE \ + cargo build --locked --features gw-config/no-builtin # Copy the built artifacts to `docker/manual-artifacts/` # # More: ./docker/manual-godwoken.compose.yml erun mkdir -p $dstdir - erun cp $srcdir/target/release/godwoken $dstdir - erun cp $srcdir/target/release/gw-tools $dstdir + erun cp $srcdir/target/debug/godwoken $dstdir + erun cp $srcdir/target/debug/gw-tools $dstdir # Remove Rust target to avoid `out of disk` issue in CI - sudo rm -rf $srcdir/target/release + if [[ "$GITHUB_ACTIONS_CLEAN_MID_ARTIFACTS" = "true" ]]; then + sudo rm -rf $srcdir/target + fi else info "skip building Godwoken" fi @@ -551,7 +584,7 @@ function manual_build() { dstdir=$WORKSPACE/docker/manual-artifacts/polyjuice/ # Download repo - prepare_repo godwoken-polyjuice "$POLYJUICE_GIT_URL" "$POLYJUICE_GIT_CHECKOUT" + prepare_repo godwoken "$POLYJUICE_GIT_URL" "$POLYJUICE_GIT_CHECKOUT" cd $srcdir erun make all-via-docker @@ -568,13 +601,15 @@ function manual_build() { dstdir=$WORKSPACE/docker/manual-artifacts/scripts/ # Download repo - prepare_repo godwoken-scripts "$SCRIPTS_GIT_URL" "$SCRIPTS_GIT_CHECKOUT" + prepare_repo godwoken "$SCRIPTS_GIT_URL" "$SCRIPTS_GIT_CHECKOUT" # Install capsule - # TODO: use cpasule from godwoken-manual-build image - if [ -z "$(command -v capsule)" ]; then - erun cargo install ckb-capsule - fi + # Fixme: error: Incompatible capsule version 0.10.1, this project requires a version that's compatible with 0.7.0 + CAPSULE_VERSION=0.7.0 + export CARGO_HOME=$WORKSPACE/packages/.cargo + (which capsule && test "$(capsule --version)" = "Capsule $CAPSULE_VERSION") \ + || erun cargo install ckb-capsule --version $CAPSULE_VERSION --force + $WORKSPACE/packages/.cargo/bin/capsule --version erun cd $srcdir/c \&\& erun make all-via-docker erun cd $srcdir \&\& capsule build --release --debug-output @@ -587,14 +622,23 @@ function manual_build() { # Copy the prebuild omni-lock to `docker/manual-artifacts/` erun $DOCKER_COMPOSE -f $WORKSPACE/docker/docker-compose.yml run \ - --rm \ - --no-deps \ + --rm --no-deps \ --volume=$dstdir:/godwoken-scripts \ --entrypoint "\"bash -c 'cp /scripts/godwoken-scripts/omni_lock /godwoken-scripts/omni_lock'\"" \ godwoken else info "skip building Scripts" fi + + if [[ "$GITHUB_ACTIONS_CLEAN_MID_ARTIFACTS" = "true" ]]; then + docker images --format '{{.Repository}}:{{.Tag}}' \ + | grep '$DOCKER_MANUAL_BUILD_IMAGE' | xargs docker rmi \ + && echo "Image removed." \ + || echo "Image does not exist." + fi + + info "End building." | tee --append /tmp/kicker.log + du -hd6 $WORKSPACE | egrep G | tee --append /tmp/kicker.log } function prepare_repo() {