CI Main #11225
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI Main | |
| on: | |
| merge_group: | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - master | |
| - 'dev-gh-*' | |
| pull_request: | |
| # Used as reusable workflow within release-testing workflow | |
| workflow_call: | |
| # runs for the same workflow are cancelled on PRs but not on master | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }} | |
| cancel-in-progress: true | |
| permissions: read-all | |
| env: | |
| CI_COMMIT_SHA: ${{ github.sha }} | |
| CI_JOB_NAME: ${{ github.job }} | |
| CI_JOB_ID: ${{ github.job }} # github does not expose this variable https://github.com/orgs/community/discussions/8945 | |
| CI_JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| CI_PIPELINE_SOURCE: ${{ github.event_name }} | |
| CI_PROJECT_DIR: ${{ github.workspace }} | |
| CI_PULL_REQUEST_TARGET_BRANCH_NAME: ${{ github.event.pull_request.base.ref }} | |
| CI_PULL_REQUEST_TARGET_BRANCH_SHA: ${{ github.event.pull_request.base.sha }} | |
| BRANCH_NAME: ${{ github.head_ref || github.ref_name }} | |
| ROOT_PIPELINE_ID: ${{ github.run_id }} | |
| RUSTFLAGS: "--remap-path-prefix=${CI_PROJECT_DIR}=/ic" | |
| CI_PULL_REQUEST_TITLE: ${{ github.event.pull_request.title }} | |
| BUILDEVENT_DATASET: "github-ci-dfinity" | |
| jobs: | |
| bazel-test-all: | |
| name: Bazel Test All | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:fb66b22cf6cf52178f554a417837791867a2c080f9d3e9d9acaa64c811e8508a | |
| options: >- | |
| -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info | |
| timeout-minutes: 90 | |
| runs-on: | |
| group: zh1 | |
| labels: dind-large | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }} | |
| - name: Before script | |
| id: before-script | |
| shell: bash | |
| run: | | |
| [ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME" | |
| - name: Login to Dockerhub | |
| shell: bash | |
| run: ./ci/scripts/docker-login.sh | |
| env: | |
| DOCKER_HUB_USER: ${{ vars.DOCKER_HUB_USER }} | |
| DOCKER_HUB_PASSWORD_RO: ${{ secrets.DOCKER_HUB_PASSWORD_RO }} | |
| - name: Set BAZEL_EXTRA_ARGS_RULES | |
| shell: bash | |
| run: | | |
| set -xeuo pipefail | |
| if [[ "${{ github.event_name }}" == 'merge_group' ]]; then | |
| echo "BAZEL_EXTRA_ARGS_RULES=--test_timeout_filters=short,moderate --flaky_test_attempts=3" >> $GITHUB_ENV | |
| fi | |
| if [[ $BRANCH_NAME =~ ^hotfix-.* ]]; then | |
| echo "BAZEL_EXTRA_ARGS_RULES=--test_timeout_filters=short,moderate" >> $GITHUB_ENV | |
| fi | |
| - name: Run Bazel Test All | |
| id: bazel-test-all | |
| uses: ./.github/actions/bazel-test-all/ | |
| env: | |
| AWS_SHARED_CREDENTIALS_CONTENT: ${{ secrets.AWS_SHARED_CREDENTIALS_FILE }} | |
| RUN_ON_DIFF_ONLY: "true" | |
| with: | |
| BAZEL_COMMAND: "test" | |
| BAZEL_TARGETS: "//..." | |
| BAZEL_CI_CONFIG: "--config=ci --repository_cache=/cache/bazel" | |
| # check if PR title contains release and set timeout filters accordingly | |
| BAZEL_EXTRA_ARGS_RULES: ${{ env.BAZEL_EXTRA_ARGS_RULES || '' }} | |
| HONEYCOMB_API_TOKEN: ${{ secrets.HONEYCOMB_API_TOKEN }} | |
| - name: Upload bazel-targets | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: bazel-targets | |
| retention-days: 14 | |
| if-no-files-found: error | |
| path: | | |
| bazel-targets | |
| - name: Upload bazel-bep | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ github.job }}-bep | |
| retention-days: 14 | |
| if-no-files-found: ignore | |
| compression-level: 9 | |
| path: | | |
| bazel-bep.pb | |
| profile.json | |
| bazel-build-all-config-check: | |
| runs-on: | |
| labels: dind-large | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:fb66b22cf6cf52178f554a417837791867a2c080f9d3e9d9acaa64c811e8508a | |
| options: >- | |
| -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info | |
| timeout-minutes: 90 | |
| name: Bazel Build All Config Check | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }} | |
| - name: Before script | |
| id: before-script | |
| shell: bash | |
| run: | | |
| [ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME" | |
| - name: Login to Dockerhub | |
| shell: bash | |
| run: ./ci/scripts/docker-login.sh | |
| env: | |
| DOCKER_HUB_USER: ${{ vars.DOCKER_HUB_USER }} | |
| DOCKER_HUB_PASSWORD_RO: ${{ secrets.DOCKER_HUB_PASSWORD_RO }} | |
| - name: Run bazel build --config=check //rs/... | |
| id: bazel-build-config-check | |
| uses: ./.github/actions/bazel-test-all/ | |
| with: | |
| BAZEL_COMMAND: "build" | |
| BAZEL_TARGETS: "//rs/..." | |
| BAZEL_CI_CONFIG: "--config=check --config=ci --keep_going" | |
| - name: Upload bazel-bep | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ github.job }}-bep | |
| retention-days: 14 | |
| if-no-files-found: ignore | |
| compression-level: 9 | |
| path: | | |
| bazel-bep.pb | |
| profile.json | |
| bazel-test-macos-intel: | |
| name: Bazel Test macOS Intel | |
| timeout-minutes: 120 | |
| runs-on: | |
| labels: macOS | |
| if: ${{ github.ref_protected }} # Bazel test darwin are still required for building artifacts for the releases, therefore run on protected branches | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }} | |
| - name: Set PATH | |
| run: | | |
| echo "/usr/local/bin" >> $GITHUB_PATH | |
| echo "$HOME/.cargo/bin:" >> $GITHUB_PATH | |
| - name: Login to Dockerhub | |
| shell: bash | |
| run: ./ci/scripts/docker-login.sh | |
| env: | |
| DOCKER_HUB_USER: ${{ vars.DOCKER_HUB_USER }} | |
| DOCKER_HUB_PASSWORD_RO: ${{ secrets.DOCKER_HUB_PASSWORD_RO }} | |
| - name: Run Bazel Test Darwin x86-64 | |
| id: bazel-test-darwin-x86-64 | |
| uses: ./.github/actions/bazel-test-all/ | |
| env: | |
| AWS_SHARED_CREDENTIALS_CONTENT: ${{ secrets.AWS_SHARED_CREDENTIALS_FILE }} | |
| with: | |
| BAZEL_CI_CONFIG: "--config=ci --config macos_ci" | |
| BAZEL_COMMAND: test | |
| BAZEL_EXTRA_ARGS: '--test_tag_filters=test_macos' | |
| BAZEL_STARTUP_ARGS: "--output_base /var/tmp/bazel-output/${ROOT_PIPELINE_ID}" | |
| BAZEL_TARGETS: "//rs/... //publish/binaries/..." | |
| HONEYCOMB_API_TOKEN: ${{ secrets.HONEYCOMB_API_TOKEN }} | |
| - name: Upload bazel-bep | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ github.job }}-bep | |
| retention-days: 14 | |
| if-no-files-found: ignore | |
| compression-level: 9 | |
| path: | | |
| bazel-bep.pb | |
| profile.json | |
| - name: Purge Bazel Output | |
| if: always() | |
| shell: bash | |
| run: | | |
| sudo rm -rf /private/var/tmp/bazel-output | |
| bazel-build-fuzzers: | |
| name: Bazel Build Fuzzers | |
| runs-on: | |
| labels: dind-large | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:fb66b22cf6cf52178f554a417837791867a2c080f9d3e9d9acaa64c811e8508a | |
| options: >- | |
| -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info | |
| timeout-minutes: 90 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }} | |
| - name: Before script | |
| id: before-script | |
| shell: bash | |
| run: | | |
| [ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME" | |
| - name: Run Bazel Build Fuzzers | |
| id: bazel-build-fuzzers | |
| uses: ./.github/actions/bazel-test-all/ | |
| with: | |
| BAZEL_COMMAND: "build" | |
| BAZEL_TARGETS: "//rs/..." | |
| BAZEL_EXTRA_ARGS: "--keep_going --config=fuzzing --build_tag_filters=libfuzzer" | |
| - name: Upload bazel-bep | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: ${{ github.job }}-bep | |
| retention-days: 14 | |
| if-no-files-found: ignore | |
| compression-level: 9 | |
| path: | | |
| bazel-bep.pb | |
| profile.json | |
| bazel-build-fuzzers-afl: | |
| name: Bazel Build Fuzzers AFL | |
| runs-on: | |
| labels: dind-large | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:fb66b22cf6cf52178f554a417837791867a2c080f9d3e9d9acaa64c811e8508a | |
| options: >- | |
| -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info | |
| timeout-minutes: 90 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }} | |
| - name: Before script | |
| id: before-script | |
| shell: bash | |
| run: | | |
| [ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME" | |
| - name: Run Bazel Build Fuzzers AFL | |
| id: bazel-build-fuzzers-afl | |
| uses: ./.github/actions/bazel-test-all/ | |
| with: | |
| BAZEL_COMMAND: "build" | |
| BAZEL_TARGETS: "//rs/..." | |
| BAZEL_EXTRA_ARGS: "--keep_going --config=afl" | |
| - name: Upload bazel-bep | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ github.job }}-bep | |
| retention-days: 14 | |
| if-no-files-found: ignore | |
| compression-level: 9 | |
| path: | | |
| bazel-bep.pb | |
| profile.json | |
| python-ci-tests: | |
| name: Python CI Tests | |
| runs-on: | |
| labels: dind-small | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:fb66b22cf6cf52178f554a417837791867a2c080f9d3e9d9acaa64c811e8508a | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Run Python CI Tests | |
| id: python-ci-tests | |
| shell: bash | |
| run: | | |
| set -xeuo pipefail | |
| export PYTHONPATH=$PWD/ci/src:$PWD/ci/src/dependencies | |
| pip3 install --ignore-installed -r requirements.txt | |
| cd ci/src | |
| pytest -m "not fails_on_merge_train" -v -o junit_family=xunit1 \ | |
| --junitxml=../../test_report.xml --cov=. --cov-report=term \ | |
| --cov-report=term-missing --cov-report=html --cov-branch | |
| env: | |
| CI_COMMIT_REF_PROTECTED: ${{ github.ref_protected }} | |
| CI_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} | |
| build-ic: | |
| name: Build IC | |
| runs-on: | |
| labels: dind-large | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:fb66b22cf6cf52178f554a417837791867a2c080f9d3e9d9acaa64c811e8508a | |
| options: >- | |
| -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info | |
| timeout-minutes: 90 | |
| if: ${{ github.event_name != 'merge_group' }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }} | |
| - name: Before script | |
| id: before-script | |
| shell: bash | |
| run: | | |
| [ -n "${NODE_NAME:-}" ] && echo "Node: $NODE_NAME" | |
| - name: Login to Dockerhub | |
| shell: bash | |
| run: ./ci/scripts/docker-login.sh | |
| env: | |
| DOCKER_HUB_USER: ${{ vars.DOCKER_HUB_USER }} | |
| DOCKER_HUB_PASSWORD_RO: ${{ secrets.DOCKER_HUB_PASSWORD_RO }} | |
| - name: Run Build IC | |
| id: build-ic | |
| shell: bash | |
| run: | | |
| set -eExuo pipefail | |
| REPO_NAME="${GITHUB_REPOSITORY##*/}" | |
| rm -rf "/cache/job/${CI_JOB_ID}/${ROOT_PIPELINE_ID}" | |
| mkdir -p "/cache/job/${CI_JOB_ID}/${ROOT_PIPELINE_ID}/artifacts" | |
| ln -s "/cache/job/${CI_JOB_ID}/${ROOT_PIPELINE_ID}/artifacts" /__w/$REPO_NAME/$REPO_NAME/artifacts | |
| buildevents cmd "$ROOT_PIPELINE_ID" "$CI_JOB_ID" build-command -- \ | |
| "$CI_PROJECT_DIR"/ci/scripts/run-build-ic.sh | |
| rm -rf "/cache/job/${CI_JOB_ID}/${ROOT_PIPELINE_ID}" | |
| env: | |
| BAZEL_COMMAND: "build" | |
| RUN_ON_DIFF_ONLY: "true" | |
| - name: Upload build-ic.tar | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: build-ic | |
| retention-days: 1 | |
| if-no-files-found: error | |
| path: | | |
| build-ic.tar | |
| build-determinism: | |
| name: Build Determinism | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| needs: [build-ic, bazel-test-all] | |
| strategy: | |
| matrix: | |
| include: | |
| - TARGET: "//publish/binaries:upload" | |
| PATH0: "release" | |
| PATH1: "build-ic/release" | |
| SETUPOS_FLAG: "false" | |
| - TARGET: "//publish/canisters:upload" | |
| PATH0: "canisters" | |
| PATH1: "build-ic/canisters" | |
| SETUPOS_FLAG: "false" | |
| - TARGET: "//ic-os/guestos/envs/prod:upload_disk-img" | |
| PATH0: "guest-os/update-img" | |
| PATH1: "build-ic/icos/guestos" | |
| SETUPOS_FLAG: "false" | |
| - TARGET: "//ic-os/hostos/envs/prod:upload_update-img" | |
| PATH0: "host-os/update-img" | |
| PATH1: "build-ic/icos/hostos" | |
| SETUPOS_FLAG: "false" | |
| - TARGET: "//ic-os/setupos/envs/prod:upload_disk-img" | |
| PATH0: "setup-os/disk-img" | |
| PATH1: "build-ic/icos/setupos" | |
| SETUPOS_FLAG: "true" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }} | |
| - name: Download bazel-targets [bazel-test-all] | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: bazel-targets | |
| - name: Download build-ic.tar [build-ic] | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: build-ic | |
| - name: Build Determinism Test | |
| id: build-determinism | |
| shell: bash | |
| run: | | |
| set -eExuo pipefail | |
| sudo apt update && sudo apt install -y curl | |
| "$CI_PROJECT_DIR"/ci/scripts/build-determinism.sh | |
| env: | |
| TARGET: ${{ matrix.TARGET }} | |
| PATH0: ${{ matrix.PATH0 }} | |
| PATH1: ${{ matrix.PATH1 }} | |
| SETUPOS_FLAG: ${{ matrix.SETUPOS_FLAG }} | |
| cargo-clippy-linux: | |
| name: Cargo Clippy Linux | |
| runs-on: | |
| labels: dind-large | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:fb66b22cf6cf52178f554a417837791867a2c080f9d3e9d9acaa64c811e8508a | |
| options: >- | |
| -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info | |
| timeout-minutes: 90 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }} | |
| - name: Filter Rust Files [*.{rs,toml,lock}] | |
| uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 | |
| id: filter | |
| if: | | |
| github.event_name == 'pull_request' || | |
| github.event_name == 'merge_group' | |
| with: | |
| filters: | | |
| cargo: | |
| - "**/*.rs" | |
| - "**/*.toml" | |
| - "**/*.lock" | |
| - name: Run Cargo Clippy Linux | |
| id: cargo-clippy-linux | |
| if: | | |
| steps.filter.outputs.cargo == 'true' || | |
| github.event_name == 'schedule' || | |
| github.event_name == 'workflow_dispatch' | |
| shell: bash | |
| run: | | |
| set -eExuo pipefail | |
| buildevents cmd "$ROOT_PIPELINE_ID" "$CI_JOB_ID" build-command -- \ | |
| "$CI_PROJECT_DIR"/ci/scripts/rust-lint.sh | |
| cargo-build-release-linux: | |
| name: Cargo Build Release Linux | |
| runs-on: | |
| labels: dind-large | |
| container: | |
| image: ghcr.io/dfinity/ic-build@sha256:fb66b22cf6cf52178f554a417837791867a2c080f9d3e9d9acaa64c811e8508a | |
| options: >- | |
| -e NODE_NAME --privileged --cgroupns host -v /cache:/cache -v /var/sysimage:/var/sysimage -v /var/tmp:/var/tmp -v /ceph-s3-info:/ceph-s3-info | |
| timeout-minutes: 90 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: ${{ github.event_name == 'pull_request' && 256 || 0 }} | |
| - name: Filter Rust Files [*.{rs,toml,lock}] | |
| uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 | |
| id: filter | |
| if: | | |
| github.event_name == 'pull_request' || | |
| github.event_name == 'merge_group' | |
| with: | |
| filters: | | |
| cargo: | |
| - "**/*.rs" | |
| - "**/*.toml" | |
| - "**/*.lock" | |
| - name: Run Cargo Build Release Linux | |
| id: cargo-build-release-linux | |
| if: | | |
| steps.filter.outputs.cargo == 'true' || | |
| github.event_name == 'schedule' || | |
| github.event_name == 'workflow_dispatch' | |
| shell: bash | |
| run: | | |
| set -eExuo pipefail | |
| buildevents cmd "$ROOT_PIPELINE_ID" "$CI_JOB_ID" build-command -- \ | |
| cargo build --release |