From a218bd991c5ae5a3f64accf64c8315f3f0da6e4a Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Thu, 19 Sep 2024 14:03:26 +0400 Subject: [PATCH 01/10] updated vars --- .github/workflows/reusable-preflight.yml | 37 ++++++++++++++++++------ 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/.github/workflows/reusable-preflight.yml b/.github/workflows/reusable-preflight.yml index 71823a97ff2e..529d3b294a5e 100644 --- a/.github/workflows/reusable-preflight.yml +++ b/.github/workflows/reusable-preflight.yml @@ -35,12 +35,23 @@ on: OLDLINUXRUNNER: value: ${{ jobs.preflight.outputs.OLDLINUXRUNNER }} - SOURCE_REF_NAME: - value: ${{ jobs.preflight.outputs.SOURCE_REF_NAME }} - description: "Name of the current branch for `push` or source branch for `pull_request`" + SOURCE_REF_SLUG: + value: ${{ jobs.preflight.outputs.SOURCE_REF_SLUG }} + description: "Name of the current branch for `push` or source branch for `pull_request` with `/` replaced by `_`. Does not exists in merge_group" + REF_SLUG: + value: ${{ jobs.preflight.outputs.REF_SLUG }} + description: | + Name of the current ref (depending on the event) with `/` replaced by `_`, e.g: + push - master + pull_request - 49_merge + merge_group - gh-readonly-queue_master_pr-49-38d43798a986430231c828b2c762997f818ac012 + COMMIT_SHA: value: ${{ jobs.preflight.outputs.COMMIT_SHA }} - description: "Sha of the current commit for `push` or head of the source branch for `pull_request`" + description: "Sha of the current revision" + COMMIT_SHA_SHORT: + value: ${{ jobs.preflight.outputs.COMMIT_SHA_SHORT }} + description: "Sha of the current revision, 8-symbols long" jobs: preflight: @@ -53,9 +64,11 @@ jobs: RUNNER: ${{ steps.set_runner.outputs.RUNNER }} OLDLINUXRUNNER: ${{ steps.set_runner.outputs.OLDLINUXRUNNER }} - SOURCE_REF_NAME: ${{ steps.set_vars.outputs.SOURCE_REF_NAME }} - COMMIT_SHA: ${{ steps.set_vars.outputs.COMMIT_SHA }} + SOURCE_REF_SLUG: ${{ steps.set_vars.outputs.SOURCE_REF_SLUG }} + REF_SLUG: ${{ steps.set_vars.outputs.REF_SLUG }} + COMMIT_SHA: ${{ steps.set_vars.outputs.COMMIT_SHA }} + COMMIT_SHA_SHORT: ${{ steps.set_vars.outputs.COMMIT_SHA_SHORT }} steps: @@ -116,9 +129,15 @@ jobs: - id: set_vars shell: bash run: | - export BRANCH_NAME=${{ github.head_ref || github.ref_name }} - echo "SOURCE_REF_NAME=${BRANCH_NAME//\//-}" >> $GITHUB_OUTPUT - echo "COMMIT_SHA=${{ github.event.pull_request.head.sha || github.sha }}" >> $GITHUB_OUTPUT + export SOURCE_REF_NAME=${{ github.head_ref || github.ref_name }} + echo "SOURCE_REF_SLUG=${SOURCE_REF_NAME//\//_}" >> $GITHUB_OUTPUT + # + export COMMIT_SHA=${{ github.sha }} + echo "COMMIT_SHA=$COMMIT_SHA" >> $GITHUB_OUTPUT + echo "COMMIT_SHA_SHORT=${COMMIT_SHACOMMIT_SHA:0:8}" >> $GITHUB_OUTPUT + # + export REF_NAME=${{ github.ref_name }} + echo "REF_SLUG=${REF_NAME//\//_}" >> $GITHUB_OUTPUT - name: log From 7ee8d19e6178bbabd55382d3444d62cc1bf3b019 Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Thu, 19 Sep 2024 16:25:20 +0400 Subject: [PATCH 02/10] new runners --- .github/workflows/reusable-preflight.yml | 45 +++++++++++++++++----- .github/workflows/subsystem-benchmarks.yml | 2 +- .github/workflows/tests-linux-stable.yml | 2 +- .github/workflows/tests-misc.yml | 2 +- 4 files changed, 39 insertions(+), 12 deletions(-) diff --git a/.github/workflows/reusable-preflight.yml b/.github/workflows/reusable-preflight.yml index 529d3b294a5e..08e44cba6c0f 100644 --- a/.github/workflows/reusable-preflight.yml +++ b/.github/workflows/reusable-preflight.yml @@ -26,15 +26,32 @@ on: IMAGE: value: ${{ jobs.preflight.outputs.IMAGE }} description: "CI image" + + # Runners + # https://github.com/paritytech/ci_cd/wiki/GitHub#paritytech-self-hosted-runners RUNNER: value: ${{ jobs.preflight.outputs.RUNNER }} description: | - Runner name. + Main runner for resource-intensive tasks By default we use spot machines that can be terminated at any time. Merge queues use persistent runners to avoid kicking off from queue when the runner is terminated. - OLDLINUXRUNNER: - value: ${{ jobs.preflight.outputs.OLDLINUXRUNNER }} - + RUNNER_OLDLINUX: + value: ${{ jobs.preflight.outputs.RUNNER_OLDLINUX }} + description: | + parity-oldlinux + By default we use spot machines that can be terminated at any time. + Merge queues use persistent runners to avoid kicking off from queue when the runner is terminated. + RUNNER_DEFAULT: + value: ${{ jobs.preflight.outputs.RUNNER_DEFAULT }} + description: "Relatively lightweight runner. When `ubuntu-latest` is not enough" + RUNNER_WEIGHTS: + value: ${{ jobs.preflight.outputs.RUNNER_WEIGHTS }} + RUNNER_BENCHMARK: + value: ${{ jobs.preflight.outputs.RUNNER_BENCHMARK }} + RUNNER_MACOS: + value: ${{ jobs.preflight.outputs.RUNNER_MACOS }} + + # Vars SOURCE_REF_SLUG: value: ${{ jobs.preflight.outputs.SOURCE_REF_SLUG }} description: "Name of the current branch for `push` or source branch for `pull_request` with `/` replaced by `_`. Does not exists in merge_group" @@ -61,8 +78,13 @@ jobs: changes_currentWorkflow: ${{ steps.set_changes.outputs.currentWorkflow_any_changed }} IMAGE: ${{ steps.set_image.outputs.IMAGE }} + RUNNER: ${{ steps.set_runner.outputs.RUNNER }} - OLDLINUXRUNNER: ${{ steps.set_runner.outputs.OLDLINUXRUNNER }} + RUNNER_OLDLINUX: ${{ steps.set_runner.outputs.RUNNER_OLDLINUX }} + RUNNER_DEFAULT: ${{ steps.set_runner.outputs.RUNNER_DEFAULT }} + RUNNER_WEIGHTS: ${{ steps.set_runner.outputs.RUNNER_WEIGHTS }} + RUNNER_BENCHMARK: ${{ steps.set_runner.outputs.RUNNER_BENCHMARK }} + RUNNER_MACOS: ${{ steps.set_runner.outputs.RUNNER_MACOS }} SOURCE_REF_SLUG: ${{ steps.set_vars.outputs.SOURCE_REF_SLUG }} REF_SLUG: ${{ steps.set_vars.outputs.REF_SLUG }} @@ -114,13 +136,18 @@ jobs: - id: set_runner shell: bash run: | + echo "RUNNER_DEFAULT=parity-default" >> $GITHUB_OUTPUT + echo "RUNNER_WEIGHTS=parity-weights" >> $GITHUB_OUTPUT + echo "RUNNER_BENCHMARK=parity-benchmark" >> $GITHUB_OUTPUT + echo "RUNNER_MACOS=parity-macos" >> $GITHUB_OUTPUT + # # Run merge queues on persistent runners if [[ $GITHUB_REF_NAME == *"gh-readonly-queue"* ]]; then - echo "RUNNER=arc-runners-polkadot-sdk-beefy-persistent" >> $GITHUB_OUTPUT - echo "OLDLINUXRUNNER=oldlinux-persistent" >> $GITHUB_OUTPUT + echo "RUNNER=parity-large-persistent" >> $GITHUB_OUTPUT + echo "RUNNER_OLDLINUX=parity-oldlinux-persistent" >> $GITHUB_OUTPUT else - echo "RUNNER=arc-runners-polkadot-sdk-beefy" >> $GITHUB_OUTPUT - echo "OLDLINUXRUNNER=oldlinux" >> $GITHUB_OUTPUT + echo "RUNNER=parity-large" >> $GITHUB_OUTPUT + echo "RUNNER_OLDLINUXRUNNER=parity-oldlinux" >> $GITHUB_OUTPUT fi # diff --git a/.github/workflows/subsystem-benchmarks.yml b/.github/workflows/subsystem-benchmarks.yml index 210714d847ff..3c0eaf30e45f 100644 --- a/.github/workflows/subsystem-benchmarks.yml +++ b/.github/workflows/subsystem-benchmarks.yml @@ -22,7 +22,7 @@ jobs: build: timeout-minutes: 80 needs: [preflight] - runs-on: ${{ needs.preflight.outputs.RUNNER }} + runs-on: ${{ needs.preflight.outputs.RUNNER_BENCHMARK }} container: image: ${{ needs.preflight.outputs.IMAGE }} strategy: diff --git a/.github/workflows/tests-linux-stable.yml b/.github/workflows/tests-linux-stable.yml index 6cf71422511c..994f1c9b78ea 100644 --- a/.github/workflows/tests-linux-stable.yml +++ b/.github/workflows/tests-linux-stable.yml @@ -68,7 +68,7 @@ jobs: runners: [ "${{ needs.preflight.outputs.RUNNER }}", - "${{ needs.preflight.outputs.OLDLINUXRUNNER }}", + "${{ needs.preflight.outputs.RUNNER_OLDLINUX }}", ] container: image: ${{ needs.preflight.outputs.IMAGE }} diff --git a/.github/workflows/tests-misc.yml b/.github/workflows/tests-misc.yml index dc6128a3f6e7..70785b849751 100644 --- a/.github/workflows/tests-misc.yml +++ b/.github/workflows/tests-misc.yml @@ -321,7 +321,7 @@ jobs: cargo-check-all-crate-macos: timeout-minutes: 30 needs: [preflight] - runs-on: parity-macos + runs-on: ${{ needs.preflight.outputs.RUNNER_MACOS }} env: SKIP_WASM_BUILD: 1 steps: From 444556550542a3309f6ce0a3e244a985b79448cd Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Thu, 19 Sep 2024 16:40:51 +0400 Subject: [PATCH 03/10] log update --- .github/workflows/reusable-preflight.yml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/reusable-preflight.yml b/.github/workflows/reusable-preflight.yml index 08e44cba6c0f..237d765d07ff 100644 --- a/.github/workflows/reusable-preflight.yml +++ b/.github/workflows/reusable-preflight.yml @@ -58,7 +58,7 @@ on: REF_SLUG: value: ${{ jobs.preflight.outputs.REF_SLUG }} description: | - Name of the current ref (depending on the event) with `/` replaced by `_`, e.g: + Name of the current revision (depending on the event) with `/` replaced by `_`, e.g: push - master pull_request - 49_merge merge_group - gh-readonly-queue_master_pr-49-38d43798a986430231c828b2c762997f818ac012 @@ -79,6 +79,8 @@ jobs: IMAGE: ${{ steps.set_image.outputs.IMAGE }} + # Runners + # https://github.com/paritytech/ci_cd/wiki/GitHub#paritytech-self-hosted-runners RUNNER: ${{ steps.set_runner.outputs.RUNNER }} RUNNER_OLDLINUX: ${{ steps.set_runner.outputs.RUNNER_OLDLINUX }} RUNNER_DEFAULT: ${{ steps.set_runner.outputs.RUNNER_DEFAULT }} @@ -99,7 +101,8 @@ jobs: # # Set changes # - - id: current_file + - name: Current file + id: current_file shell: bash run: | echo "currentWorkflowFile=$(echo ${{ github.workflow_ref }} | sed -nE "s/.*(\.github\/workflows\/[a-zA-Z0-9_-]*\.y[a]?ml)@refs.*/\1/p")" >> $GITHUB_OUTPUT @@ -133,7 +136,8 @@ jobs: # By default we use spot machines that can be terminated at any time. # Merge queues use persistent runners to avoid kicking off from queue when the runner is terminated. # - - id: set_runner + - name: Set runner + id: set_runner shell: bash run: | echo "RUNNER_DEFAULT=parity-default" >> $GITHUB_OUTPUT @@ -153,7 +157,8 @@ jobs: # # Set vars # - - id: set_vars + - name: Set vars + id: set_vars shell: bash run: | export SOURCE_REF_NAME=${{ github.head_ref || github.ref_name }} @@ -175,3 +180,8 @@ jobs: echo "github.ref: ${{ github.ref }}" echo "github.ref_name: ${{ github.ref_name }}" echo "github.sha: ${{ github.sha }}" + # + echo "COMMIT_SHAgithub.sha: ${{ steps.set_vars.outputs.COMMIT_SHA }}" + echo "COMMIT_SHA_SHORT: ${{ steps.set_vars.outputs.COMMIT_SHA_SHORT }}" + echo "SOURCE_REF_SLUG: ${{ steps.set_vars.outputs.SOURCE_REF_SLUG }}" + echo "REF_SLUG: ${{ steps.set_vars.outputs.REF_SLUG }}" From c09857c84aaecdd6a49e6fb5e8a93c2bacab0b07 Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Thu, 19 Sep 2024 17:46:24 +0400 Subject: [PATCH 04/10] typo --- .github/workflows/reusable-preflight.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable-preflight.yml b/.github/workflows/reusable-preflight.yml index 237d765d07ff..c1be149b0a57 100644 --- a/.github/workflows/reusable-preflight.yml +++ b/.github/workflows/reusable-preflight.yml @@ -151,7 +151,7 @@ jobs: echo "RUNNER_OLDLINUX=parity-oldlinux-persistent" >> $GITHUB_OUTPUT else echo "RUNNER=parity-large" >> $GITHUB_OUTPUT - echo "RUNNER_OLDLINUXRUNNER=parity-oldlinux" >> $GITHUB_OUTPUT + echo "RUNNER_OLDLINUX=parity-oldlinux" >> $GITHUB_OUTPUT fi # From a0b25d399ca6c0f1d7d53f56f93764fcb0ffd402 Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Mon, 23 Sep 2024 16:37:46 +0400 Subject: [PATCH 05/10] log versions and vars --- .github/workflows/reusable-preflight.yml | 43 ++++++++++++++++++++---- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/.github/workflows/reusable-preflight.yml b/.github/workflows/reusable-preflight.yml index c1be149b0a57..38bc830251db 100644 --- a/.github/workflows/reusable-preflight.yml +++ b/.github/workflows/reusable-preflight.yml @@ -71,6 +71,10 @@ on: description: "Sha of the current revision, 8-symbols long" jobs: + + # + # + # preflight: runs-on: ubuntu-latest outputs: @@ -177,11 +181,38 @@ jobs: run: | echo "workflow file: ${{ steps.current_file.outputs.currentWorkflowFile }}" echo "Modified: ${{ steps.set_changes.outputs.modified_keys }}" + + # + # + # + ci-versions: + needs: [preflight] + runs-on: ubuntu-latest + container: + image: ${{ needs.preflight.outputs.IMAGE }} + steps: + + - name: Info rust + run: | + rustup show + cargo --version + yarn --version + echo $( substrate-contracts-node --version | awk 'NF' ) + estuary --version + cargo-contract --version + + - name: Info forklift + run: forklift version + + - name: Info vars + run: | + echo "COMMIT_SHAgithub.sha: ${{ needs.preflight.outputs.COMMIT_SHA }}" + echo "COMMIT_SHA_SHORT: ${{ needs.preflight.outputs.COMMIT_SHA_SHORT }}" + echo "SOURCE_REF_SLUG: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }}" + echo "REF_SLUG: ${{ needs.preflight.outputs.REF_SLUG }}" + echo "RUNNER: ${{ needs.preflight.outputs.RUNNER }}" + echo "IMAGE: ${{ needs.preflight.outputs.IMAGE }}" + # echo "github.ref: ${{ github.ref }}" echo "github.ref_name: ${{ github.ref_name }}" - echo "github.sha: ${{ github.sha }}" - # - echo "COMMIT_SHAgithub.sha: ${{ steps.set_vars.outputs.COMMIT_SHA }}" - echo "COMMIT_SHA_SHORT: ${{ steps.set_vars.outputs.COMMIT_SHA_SHORT }}" - echo "SOURCE_REF_SLUG: ${{ steps.set_vars.outputs.SOURCE_REF_SLUG }}" - echo "REF_SLUG: ${{ steps.set_vars.outputs.REF_SLUG }}" + echo "github.sha: ${{ github.sha }}" \ No newline at end of file From b0c3e6c9950a1382a89f29e3d439c5b61d82c9fd Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Mon, 23 Sep 2024 16:41:20 +0400 Subject: [PATCH 06/10] COMMIT_SHA_SHORT fix --- .github/workflows/reusable-preflight.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable-preflight.yml b/.github/workflows/reusable-preflight.yml index 38bc830251db..fae7daa9a31b 100644 --- a/.github/workflows/reusable-preflight.yml +++ b/.github/workflows/reusable-preflight.yml @@ -170,7 +170,7 @@ jobs: # export COMMIT_SHA=${{ github.sha }} echo "COMMIT_SHA=$COMMIT_SHA" >> $GITHUB_OUTPUT - echo "COMMIT_SHA_SHORT=${COMMIT_SHACOMMIT_SHA:0:8}" >> $GITHUB_OUTPUT + echo "COMMIT_SHA_SHORT=${COMMIT_SHA:0:8}" >> $GITHUB_OUTPUT # export REF_NAME=${{ github.ref_name }} echo "REF_SLUG=${REF_NAME//\//_}" >> $GITHUB_OUTPUT @@ -206,7 +206,7 @@ jobs: - name: Info vars run: | - echo "COMMIT_SHAgithub.sha: ${{ needs.preflight.outputs.COMMIT_SHA }}" + echo "COMMIT_SHA: ${{ needs.preflight.outputs.COMMIT_SHA }}" echo "COMMIT_SHA_SHORT: ${{ needs.preflight.outputs.COMMIT_SHA_SHORT }}" echo "SOURCE_REF_SLUG: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }}" echo "REF_SLUG: ${{ needs.preflight.outputs.REF_SLUG }}" From 9b36ec88605076c661a83b60b0c7776983416858 Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Mon, 23 Sep 2024 16:53:39 +0400 Subject: [PATCH 07/10] yarn version annotation --- .github/workflows/reusable-preflight.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable-preflight.yml b/.github/workflows/reusable-preflight.yml index fae7daa9a31b..59b3734136ac 100644 --- a/.github/workflows/reusable-preflight.yml +++ b/.github/workflows/reusable-preflight.yml @@ -196,7 +196,7 @@ jobs: run: | rustup show cargo --version - yarn --version + echo "yarn version: $(yarn --version)" echo $( substrate-contracts-node --version | awk 'NF' ) estuary --version cargo-contract --version From d6e47710d8babf63c26c7bd581dafeb41cc81211 Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Tue, 24 Sep 2024 13:33:37 +0400 Subject: [PATCH 08/10] ci-versions checkout --- .github/workflows/reusable-preflight.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/reusable-preflight.yml b/.github/workflows/reusable-preflight.yml index 59b3734136ac..e1799adddcaf 100644 --- a/.github/workflows/reusable-preflight.yml +++ b/.github/workflows/reusable-preflight.yml @@ -191,11 +191,14 @@ jobs: container: image: ${{ needs.preflight.outputs.IMAGE }} steps: + - uses: actions/checkout@v4 - name: Info rust run: | rustup show cargo --version + cargo +nightly --version + cargo clippy --version echo "yarn version: $(yarn --version)" echo $( substrate-contracts-node --version | awk 'NF' ) estuary --version From c50df7ff31743b80f5cf8cb4839af7992911eee3 Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Tue, 24 Sep 2024 13:35:02 +0400 Subject: [PATCH 09/10] Update .github/workflows/reusable-preflight.yml Co-authored-by: Oliver Tale-Yazdi --- .github/workflows/reusable-preflight.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/reusable-preflight.yml b/.github/workflows/reusable-preflight.yml index e1799adddcaf..54c7d090f154 100644 --- a/.github/workflows/reusable-preflight.yml +++ b/.github/workflows/reusable-preflight.yml @@ -199,6 +199,8 @@ jobs: cargo --version cargo +nightly --version cargo clippy --version + cargo +nightly --version + cargo clippy --version echo "yarn version: $(yarn --version)" echo $( substrate-contracts-node --version | awk 'NF' ) estuary --version From f47d0cc58e913fadf0410896d6de79f0cf8bbc8b Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Tue, 24 Sep 2024 13:36:04 +0400 Subject: [PATCH 10/10] remove redundant version --- .github/workflows/reusable-preflight.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/reusable-preflight.yml b/.github/workflows/reusable-preflight.yml index 54c7d090f154..e1799adddcaf 100644 --- a/.github/workflows/reusable-preflight.yml +++ b/.github/workflows/reusable-preflight.yml @@ -199,8 +199,6 @@ jobs: cargo --version cargo +nightly --version cargo clippy --version - cargo +nightly --version - cargo clippy --version echo "yarn version: $(yarn --version)" echo $( substrate-contracts-node --version | awk 'NF' ) estuary --version