Updates srtool to 1.69.0 in the CI (#2339)

* Updates workflows to use bare-metal instead of self-hosted * Updates docker login action * forcing submodules * Forces master ref * dirty clean * reset reset moonbeam repo * more hacks? * testing differently * testing registry * Forces reclone moonbeam * fix srtool command line * break scripts * Testing recent srtool * Testing recent srtool * more dangerous srtool * Restore valid srtool * testing root * no root * Custom srtool * adds logs * removes logs * new srtool image * more logs * force docker user * adds cleanup docker run * better cleanup * syntax * typo * better clean * force new scripts * restore ssh custom * Forces chwon to srtool * better chown user id * again ? * retry * chmod instead * hacking ? * copy ? * cat entry * retrying * restore * simplify docker call * cleaner * normlal srtool * more debug * fix original scripts * Setup final action * Better cleanup * fix cleanup * Reset ssh auth for srtool * Testing private again * adds check for file permission * more logs * More checks * better ls * more * more check * better release note * Updates rust-toolchain * Adds custom script to get docker user id * Cleanup * Adds doc * revert tag for custom srtool
moonbeam-foundation · Jun 14, 2023 · 116df29 · 116df29
1 parent 670eddf
commit 116df29
Show file tree

Hide file tree

Showing 12 changed files with 103 additions and 17 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -227,7 +227,7 @@ jobs:
         if: |
           github.event_name == 'push' ||
           github.event.pull_request.head.repo.full_name == github.repository
-        uses: docker/login-action@v2.1.0
+        uses: docker/login-action@v2.2.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -468,7 +468,7 @@ jobs:
           driver-opts: |
             image=moby/buildkit:master
       - name: Login to DockerHub
-        uses: docker/login-action@v2.1.0
+        uses: docker/login-action@v2.2.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}

diff --git a/.github/workflows/client-release-issue.yml b/.github/workflows/client-release-issue.yml
@@ -11,7 +11,7 @@ on:
 
 jobs:
   setup-scripts:
-    runs-on: self-hosted
+    runs-on: bare-metal
     steps:
       - uses: actions/checkout@v3
       - name: Upload tools

diff --git a/.github/workflows/prepare-binary.yml b/.github/workflows/prepare-binary.yml
@@ -12,7 +12,7 @@ jobs:
   ####### Building binaries #######
 
   build-binary:
-    runs-on: self-hosted
+    runs-on: bare-metal
     strategy:
       matrix:
         cpu: ["", "skylake"]
@@ -73,7 +73,7 @@ jobs:
           driver-opts: |
             image=moby/buildkit:master
       - name: Login to DockerHub
-        uses: docker/login-action@v2.1.0
+        uses: docker/login-action@v2.2.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}

diff --git a/.github/workflows/publish-binary.yml b/.github/workflows/publish-binary.yml
@@ -14,7 +14,7 @@ on:
 jobs:
   ####### Building binaries #######
   setup-scripts:
-    runs-on: self-hosted
+    runs-on: bare-metal
     steps:
       - uses: actions/checkout@v3
       - name: Upload tools
@@ -25,7 +25,7 @@ jobs:
 
   build-binary:
     needs: ["setup-scripts"]
-    runs-on: self-hosted
+    runs-on: bare-metal
     strategy:
       matrix:
         cpu: ["", "skylake", "znver3"]

diff --git a/.github/workflows/publish-docker-runtime.yml b/.github/workflows/publish-docker-runtime.yml
@@ -16,7 +16,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Login to DockerHub
-        uses: docker/login-action@v2.1.0
+        uses: docker/login-action@v2.2.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}

diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml
@@ -16,7 +16,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Login to DockerHub
-        uses: docker/login-action@v2.1.0
+        uses: docker/login-action@v2.2.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}

diff --git a/.github/workflows/publish-runtime.yml b/.github/workflows/publish-runtime.yml
@@ -15,7 +15,7 @@ jobs:
   ####### Build runtimes with srtool #######
 
   setup-scripts:
-    runs-on: self-hosted
+    runs-on: bare-metal
     steps:
       - uses: actions/checkout@v3
       - name: Upload scripts
@@ -31,24 +31,25 @@ jobs:
 
   build-srtool-runtimes:
     needs: ["setup-scripts"]
-    runs-on: self-hosted
+    runs-on: bare-metal
     strategy:
       matrix:
         chain: ["moonbase", "moonriver", "moonbeam"]
         srtool_image:
           - purestake/srtool
         srtool_image_tag:
-          - 1.66.1
+          - 1.69.0
     steps:
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.event.inputs.to }}
       - name: Login to DockerHub
-        uses: docker/login-action@v2.1.0
+        uses: docker/login-action@v2.2.0
         if: github.repository == 'purestake/moonbeam'
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+          registry: index.docker.io
       - name: Download original scripts
         uses: actions/download-artifact@v3
         with:
@@ -78,6 +79,10 @@ jobs:
           GH_WORKFLOW_MATRIX_SRTOOL_IMAGE: ${{ matrix.srtool_image }}
           GH_WORKFLOW_MATRIX_SRTOOL_IMAGE_TAG: ${{ matrix.srtool_image_tag }}
         run: |
+          # Ensure we have permissions to write to the runtime folder target for the docker user
+          mkdir -p runtime/${GH_WORKFLOW_MATRIX_CHAIN}/target
+          chmod uog+rwX runtime/${GH_WORKFLOW_MATRIX_CHAIN}/target
+
           chmod u+x ./original-scripts/build-runtime-srtool.sh
           ./original-scripts/build-runtime-srtool.sh
       - name: Summary
@@ -92,6 +97,14 @@ jobs:
           path: |
             ${{ matrix.chain }}-runtime.compact.compressed.wasm
             ${{ matrix.chain }}-srtool-digest.json
+      - name: Cleanup
+        run: |
+          echo "Removing runtime/${{ matrix.chain }}/target"
+          ls -lna runtime/${{ matrix.chain }}/target
+
+          # Removing using docker root to ensure we have permissions
+          docker run -u 0 -v $(pwd)/runtime/${{ matrix.chain }}/:/runtime \
+            -it --entrypoint bash ${{ matrix.srtool_image }}:${{ matrix.srtool_image_tag }} -c "id; ls -lna /runtime/target; rm -rf /runtime/target"
 
   ####### Prepare the release draft #######
 

diff --git a/.github/workflows/runtime-release-issue.yml b/.github/workflows/runtime-release-issue.yml
@@ -14,7 +14,7 @@ on:
 
 jobs:
   setup-scripts:
-    runs-on: self-hosted
+    runs-on: bare-metal
     steps:
       - uses: actions/checkout@v3
       - name: Upload tools

diff --git a/AUTOMATION.md b/AUTOMATION.md
@@ -0,0 +1,59 @@
+# Automation
+
+This section of the documentation is dedicated to the automation processes for the CI.
+
+## Architecture
+
+Automation is using Github Action, where all the actions are described in [.github/workflows](.github/workflows)
+
+### bare-metal
+
+label bare-metal refers to our CI servers managed by opslayer. Those are dedicated machines, optimized to reduce the compilation and testing time of the actions.
+
+## Cancellation
+
+It is possible to cancel actions directly in github action UI or using (replace "coverage.yml" by the desired
+action and "my-branch" by the desired pr branch):  
+`gh run list --workflow="coverage.yml" --json status,headBranch,databaseId | jq '.[] | select(.headBranch == "my-branch" and (.status == "in_progress" or .status == "queued")) | .databaseId' | xargs -n1 --no-run-if-empty gh run cancel`
+
+## Actions
+
+### Cancel
+
+Cancel allows to cancel previous execution of the same action for the same PR in order to release allocated resources. This is the case of
+[build](#build) and [coverage](#coverage) actions
+
+### Build
+
+[.github/workflows/build.yml](.github/workflows/build.yml)  
+`gh workflow run build -r my-branch-or-tag`
+
+Performs multiple actions to ensure the code provided is high quality:
+
+- Checks formatting/copyrights/links/locks/...
+- Builds the runtime and binary (with and without features)
+- Prepare external binaries (polkadot) matching the new code
+- Tests:
+  - Rust tests (mostly unit), including tracing features
+  - Typescript (mostly dev integrations), including full relay chain & tracing
+  - Fork & Dev upgrade tests
+- Build a docker image based on the sha and push to docker.io
+
+### Coverage
+
+[.github/workflows/coverage.yml](.github/workflows/coverage.yml)  
+`gh workflow run coverage -r my-branch-or-tag`
+
+Similar to [build](#build), coverage ensure the quality of our code
+and test by re-building our binary with `grcov` and re-executing our tests.
+
+The output is stored as an artefact and published in the comments of the PR.
+
+### Publish runtime draft
+
+[.github/workflows/publish-runtime.yml](.github/workflows/publish-runtime.yml)  
+`gh workflow run "Publish Runtime Draft" -r master -f from=runtime-2302 -f to=runtime-2400`
+
+Builds the runtime of each network using srtool and then draft
+the release notes using the git commits between the given tags.
+The draft is "not published" (TODO: change the name of the action)
diff --git a/rust-toolchain b/rust-toolchain
@@ -1,5 +1,5 @@
 [toolchain]
-channel = "1.68.2"
+channel = "1.69.0"
 components = [ "rustfmt", "clippy" ]
 targets = [ "wasm32-unknown-unknown" ]
 profile = "minimal"
diff --git a/scripts/build-runtime-srtool.sh b/scripts/build-runtime-srtool.sh
@@ -1,7 +1,15 @@
+
+# CARGO_NET_GIT_FETCH_WITH_CLI=true and --entrypoint /srtool/entrypoint.sh
+# are required to allow srtool to fetch from github private repositories
+
+# self-hosted runner uses user `maintenance` to match srtool `builder` user 1001
+# $(~/srtool/uid-gid-mapping.sh 1001 | xargs) is used to map the user and group
+
 # Docker command to generate JSON blob of the runtime
 CMD="docker run \
   -i \
   --rm \
+  $(~/srtool/uid-gid-mapping.sh 1001 | xargs) \
   -e CARGO_NET_GIT_FETCH_WITH_CLI=true \
   -e PACKAGE=${GH_WORKFLOW_MATRIX_CHAIN}-runtime \
   -e RUNTIME_DIR=runtime/${GH_WORKFLOW_MATRIX_CHAIN} \

diff --git a/tools/github/print-runtime-release-issue.ts b/tools/github/print-runtime-release-issue.ts
@@ -28,13 +28,16 @@ async function main() {
   const newVersion = argv.to;
   const lastClientVersion = argv.client;
 
-  const commonTemplate = `
+  const commonTemplate =
+    `
 ## Release
 - [ ] Check all proxy types.
 - [ ] Re-run all extrinsics/hooks benchmarks.
 - [ ] Tag master with runtime-${newVersion} and push to github
 - [ ] Start the github action Publish Runtime Draft
 with runtime-${previousVersion} => runtime-${newVersion}
+  - \`gh workflow run "Publish Runtime Draft" -r 'master' ` +
+    `-f from=runtime-${previousVersion} -f to=runtime-${newVersion}\`
 - [ ] Review the generated Draft and clean a bit the messages if needed (keep it draft)
 - [ ] Create the tracing runtime on moonbeam-runtime-overrides
 (see https://github.com/PureStake/moonbeam-runtime-overrides/blob/master/README.md)
@@ -49,7 +52,8 @@ with ${lastClientVersion} and master
 
   // Detect if it's a major release or hotfix
   if (newVersion.endsWith("00")) {
-    const template = `
+    const template =
+      `
 ## Requirements
 - [ ] To be manually edited (add pending PRs)
 
@@ -64,6 +68,8 @@ ${commonTemplate}
 
 ## Post Release
 - [ ] Publish the docker runtime image (trigger the github action "Publish Docker runtime")
+  - \`gh workflow run "Publish Runtime Draft" -r 'master' ` +
+      `-f from=runtime-${previousVersion} -f to=runtime-${newVersion}\`
 - [ ] Create a PR that increment spec version (like #1051)
     `;
     console.log(template);