From 646b4e10f580564ef7da248cf55ad969443e9371 Mon Sep 17 00:00:00 2001
From: Eduard S <eduardsanou@posteo.net>
Date: Thu, 20 Jul 2023 17:29:47 +0200
Subject: [PATCH] Clean up github workflows

- Remove bitrot task which was building the benchmarks and examples but we don't have any
- Merge cargo fmt, clippy and doc links into a single job to reduce noise
- Split heavy tests and light tests into two jobs to parallelize the work
- Merge all feature tests into a single job to reduce noise.
- Disable integration tests on pull requests because they are taking a long time currently.  Enable them only when the trigger-integration-test label is set, when they are scheduled via cron, or when pushing to main.
---
 .github/workflows/ci.yml            | 151 ++++++----------------------
 .github/workflows/integration.yml   |   5 +-
 .github/workflows/lints.yml         |  19 +++-
 .github/workflows/test-features.yml |  29 +++---
 4 files changed, 63 insertions(+), 141 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0887779a974..6b4ef97b5d5 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1,4 +1,4 @@
-name: CI checks
+name: Main tests
 
 on:
   merge_group:
@@ -28,13 +28,13 @@ jobs:
           concurrent_skipping: 'same_content_newer'
           paths_ignore: '["**/README.md"]'
 
-  test:
+  lighttest:
     needs: [skip_check]
     if: |
       github.event.pull_request.draft == false &&
       (github.event.action == 'ready_for_review' || needs.skip_check.outputs.should_skip != 'true')
 
-    name: Test
+    name: Light unit tests
     runs-on: ["${{github.run_id}}", self-hosted, c5.9xlarge]
 
     steps:
@@ -71,79 +71,25 @@ jobs:
         with:
           command: test
           args: --verbose --release --all --all-features --exclude integration-tests --exclude circuit-benchmarks
-      - name: Run heavy tests # heavy tests are run serially to avoid OOM
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --verbose --release --all --all-features --exclude integration-tests --exclude circuit-benchmarks serial_ -- --ignored --test-threads 1
 
-  build:
+  heavytests:
     needs: [skip_check]
     if: |
       github.event.pull_request.draft == false &&
       (github.event.action == 'ready_for_review' || needs.skip_check.outputs.should_skip != 'true')
 
-    name: Build target ${{ matrix.target }}
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        target:
-          - wasm32-unknown-unknown
-          - wasm32-wasi
+    name: Heavy unit tests
+    runs-on: ["${{github.run_id}}", self-hosted, c5.9xlarge]
 
     steps:
       - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1
         with:
           override: false
-      - name: Add target
-        run: rustup target add ${{ matrix.target }}
-      # Go cache for building geth-utils
-      - name: Go cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
-      - name: Cargo cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ runner.os }}-${{ matrix.target }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-      - name: cargo build
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --all-features
-      # Make sure benchmarks compile.
-      - name: cargo build benchmarks no-run
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --verbose --release --all-features -p circuit-benchmarks --no-run
-
-  bitrot:
-    needs: [skip_check]
-    if: |
-      github.event.pull_request.draft == false &&
-      (github.event.action == 'ready_for_review' || needs.skip_check.outputs.should_skip != 'true')
-
-    name: Bitrot check
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
+      - name: Setup golang
+        uses: actions/setup-go@v3
         with:
-          override: false
+          go-version: ~1.19
       # Go cache for building geth-utils
       - name: Go cache
         uses: actions/cache@v3
@@ -164,27 +110,32 @@ jobs:
             ~/.cargo/git/db/
             target/
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-      # Build benchmarks to prevent bitrot
-      - name: Build benchmarks
+      - name: Run heavy tests # heavy tests are run serially to avoid OOM
         uses: actions-rs/cargo@v1
         with:
-          command: build
-          args: --benches --examples --all-features
+          command: test
+          args: --verbose --release --all --all-features --exclude integration-tests --exclude circuit-benchmarks serial_ -- --ignored --test-threads 1
 
-  doc-links:
+  build:
     needs: [skip_check]
     if: |
       github.event.pull_request.draft == false &&
       (github.event.action == 'ready_for_review' || needs.skip_check.outputs.should_skip != 'true')
 
-    name: Intra-doc links
+    name: Build target ${{ matrix.target }}
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        target:
+          - x86_64-unknown-linux-gnu
 
     steps:
       - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1
         with:
           override: false
+      - name: Add target
+        run: rustup target add ${{ matrix.target }}
       # Go cache for building geth-utils
       - name: Go cache
         uses: actions/cache@v3
@@ -204,61 +155,15 @@ jobs:
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
             target/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-      - name: cargo fetch
-        uses: actions-rs/cargo@v1
-        with:
-          command: fetch
-
-      # Ensure intra-documentation links all resolve correctly
-      # Requires #![deny(intra_doc_link_resolution_failure)] in crates.
-      - name: Check intra-doc links
-        uses: actions-rs/cargo@v1
-        with:
-          command: doc
-          args: --no-deps --all --document-private-items
-
-  fmt:
-    needs: [skip_check]
-    if: |
-      github.event.pull_request.draft == false &&
-      (github.event.action == 'ready_for_review' || needs.skip_check.outputs.should_skip != 'true')
-
-    name: Rustfmt
-    timeout-minutes: 30
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          override: false
-      # Go cache for building geth-utils
-      - name: Go cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
-      - name: Cargo cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-      - name: cargo check
+          key: ${{ runner.os }}-${{ matrix.target }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+      - name: cargo build
         uses: actions-rs/cargo@v1
         with:
-          command: check
+          command: build
           args: --all-features
-      - run: rustup component add rustfmt
-      - uses: actions-rs/cargo@v1
+      # Make sure benchmarks compile.
+      - name: cargo build benchmarks no-run
+        uses: actions-rs/cargo@v1
         with:
-          command: fmt
-          args: --all -- --check
+          command: test
+          args: --verbose --release --all-features -p circuit-benchmarks --no-run
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
index e2183badad0..2ae9f721391 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -28,7 +28,10 @@ concurrency:
 
 jobs:
   set-outputs:
-    if: github.event.pull_request.draft == false
+    if: github.event_name == 'schedule' ||
+        github.event_name == 'push' ||
+        github.event_name == 'workflow_dispatch' ||
+        contains(github.event.pull_request.labels.*.name, 'trigger-integration-tests')
 
     runs-on: ubuntu-latest
     outputs:
diff --git a/.github/workflows/lints.yml b/.github/workflows/lints.yml
index 7f59474bb83..ec70e24d4ed 100644
--- a/.github/workflows/lints.yml
+++ b/.github/workflows/lints.yml
@@ -22,13 +22,13 @@ jobs:
           concurrent_skipping: 'same_content_newer'
           paths_ignore: '["**/README.md"]'
 
-  clippy:
+  lints:
     needs: [skip_check]
     if: |
       github.event.pull_request.draft == false &&
       (github.event.action == 'ready_for_review' || needs.skip_check.outputs.should_skip != 'true')
 
-    name: Clippy
+    name: Various lints
     timeout-minutes: 30
     runs-on: ubuntu-latest
 
@@ -58,9 +58,24 @@ jobs:
             ~/.cargo/git/db/
             target/
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Check code format
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
       - name: Run clippy
         uses: actions-rs/clippy-check@v1
         with:
           name: Clippy
           token: ${{ secrets.GITHUB_TOKEN }}
           args: --all-features --all-targets -- -D warnings
+
+      # Ensure intra-documentation links all resolve correctly
+      # Requires #![deny(intra_doc_link_resolution_failure)] in crates.
+      - name: Check intra-doc links
+        uses: actions-rs/cargo@v1
+        with:
+          command: doc
+          args: --no-deps --all --document-private-items
diff --git a/.github/workflows/test-features.yml b/.github/workflows/test-features.yml
index c22f67eec37..71e82633e35 100644
--- a/.github/workflows/test-features.yml
+++ b/.github/workflows/test-features.yml
@@ -26,6 +26,7 @@ jobs:
     if: |
       github.event.pull_request.draft == false &&
       (github.event.action == 'ready_for_review' || needs.skip_check.outputs.should_skip != 'true')
+    name: Validate features exist
     timeout-minutes: 30
     runs-on: ubuntu-latest
     strategy:
@@ -64,19 +65,17 @@ jobs:
             target/
           key: ${{ github.workflow }}-${{ matrix.crate }}-${{ matrix.feature }}-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
 
-      - name: '${{ matrix.crate }}: ${{ matrix.feature }}'
+      - name: Import with features
         run: |
-          GIT_ROOT=$(pwd)
-          cd /tmp
-          cargo new foobar
-          cd foobar
-          cp "${GIT_ROOT}/rust-toolchain" . || true
-          cargo add --path "${GIT_ROOT}/${{ matrix.crate }}" --features '${{ matrix.feature }}'
-          cd ../
-          rm -rf foobar
-
-  test_features_complete:
-    needs: [test_features]
-    runs-on: ubuntu-latest
-    steps:
-      - run: echo dummy
+          for crate in zkevm-circuits; do
+            for feature in default test-circuits test-util warn-unimplemented; do
+              GIT_ROOT=$(pwd)
+              cd /tmp
+              cargo new foobar
+              cd foobar
+              cp "${GIT_ROOT}/rust-toolchain" . || true
+              cargo add --path "${GIT_ROOT}/${crate}" --features "${feature}"
+              cd ../
+              rm -rf foobar
+            done
+          done