From 53aef908779487d028fa71a451b39780457d275a Mon Sep 17 00:00:00 2001 From: Sebastian Lorenz Date: Fri, 22 Aug 2025 00:41:49 +0200 Subject: [PATCH 1/8] ci: add justfile and nextest with test groups & mutex --- .config/nextest.toml | 14 +++ .github/workflows/audit.yml | 2 +- .github/workflows/ci.yml | 182 +++++++++++++--------------- .github/workflows/code-coverage.yml | 81 ------------- CLAUDE.md | 64 ++++------ flake.nix | 4 +- justfile | 81 +++++++++++++ 7 files changed, 206 insertions(+), 222 deletions(-) create mode 100644 .config/nextest.toml delete mode 100644 .github/workflows/code-coverage.yml create mode 100644 justfile diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 00000000000..1b4627e5ded --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,14 @@ +[[profile.default.overrides]] +filter = 'package(graphman-server)' +priority = -1 +threads-required = 'num-test-threads' # Global mutex + +[[profile.default.overrides]] +filter = 'package(test-store)' +priority = -1 +threads-required = 'num-test-threads' # Global mutex + +[[profile.default.overrides]] +filter = 'package(graph-tests)' +priority = -1 +threads-required = 'num-test-threads' # Global mutex diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 5c5129c51a2..96fa5ba1cb8 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 - uses: rustsec/audit-check@69366f33c96575abad1ee0dba8212993eecbe998 #v2.0.0 with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4163a7f597e..72f28f8fe82 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,21 +38,27 @@ jobs: env: RUSTFLAGS: "-C link-arg=-fuse-ld=lld -D warnings" steps: - - name: Tune GitHub hosted runner to reduce flakiness - # https://github.com/smorimoto/tune-github-hosted-runner-network/blob/main/action.yml - run: sudo ethtool -K eth0 tx off rx off - - name: Checkout sources - uses: actions/checkout@v4 - # Don't use the rust-cache as it leads to 'no space left on device' errors - # - uses: Swatinem/rust-cache@v2 - - name: Install lld - run: sudo apt-get install -y lld protobuf-compiler + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + - name: Setup dependencies + run: | + sudo apt-get update + sudo apt-get install -y lld protobuf-compiler - - name: Run unit tests - uses: actions-rs/cargo@v1 + - uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 + with: + cache: false + + - name: Setup just + uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 + + - name: Install cargo-nextest + uses: baptiste0928/cargo-install@e38323ef017552d7f7af73a3f4db467f278310ed # v3 with: - command: test - args: --workspace --exclude graph-tests -- --nocapture + crate: cargo-nextest + version: ^0.9 + + - name: Run unit tests + run: just test-unit --verbose runner-tests: name: Subgraph Runner integration tests @@ -83,17 +89,30 @@ jobs: RUSTFLAGS: "-C link-arg=-fuse-ld=lld -D warnings --cfg test_with_ipfs" RUNNER_TESTS_WAIT_FOR_SYNC_SECS: "600" steps: - - name: Tune GitHub hosted runner to reduce flakiness - # https://github.com/smorimoto/tune-github-hosted-runner-network/blob/main/action.yml - run: sudo ethtool -K eth0 tx off rx off - - name: Checkout sources - uses: actions/checkout@v4 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + - name: Setup dependencies + run: | + sudo apt-get update + sudo apt-get install -y lld protobuf-compiler + + - uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 + with: + cache: false + + - name: Setup just + uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 + + - name: Install cargo-nextest + uses: baptiste0928/cargo-install@e38323ef017552d7f7af73a3f4db467f278310ed # v3 + with: + crate: cargo-nextest + version: ^0.9 - name: Install pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4 - name: Install Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: node-version: 20 cache: pnpm @@ -101,24 +120,8 @@ jobs: - name: Install Node.js dependencies run: pnpm install - - uses: Swatinem/rust-cache@v2 - - - name: Install lld - run: sudo apt-get install -y lld protobuf-compiler - - name: Run runner tests - id: runner-tests-1 - uses: actions-rs/cargo@v1 - with: - command: test - args: --package graph-tests --test runner_tests - - - name: Run file link resolver test - id: file-link-resolver-test - uses: actions-rs/cargo@v1 - with: - command: test - args: --package graph-tests --test file_link_resolver + run: just test-runner --verbose integration-tests: name: Run integration tests @@ -146,18 +149,30 @@ jobs: env: RUSTFLAGS: "-C link-arg=-fuse-ld=lld -D warnings" steps: - - name: Tune GitHub hosted runner to reduce flakiness - # https://github.com/smorimoto/tune-github-hosted-runner-network/blob/main/action.yml - run: sudo ethtool -K eth0 tx off rx off - - name: Checkout sources - uses: actions/checkout@v4 - - uses: Swatinem/rust-cache@v2 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + - name: Setup dependencies + run: | + sudo apt-get update + sudo apt-get install -y lld protobuf-compiler + + - uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 + with: + cache: false + + - name: Setup just + uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 + + - name: Install cargo-nextest + uses: baptiste0928/cargo-install@e38323ef017552d7f7af73a3f4db467f278310ed # v3 + with: + crate: cargo-nextest + version: ^0.9 - name: Install pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4 - name: Install Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: node-version: 20 cache: pnpm @@ -165,39 +180,19 @@ jobs: - name: Install Node.js dependencies run: pnpm install - - name: Install lld and jq - run: sudo apt-get install -y lld jq protobuf-compiler - - - name: Install Foundry - uses: foundry-rs/foundry-toolchain@v1 - with: - # Pinned to specific version since newer versions do not produce - # deterministic block hashes. Unpin once that's fixed upstream - version: v1.2.3 - name: Start anvil run: anvil --gas-limit 100000000000 --base-fee 1 --block-time 2 --timestamp 1743944919 --port 3021 & - - name: Install graph CLI - run: curl -sSL http://cli.thegraph.com/install.sh | sudo bash - - name: Build graph-node - uses: actions-rs/cargo@v1 - with: - command: build - args: --bin graph-node --test integration_tests + run: just build --test integration_tests - name: Run integration tests - id: integration-tests-1 - uses: actions-rs/cargo@v1 - env: - # Basically, unlimted concurrency - N_CONCURRENT_TESTS: "1000" - with: - command: test - args: --test integration_tests -- --nocapture + run: just test-integration --verbose + - name: Cat graph-node.log if: always() run: cat tests/integration-tests/graph-node.log || echo "No graph-node.log" + rustfmt: name: Check rustfmt style runs-on: ubuntu-latest @@ -205,29 +200,16 @@ jobs: env: RUSTFLAGS: "-D warnings" steps: - - uses: actions/checkout@v4 - - - name: Check formatting - uses: actions-rs/cargo@v1 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + - uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 with: - command: fmt - args: --all -- --check + cache: false - clippy: - name: Clippy linting - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v4 - # Unlike rustfmt, Clippy actually compiles stuff so it benefits from - # caching. - - uses: Swatinem/rust-cache@v2 - - name: Install deps - run: sudo apt-get install -y protobuf-compiler - - name: Run Clippy - uses: actions-rs/cargo@v1 - with: - command: check-clippy + - name: Setup just + uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 + + - name: Check formatting + run: just format --check release-check: name: Build in release mode @@ -236,14 +218,18 @@ jobs: env: RUSTFLAGS: "-D warnings" steps: - - uses: actions/checkout@v4 - - uses: Swatinem/rust-cache@v2 - - name: Install dependencies + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + - name: Setup dependencies run: | sudo apt-get update - sudo apt-get -y install libpq-dev protobuf-compiler - - name: Cargo check (release) - uses: actions-rs/cargo@v1 + sudo apt-get install -y protobuf-compiler + + - uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 with: - command: check - args: --release + cache: false + + - name: Setup just + uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 + + - name: Cargo check (release) + run: just check --release diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml deleted file mode 100644 index 69049cb5567..00000000000 --- a/.github/workflows/code-coverage.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: Code coverage - -on: - workflow_dispatch: - schedule: - # Run it every 3 days. - - cron: "0 3 * * *" - -env: - CARGO_TERM_COLOR: always - RUST_BACKTRACE: full - THEGRAPH_STORE_POSTGRES_DIESEL_URL: "postgresql://postgres:postgres@localhost:5432/graph_node_test" - RUSTFLAGS: "-C link-arg=-fuse-ld=lld -D warnings" - N_CONCURRENT_TESTS: "4" - TESTS_GANACHE_HARD_WAIT_SECONDS: "30" - -jobs: - # Heavily inspired from . - coverage: - name: Code coverage of integration tests - runs-on: ubuntu-latest - timeout-minutes: 60 - services: - ipfs: - image: ipfs/go-ipfs:v0.10.0 - ports: - - 5001:5001 - postgres: - image: postgres - env: - POSTGRES_PASSWORD: postgres - POSTGRES_DB: graph_node_test - POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C" - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - steps: - - uses: actions/checkout@v4 - - uses: Swatinem/rust-cache@v2 - - - name: Install pnpm - uses: pnpm/action-setup@v4 - - - name: Install Node.js - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: pnpm - - - name: Install Node.js dependencies - run: pnpm install - - - name: Install lld - run: sudo apt-get install -y lld jq protobuf-compiler - - uses: actions-rs/cargo@v1 - with: - command: install - args: cargo-llvm-cov - - - name: Build graph-node - uses: actions-rs/cargo@v1 - with: - command: build - args: --bin graph-node - - - name: Generate code coverage - run: cargo llvm-cov --package graph-tests --lcov --output-path lcov.info -- --nocapture - - uses: actions/upload-artifact@v4 - with: - name: code-coverage-info - path: lcov.info - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v5 - with: - # No token needed, because the repo is public. - files: lcov.info - fail_ci_if_error: true diff --git a/CLAUDE.md b/CLAUDE.md index 84042f4b8f9..9b91bfeda7d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,28 +28,21 @@ Unit tests are inlined with source code. 2. IPFS running on localhost:5001 3. PNPM 4. Foundry (for smart contract compilation) -5. Environment variable `THEGRAPH_STORE_POSTGRES_DIESEL_URL` set +5. Environment variable `THEGRAPH_STORE_POSTGRES_DIESEL_URL` set to `postgresql://graph:graph@127.0.0.1:5432/graph-test` The environment dependencies and environment setup are operated by the human. **Running Unit Tests:** ```bash -# REQUIRED: Export database connection string before running unit tests (do this **once** at the beginning of a testing session) -export THEGRAPH_STORE_POSTGRES_DIESEL_URL="postgresql://graph:graph@127.0.0.1:5432/graph-test" +# Run unit tests +just test-unit -# Run unit tests (excluding integration tests) -cargo test --workspace --exclude graph-tests - -# Run specific tests -cargo test -p graph data_source::common::tests -cargo test +# Run specific tests (e.g. `data_source::common::tests`) +just test-unit data_source::common::tests ``` **⚠️ Test Verification Requirements:** -- **ALWAYS verify tests actually ran** - Check the output for "test result: ok. X passed" where X > 0 -- **If output shows "0 passed" or "0 tests run"**, the test filter/path was wrong - fix and re-run -- **Never trust exit code 0 alone** - Cargo can exit successfully even when no tests matched your filter -- **For specific test names**, ensure the test name appears in the output as "test {name} ... ok" +When filtering for specific tests, ensure the intended test name(s) appear in the output. ### Runner Tests (Integration Tests) @@ -58,24 +51,19 @@ cargo test 2. IPFS running on localhost:5001 3. PNPM 4. Foundry (for smart contract compilation) -5. Environment variable `THEGRAPH_STORE_POSTGRES_DIESEL_URL` set +5. Environment variable `THEGRAPH_STORE_POSTGRES_DIESEL_URL` set to `postgresql://graph:graph@127.0.0.1:5432/graph-test` **Running Runner Tests:** ```bash -# REQUIRED: Export database connection string before running unit tests (do this **once** at the beginning of a testing session) -export THEGRAPH_STORE_POSTGRES_DIESEL_URL="postgresql://graph:graph@127.0.0.1:5432/graph-test" - -# Run all runner tests -cargo test -p graph-tests --test runner_tests -- --nocapture +# Run runner tests. +just test-runner -# Run specific runner tests -cargo test -p graph-tests --test runner_tests test_name -- --nocapture +# Run specific tests (e.g. `block_handlers`) +just test-runner block_handlers ``` **⚠️ Test Verification Requirements:** -- **ALWAYS verify tests actually ran** - Check the output for "test result: ok. X passed" where X > 0 -- **If output shows "0 passed" or "0 tests run"**, the test filter/path was wrong - fix and re-run -- **Never trust exit code 0 alone** - Cargo can exit successfully even when no tests matched your filter +When filtering for specific tests, ensure the intended test name(s) appear in the output. **Important Notes:** - Runner tests take moderate time (10-20 seconds) @@ -97,13 +85,13 @@ The environment dependencies and environment setup are operated by the human. **Running Integration Tests:** ```bash # REQUIRED: Build graph-node binary before running integration tests -cargo build --bin graph-node +just build # Run all integration tests -cargo test -p graph-tests --test integration_tests -- --nocapture +just test-integration # Run a specific integration test case (e.g., "grafted" test case) -TEST_CASE=grafted cargo test -p graph-tests --test integration_tests -- --nocapture +TEST_CASE=grafted just test-integration ``` **⚠️ Test Verification Requirements:** @@ -119,13 +107,13 @@ TEST_CASE=grafted cargo test -p graph-tests --test integration_tests -- --nocapt ### Code Quality ```bash # 🚨 MANDATORY: Format all code IMMEDIATELY after any .rs file edit -cargo fmt --all +just format # 🚨 MANDATORY: Check code for warnings and errors - MUST have zero warnings -cargo check +just check -# 🚨 MANDATORY: Build in release mode to catch linking/optimization issues that cargo check misses -cargo check --release +# 🚨 MANDATORY: Check in release mode to catch linking/optimization issues that cargo check misses +just check --release ``` 🚨 **CRITICAL REQUIREMENTS for ANY implementation**: @@ -225,11 +213,8 @@ Currently, the human is required to operate the service dependencies as illustra # PostgreSQL: localhost:5432, IPFS: localhost:5001 nix run .#unit -# Claude: Export the database connection string before running unit tests -export THEGRAPH_STORE_POSTGRES_DIESEL_URL="postgresql://graph:graph@127.0.0.1:5432/graph-test" - # Claude: Run unit tests -cargo test --workspace --exclude graph-tests +just test-unit ``` **Runner Tests:** @@ -238,11 +223,8 @@ cargo test --workspace --exclude graph-tests # PostgreSQL: localhost:5432, IPFS: localhost:5001 nix run .#unit # NOTE: Runner tests are using the same nix services stack as the unit test -# Claude: Export the database connection string before running runner tests -export THEGRAPH_STORE_POSTGRES_DIESEL_URL="postgresql://graph:graph@127.0.0.1:5432/graph-test" - # Claude: Run runner tests -cargo test -p graph-tests --test runner_tests -- --nocapture +just test-runner ``` **Integration Tests:** @@ -252,10 +234,10 @@ cargo test -p graph-tests --test runner_tests -- --nocapture nix run .#integration # Claude: Build graph-node binary before running integration tests -cargo build --bin graph-node +just build # Claude: Run integration tests -cargo test -p graph-tests --test integration_tests +just test-integration ``` **Services Configuration:** diff --git a/flake.nix b/flake.nix index 265b378caac..da2411c844e 100644 --- a/flake.nix +++ b/flake.nix @@ -51,6 +51,8 @@ corepack nodejs postgresql + just + cargo-nextest ]); }; @@ -85,7 +87,7 @@ shared_preload_libraries = "pg_stat_statements"; log_statement = "all"; default_text_search_config = "pg_catalog.english"; - max_connections = 200; + max_connections = 500; }; }; in { diff --git a/justfile b/justfile new file mode 100644 index 00000000000..f0fcab284ce --- /dev/null +++ b/justfile @@ -0,0 +1,81 @@ +# Display available commands and their descriptions (default target) +default: + @just --list + +# Format all Rust code (cargo fmt) +format *EXTRA_FLAGS: + cargo fmt --all {{EXTRA_FLAGS}} + +# Check Rust code (cargo check) +check *EXTRA_FLAGS: + cargo check {{EXTRA_FLAGS}} + +# Build graph-node (cargo build --bin graph-node) +build *EXTRA_FLAGS: + cargo build --bin graph-node {{EXTRA_FLAGS}} + +# Run all tests (unit and integration) +test *EXTRA_FLAGS: + #!/usr/bin/env bash + set -e # Exit on error + + # Ensure that the `THEGRAPH_STORE_POSTGRES_DIESEL_URL` environment variable is set. + if [ -z "$THEGRAPH_STORE_POSTGRES_DIESEL_URL" ]; then + echo "Error: THEGRAPH_STORE_POSTGRES_DIESEL_URL is not set" + exit 1 + fi + + if command -v "cargo-nextest" &> /dev/null; then + cargo nextest run {{EXTRA_FLAGS}} --workspace + else + cargo test {{EXTRA_FLAGS}} --workspace -- --nocapture + fi + +# Run unit tests +test-unit *EXTRA_FLAGS: + #!/usr/bin/env bash + set -e # Exit on error + + # Ensure that the `THEGRAPH_STORE_POSTGRES_DIESEL_URL` environment variable is set. + if [ -z "$THEGRAPH_STORE_POSTGRES_DIESEL_URL" ]; then + echo "Error: THEGRAPH_STORE_POSTGRES_DIESEL_URL is not set" + exit 1 + fi + + if command -v "cargo-nextest" &> /dev/null; then + cargo nextest run {{EXTRA_FLAGS}} --workspace --exclude graph-tests + else + cargo test {{EXTRA_FLAGS}} --workspace --exclude graph-tests -- --nocapture + fi + +# Run runner tests +test-runner *EXTRA_FLAGS: + #!/usr/bin/env bash + set -e # Exit on error + + # Ensure that the `THEGRAPH_STORE_POSTGRES_DIESEL_URL` environment variable is set. + if [ -z "$THEGRAPH_STORE_POSTGRES_DIESEL_URL" ]; then + echo "Error: THEGRAPH_STORE_POSTGRES_DIESEL_URL is not set" + exit 1 + fi + + if command -v "cargo-nextest" &> /dev/null; then + cargo nextest run {{EXTRA_FLAGS}} --package graph-tests --test runner_tests + else + cargo test {{EXTRA_FLAGS}} --package graph-tests --test runner_tests -- --nocapture + fi + +# Run integration tests +test-integration *EXTRA_FLAGS: + #!/usr/bin/env bash + set -e # Exit on error + + if command -v "cargo-nextest" &> /dev/null; then + cargo nextest run {{EXTRA_FLAGS}} --package graph-tests --test integration_tests + else + cargo test {{EXTRA_FLAGS}} --package graph-tests --test integration_tests -- --nocapture + fi + +# Clean workspace (cargo clean) +clean: + cargo clean From 5f19646c5912b4450f9e01eb0e8d20ded72d2ab8 Mon Sep 17 00:00:00 2001 From: Sebastian Lorenz Date: Fri, 22 Aug 2025 02:16:25 +0200 Subject: [PATCH 2/8] ci: enable cache --- .github/workflows/ci.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72f28f8fe82..a2c31e7c8bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,8 +45,6 @@ jobs: sudo apt-get install -y lld protobuf-compiler - uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 - with: - cache: false - name: Setup just uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 @@ -96,8 +94,6 @@ jobs: sudo apt-get install -y lld protobuf-compiler - uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 - with: - cache: false - name: Setup just uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 @@ -156,8 +152,6 @@ jobs: sudo apt-get install -y lld protobuf-compiler - uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 - with: - cache: false - name: Setup just uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 @@ -202,8 +196,6 @@ jobs: steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 - uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 - with: - cache: false - name: Setup just uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 @@ -225,8 +217,6 @@ jobs: sudo apt-get install -y protobuf-compiler - uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 - with: - cache: false - name: Setup just uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 From 7de1d72943d9428bed601b8a4da4d32c91c992da Mon Sep 17 00:00:00 2001 From: Sebastian Lorenz Date: Fri, 22 Aug 2025 02:31:11 +0200 Subject: [PATCH 3/8] ci: add slow timeout --- .config/nextest.toml | 1 + .github/workflows/ci.yml | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/.config/nextest.toml b/.config/nextest.toml index 1b4627e5ded..eaa4610afac 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -12,3 +12,4 @@ threads-required = 'num-test-threads' # Global mutex filter = 'package(graph-tests)' priority = -1 threads-required = 'num-test-threads' # Global mutex +slow-timeout = { period = "300s", terminate-after = 4 } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a2c31e7c8bc..5e6744a35bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -162,6 +162,13 @@ jobs: crate: cargo-nextest version: ^0.9 + - name: Install Foundry + uses: foundry-rs/foundry-toolchain@82dee4ba654bd2146511f85f0d013af94670c4de # v1 + with: + # Pinned to specific version since newer versions do not produce + # deterministic block hashes. Unpin once that's fixed upstream + version: v1.2.3 + - name: Install pnpm uses: pnpm/action-setup@a7487c7e89a18df4991f7f222e4898a00d66ddda # v4 From bd3a6eb593b65d1b06278095d9ec0fcbb91ff206 Mon Sep 17 00:00:00 2001 From: Sebastian Lorenz Date: Fri, 22 Aug 2025 09:55:25 +0200 Subject: [PATCH 4/8] ci: test priorities --- .config/nextest.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index eaa4610afac..3fa8a93aced 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -5,11 +5,11 @@ threads-required = 'num-test-threads' # Global mutex [[profile.default.overrides]] filter = 'package(test-store)' -priority = -1 +priority = -2 threads-required = 'num-test-threads' # Global mutex [[profile.default.overrides]] filter = 'package(graph-tests)' -priority = -1 +priority = -3 threads-required = 'num-test-threads' # Global mutex slow-timeout = { period = "300s", terminate-after = 4 } From bb4f7916850818c03c2f782b8ae77b0d151cdd46 Mon Sep 17 00:00:00 2001 From: Sebastian Lorenz Date: Fri, 22 Aug 2025 18:31:03 +0200 Subject: [PATCH 5/8] ci: thick runners --- .github/workflows/ci.yml | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e6744a35bd..77edcd35ef8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,11 +1,16 @@ name: Continuous Integration - on: push: branches: [master] pull_request: - types: [opened, synchronize, reopened] - workflow_dispatch: + branches: [master] + +permissions: + contents: read + +concurrency: + cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.ref }} env: CARGO_TERM_COLOR: always @@ -15,8 +20,8 @@ env: jobs: unit-tests: name: Run unit tests - runs-on: ubuntu-latest-m - timeout-minutes: 60 + runs-on: nscloud-ubuntu-22.04-amd64-16x32 + timeout-minutes: 20 services: ipfs: image: ipfs/go-ipfs:v0.10.0 @@ -28,6 +33,7 @@ jobs: POSTGRES_PASSWORD: postgres POSTGRES_DB: graph_node_test POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C" + POSTGRESQL_EXTRA_FLAGS: "-c max_connections=1000" options: >- --health-cmd pg_isready --health-interval 10s @@ -60,8 +66,8 @@ jobs: runner-tests: name: Subgraph Runner integration tests - runs-on: ubuntu-latest - timeout-minutes: 60 + runs-on: nscloud-ubuntu-22.04-amd64-16x32 + timeout-minutes: 20 services: ipfs: image: ipfs/go-ipfs:v0.10.0 @@ -121,8 +127,8 @@ jobs: integration-tests: name: Run integration tests - runs-on: ubuntu-latest - timeout-minutes: 60 + runs-on: nscloud-ubuntu-22.04-amd64-16x32 + timeout-minutes: 20 services: ipfs: image: ipfs/go-ipfs:v0.10.0 @@ -135,6 +141,7 @@ jobs: POSTGRES_PASSWORD: let-me-in POSTGRES_DB: graph-node POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C" + POSTGRESQL_EXTRA_FLAGS: "-c max_connections=1000" options: >- --health-cmd pg_isready --health-interval 10s From 37bc09e682f77998b33783a1cbb5929f0cdcff69 Mon Sep 17 00:00:00 2001 From: Sebastian Lorenz Date: Fri, 22 Aug 2025 20:26:38 +0200 Subject: [PATCH 6/8] ci: set max connections --- .github/workflows/ci.yml | 65 ++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77edcd35ef8..4116e5cd2c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,8 @@ concurrency: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: full - THEGRAPH_STORE_POSTGRES_DIESEL_URL: "postgresql://postgres:postgres@localhost:5432/graph_node_test" + RUSTFLAGS: "-C link-arg=-fuse-ld=lld -D warnings" + THEGRAPH_STORE_POSTGRES_DIESEL_URL: "postgresql://graph:graph@localhost:5432/graph-test" jobs: unit-tests: @@ -30,21 +31,29 @@ jobs: postgres: image: postgres env: - POSTGRES_PASSWORD: postgres - POSTGRES_DB: graph_node_test + POSTGRES_USER: graph + POSTGRES_PASSWORD: graph + POSTGRES_DB: graph-test POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C" - POSTGRESQL_EXTRA_FLAGS: "-c max_connections=1000" options: >- - --health-cmd pg_isready + --health-cmd "pg_isready -U graph" --health-interval 10s --health-timeout 5s --health-retries 5 + --name postgres ports: - 5432:5432 - env: - RUSTFLAGS: "-C link-arg=-fuse-ld=lld -D warnings" steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + - name: Alter postgres max connections + run: | + docker exec -i postgres bash << EOF + sed -i -e 's/max_connections = 100/max_connections = 1000/' /var/lib/postgresql/data/postgresql.conf + sed -i -e 's/shared_buffers = 128MB/shared_buffers = 2GB/' /var/lib/postgresql/data/postgresql.conf + EOF + docker restart --timeout 0 postgres + sleep 5 + - name: Setup dependencies run: | sudo apt-get update @@ -74,26 +83,31 @@ jobs: ports: - 5001:5001 postgres: - image: bitnami/postgresql + image: postgres env: - POSTGRES_PASSWORD: postgres - POSTGRES_DB: graph_node_test + POSTGRES_USER: graph + POSTGRES_PASSWORD: graph + POSTGRES_DB: graph-test POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C" - POSTGRESQL_EXTRA_FLAGS: "-c max_connections=1000" - POSTGRESQL_REPLICATION_USE_PASSFILE: no options: >- - --health-cmd "pg_isready -U postgres" + --health-cmd "pg_isready -U graph" --health-interval 10s --health-timeout 5s - --health-retries 10 + --health-retries 5 + --name postgres ports: - 5432:5432 - env: - GRAPH_IPFS_REQUEST_TIMEOUT: "60" - RUSTFLAGS: "-C link-arg=-fuse-ld=lld -D warnings --cfg test_with_ipfs" - RUNNER_TESTS_WAIT_FOR_SYNC_SECS: "600" steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + - name: Alter postgres max connections + run: | + docker exec -i postgres bash << EOF + sed -i -e 's/max_connections = 100/max_connections = 1000/' /var/lib/postgresql/data/postgresql.conf + sed -i -e 's/shared_buffers = 128MB/shared_buffers = 2GB/' /var/lib/postgresql/data/postgresql.conf + EOF + docker restart --timeout 0 postgres + sleep 5 + - name: Setup dependencies run: | sudo apt-get update @@ -141,18 +155,25 @@ jobs: POSTGRES_PASSWORD: let-me-in POSTGRES_DB: graph-node POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C" - POSTGRESQL_EXTRA_FLAGS: "-c max_connections=1000" options: >- - --health-cmd pg_isready + --health-cmd "pg_isready -U graph-node" --health-interval 10s --health-timeout 5s --health-retries 5 + --name postgres ports: - 3011:5432 - env: - RUSTFLAGS: "-C link-arg=-fuse-ld=lld -D warnings" steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + - name: Alter postgres max connections + run: | + docker exec -i postgres bash << EOF + sed -i -e 's/max_connections = 100/max_connections = 1000/' /var/lib/postgresql/data/postgresql.conf + sed -i -e 's/shared_buffers = 128MB/shared_buffers = 2GB/' /var/lib/postgresql/data/postgresql.conf + EOF + docker restart --timeout 0 postgres + sleep 5 + - name: Setup dependencies run: | sudo apt-get update From 435a100906b687f29ea6bf3f00ef0a334dc427f5 Mon Sep 17 00:00:00 2001 From: Sebastian Lorenz Date: Fri, 22 Aug 2025 21:35:11 +0200 Subject: [PATCH 7/8] ci: clippy --- .cargo/config.toml | 4 ---- .github/workflows/ci.yml | 23 ++++++++++++++++++++++- justfile | 4 ++++ 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 810054d3158..f6795e3c6ec 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,7 +1,3 @@ -[alias] -# Warnings create a lot of noise, we only print errors. -check-clippy = "clippy --no-deps -- --allow warnings" - # Can be safely removed once Cargo's sparse protocol (see # https://blog.rust-lang.org/2023/03/09/Rust-1.68.0.html#cargos-sparse-protocol) # becomes the default. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4116e5cd2c3..4a617592a3d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -238,10 +238,31 @@ jobs: - name: Check formatting run: just format --check + clippy: + name: Clippy linting + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + RUSTFLAGS: "-D warnings" + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + - name: Setup dependencies + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler + + - uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 + + - name: Setup just + uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 + + - name: Run linting + run: just lint + release-check: name: Build in release mode runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 10 env: RUSTFLAGS: "-D warnings" steps: diff --git a/justfile b/justfile index f0fcab284ce..94f3c867bf7 100644 --- a/justfile +++ b/justfile @@ -6,6 +6,10 @@ default: format *EXTRA_FLAGS: cargo fmt --all {{EXTRA_FLAGS}} +# Run Clippy linting (cargo clippy) +lint: + cargo clippy --no-deps -- --allow warnings + # Check Rust code (cargo check) check *EXTRA_FLAGS: cargo check {{EXTRA_FLAGS}} From d564685fc26b096e9137942727780951231a978b Mon Sep 17 00:00:00 2001 From: Sebastian Lorenz Date: Fri, 22 Aug 2025 22:28:43 +0200 Subject: [PATCH 8/8] ci: improve max connection config handling --- .github/workflows/ci.yml | 57 ++++++++++++++-------------------------- 1 file changed, 19 insertions(+), 38 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a617592a3d..a3c85982556 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: POSTGRES_USER: graph POSTGRES_PASSWORD: graph POSTGRES_DB: graph-test - POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C" + POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C -c max_connections=1000 -c shared_buffers=2GB" options: >- --health-cmd "pg_isready -U graph" --health-interval 10s @@ -44,22 +44,15 @@ jobs: ports: - 5432:5432 steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 - - name: Alter postgres max connections - run: | - docker exec -i postgres bash << EOF - sed -i -e 's/max_connections = 100/max_connections = 1000/' /var/lib/postgresql/data/postgresql.conf - sed -i -e 's/shared_buffers = 128MB/shared_buffers = 2GB/' /var/lib/postgresql/data/postgresql.conf - EOF - docker restart --timeout 0 postgres - sleep 5 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 - name: Setup dependencies run: | sudo apt-get update sudo apt-get install -y lld protobuf-compiler - - uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 + - name: Setup rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 - name: Setup just uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 @@ -88,7 +81,7 @@ jobs: POSTGRES_USER: graph POSTGRES_PASSWORD: graph POSTGRES_DB: graph-test - POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C" + POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C -c max_connections=1000 -c shared_buffers=2GB" options: >- --health-cmd "pg_isready -U graph" --health-interval 10s @@ -98,22 +91,15 @@ jobs: ports: - 5432:5432 steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 - - name: Alter postgres max connections - run: | - docker exec -i postgres bash << EOF - sed -i -e 's/max_connections = 100/max_connections = 1000/' /var/lib/postgresql/data/postgresql.conf - sed -i -e 's/shared_buffers = 128MB/shared_buffers = 2GB/' /var/lib/postgresql/data/postgresql.conf - EOF - docker restart --timeout 0 postgres - sleep 5 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 - name: Setup dependencies run: | sudo apt-get update sudo apt-get install -y lld protobuf-compiler - - uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 + - name: Setup rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 - name: Setup just uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 @@ -154,7 +140,7 @@ jobs: POSTGRES_USER: graph-node POSTGRES_PASSWORD: let-me-in POSTGRES_DB: graph-node - POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C" + POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C -c max_connections=1000 -c shared_buffers=2GB" options: >- --health-cmd "pg_isready -U graph-node" --health-interval 10s @@ -164,22 +150,15 @@ jobs: ports: - 3011:5432 steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 - - name: Alter postgres max connections - run: | - docker exec -i postgres bash << EOF - sed -i -e 's/max_connections = 100/max_connections = 1000/' /var/lib/postgresql/data/postgresql.conf - sed -i -e 's/shared_buffers = 128MB/shared_buffers = 2GB/' /var/lib/postgresql/data/postgresql.conf - EOF - docker restart --timeout 0 postgres - sleep 5 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 - name: Setup dependencies run: | sudo apt-get update sudo apt-get install -y lld protobuf-compiler - - uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 + - name: Setup rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 - name: Setup just uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 @@ -229,7 +208,7 @@ jobs: env: RUSTFLAGS: "-D warnings" steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 - uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 - name: Setup just @@ -245,13 +224,14 @@ jobs: env: RUSTFLAGS: "-D warnings" steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 - name: Setup dependencies run: | sudo apt-get update sudo apt-get install -y protobuf-compiler - - uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 + - name: Setup rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 - name: Setup just uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3 @@ -266,13 +246,14 @@ jobs: env: RUSTFLAGS: "-D warnings" steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 - name: Setup dependencies run: | sudo apt-get update sudo apt-get install -y protobuf-compiler - - uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 + - name: Setup rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@fb51252c7ba57d633bc668f941da052e410add48 # v1 - name: Setup just uses: extractions/setup-just@e33e0265a09d6d736e2ee1e0eb685ef1de4669ff # v3