From a3da878618ba63f49dcb39791dc847550284699d Mon Sep 17 00:00:00 2001 From: macpie Date: Tue, 14 May 2024 10:04:51 -0700 Subject: [PATCH 01/21] Revert "Revert "Attempt to make CI faster" (#809)" This reverts commit a6d1430067f27ff239ddb54130cc44d688fe21b3. --- .github/workflows/CI.yml | 55 +++++++++++++ .github/workflows/Tests.yml | 157 ++++++++++++++++++++++++++++++++++++ .github/workflows/rust.yml | 71 ---------------- 3 files changed, 212 insertions(+), 71 deletions(-) create mode 100644 .github/workflows/CI.yml create mode 100644 .github/workflows/Tests.yml delete mode 100644 .github/workflows/rust.yml diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 000000000..4064d2f0e --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,55 @@ +name: CI + +on: + workflow_run: + workflows: [Tests] + types: [completed] + +env: + CARGO_INCREMENTAL: 0 # this setting is automatically applied by rust-cache but documented here for explicitness + CARGO_NET_RETRY: 10 + RUST_BACKTRACE: short + RUSTFLAGS: "-D warnings" + RUSTUP_MAX_RETRIES: 10 + +jobs: + on-failure: + runs-on: oracles-20.04 + if: ${{ github.event.workflow_run.conclusion == 'failure' }} + steps: + - run: echo 'The triggering workflow failed' + on-success: + runs-on: oracles-20.04 + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-build-release + cancel-in-progress: true + if: ${{ github.event.workflow_run.conclusion == 'success' }} && contains(github.ref, 'refs/tags/') + steps: + - uses: actions/checkout@v3 + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy, rustfmt + + - name: Install protoc + run: sudo apt-get install -y protobuf-compiler + + - name: Cache + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Build Release + run: cargo build --all --release + + - name: Debian packaging + env: + PACKAGECLOUD_API_KEY: ${{ secrets.PACKAGECLOUD_API_KEY }} + run: | + chmod +x ./.github/scripts/make_debian.sh + ./.github/scripts/make_debian.sh diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml new file mode 100644 index 000000000..53fb79e4f --- /dev/null +++ b/.github/workflows/Tests.yml @@ -0,0 +1,157 @@ +name: Tests + +on: + pull_request: + branches: [main] + push: + branches: ["main"] + tags: ["*"] + +env: + CARGO_INCREMENTAL: 0 # this setting is automatically applied by rust-cache but documented here for explicitness + CARGO_NET_RETRY: 10 + RUST_BACKTRACE: short + RUSTFLAGS: "-D warnings" + RUSTUP_MAX_RETRIES: 10 + +jobs: + + build: + runs-on: oracles-20.04 + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-build + cancel-in-progress: true + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Rust install + uses: dtolnay/rust-toolchain@stable + + - name: Install protoc + run: sudo apt-get install -y protobuf-compiler + + - name: Cache + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Build + run: cargo build --all --tests + + fmt: + needs: build + runs-on: oracles-20.04 + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-fmt + cancel-in-progress: true + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Rust install + uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt + + - name: Install protoc + run: sudo apt-get install -y protobuf-compiler + + - name: Cache + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Check formatting + run: cargo fmt -- --check + clippy: + needs: build + runs-on: oracles-20.04 + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-clippy + cancel-in-progress: true + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Rust install + uses: dtolnay/rust-toolchain@stable + with: + components: clippy + + - name: Install protoc + run: sudo apt-get install -y protobuf-compiler + + - name: Cache + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Clippy + run: cargo clippy --all-targets -- -Dclippy::all -D warnings + + tests: + needs: build + runs-on: oracles-20.04 + strategy: + fail-fast: false + matrix: + package: [boost-manager,file-store,ingest,iot-config,iot-packet-verifier,iot-verifier,mobile-config,mobile-packet-verifier,mobile-verifier,reward-scheduler,task-manager] + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-tests-${{ matrix.package }} + cancel-in-progress: true + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Rust install + uses: dtolnay/rust-toolchain@stable + + - name: Install protoc + run: sudo apt-get install -y protobuf-compiler + + - name: Cache + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Run unit and integration tests + env: + DATABASE_URL: "postgres://postgres:postgres@localhost:5432/postgres" + run: cargo test -p ${{ matrix.package }} -- --include-ignored \ No newline at end of file diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml deleted file mode 100644 index c141ecc9e..000000000 --- a/.github/workflows/rust.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: CI - -on: - pull_request: - branches: [main] - push: - branches: [main] - tags: ["*"] - -env: - CARGO_INCREMENTAL: 0 # this setting is automatically applied by rust-cache but documented here for explicitness - CARGO_NET_RETRY: 10 - RUST_BACKTRACE: short - RUSTFLAGS: "-D warnings" - RUSTUP_MAX_RETRIES: 10 - -jobs: - build: - runs-on: oracles-20.04 - - services: - postgres: - image: postgres - env: - POSTGRES_PASSWORD: postgres - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - - steps: - - uses: actions/checkout@v3 - - uses: dtolnay/rust-toolchain@stable - with: - components: clippy, rustfmt - - - name: Install protoc - run: sudo apt-get install -y protobuf-compiler - - - name: Setup cache - uses: Swatinem/rust-cache@v2 - - - name: Cancel previous runs - uses: styfle/cancel-workflow-action@0.11.0 - with: - access_token: ${{ github.token }} - - - name: Build - run: cargo build --all --release - - - name: Check formatting - run: cargo fmt -- --check - - - name: Clippy - run: cargo clippy --all-targets -- -Dclippy::all -D warnings - - - name: Run unit and integration tests - env: - DATABASE_URL: "postgres://postgres:postgres@localhost:5432/postgres" - run: cargo test -r -- --include-ignored - - - name: Debian packaging - if: contains(github.ref, 'refs/tags/') - env: - PACKAGECLOUD_API_KEY: ${{ secrets.PACKAGECLOUD_API_KEY }} - run: | - chmod +x ./.github/scripts/make_debian.sh - ./.github/scripts/make_debian.sh From 1a45acc677bf9dfb3c09f3ec772ec92d145c27fe Mon Sep 17 00:00:00 2001 From: Macpie Date: Tue, 14 May 2024 10:10:47 -0700 Subject: [PATCH 02/21] Test Debian packaging without tag --- .github/workflows/CI.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 4064d2f0e..96c06963f 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -15,15 +15,16 @@ env: jobs: on-failure: runs-on: oracles-20.04 - if: ${{ github.event.workflow_run.conclusion == 'failure' }} + if: ${{ github.event.workflow_run.conclusion == 'failure' }} steps: - run: echo 'The triggering workflow failed' on-success: runs-on: oracles-20.04 - concurrency: + concurrency: group: ${{ github.workflow }}-${{ github.ref }}-build-release cancel-in-progress: true - if: ${{ github.event.workflow_run.conclusion == 'success' }} && contains(github.ref, 'refs/tags/') + # if: ${{ github.event.workflow_run.conclusion == 'success' }} && contains(github.ref, 'refs/tags/') + if: ${{ github.event.workflow_run.conclusion == 'success' }} steps: - uses: actions/checkout@v3 - uses: dtolnay/rust-toolchain@stable @@ -43,13 +44,14 @@ jobs: ~/.cargo/git/db/ target/ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - + - name: Build Release - run: cargo build --all --release + run: cargo build --all --release - name: Debian packaging env: - PACKAGECLOUD_API_KEY: ${{ secrets.PACKAGECLOUD_API_KEY }} + # PACKAGECLOUD_API_KEY: ${{ secrets.PACKAGECLOUD_API_KEY }} + PACKAGECLOUD_API_KEY: XXX run: | chmod +x ./.github/scripts/make_debian.sh ./.github/scripts/make_debian.sh From 81e4fbcb3228b0636ee1901451bccbf9b1bd58fb Mon Sep 17 00:00:00 2001 From: Macpie Date: Tue, 14 May 2024 10:28:28 -0700 Subject: [PATCH 03/21] Remove comments --- .github/workflows/CI.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 96c06963f..bc7cdc332 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -23,7 +23,6 @@ jobs: concurrency: group: ${{ github.workflow }}-${{ github.ref }}-build-release cancel-in-progress: true - # if: ${{ github.event.workflow_run.conclusion == 'success' }} && contains(github.ref, 'refs/tags/') if: ${{ github.event.workflow_run.conclusion == 'success' }} steps: - uses: actions/checkout@v3 @@ -50,7 +49,6 @@ jobs: - name: Debian packaging env: - # PACKAGECLOUD_API_KEY: ${{ secrets.PACKAGECLOUD_API_KEY }} PACKAGECLOUD_API_KEY: XXX run: | chmod +x ./.github/scripts/make_debian.sh From e0d3949330b3dc11522df755bb4c6389ab4ba1ef Mon Sep 17 00:00:00 2001 From: Macpie Date: Tue, 14 May 2024 10:38:37 -0700 Subject: [PATCH 04/21] Rename to try to trigger --- .github/workflows/{CI.yml => Package.yml} | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) rename .github/workflows/{CI.yml => Package.yml} (86%) diff --git a/.github/workflows/CI.yml b/.github/workflows/Package.yml similarity index 86% rename from .github/workflows/CI.yml rename to .github/workflows/Package.yml index bc7cdc332..725884984 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/Package.yml @@ -1,4 +1,4 @@ -name: CI +name: Package on: workflow_run: @@ -23,12 +23,11 @@ jobs: concurrency: group: ${{ github.workflow }}-${{ github.ref }}-build-release cancel-in-progress: true + # if: ${{ github.event.workflow_run.conclusion == 'success' }} && contains(github.ref, 'refs/tags/') if: ${{ github.event.workflow_run.conclusion == 'success' }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - with: - components: clippy, rustfmt - name: Install protoc run: sudo apt-get install -y protobuf-compiler @@ -49,6 +48,7 @@ jobs: - name: Debian packaging env: + # PACKAGECLOUD_API_KEY: ${{ secrets.PACKAGECLOUD_API_KEY }} PACKAGECLOUD_API_KEY: XXX run: | chmod +x ./.github/scripts/make_debian.sh From 3eed7a2690765a0d49101e70497d7778941aaa17 Mon Sep 17 00:00:00 2001 From: Macpie Date: Tue, 14 May 2024 10:38:58 -0700 Subject: [PATCH 05/21] Comment matrix for now --- .github/workflows/Tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index 53fb79e4f..7c477e644 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -114,7 +114,8 @@ jobs: strategy: fail-fast: false matrix: - package: [boost-manager,file-store,ingest,iot-config,iot-packet-verifier,iot-verifier,mobile-config,mobile-packet-verifier,mobile-verifier,reward-scheduler,task-manager] + # package: [boost-manager,file-store,ingest,iot-config,iot-packet-verifier,iot-verifier,mobile-config,mobile-packet-verifier,mobile-verifier,reward-scheduler,task-manager] + package: [] concurrency: group: ${{ github.workflow }}-${{ github.ref }}-tests-${{ matrix.package }} cancel-in-progress: true From 8a91643dae6503ea2a36f87a3eeb33e625e5f561 Mon Sep 17 00:00:00 2001 From: Macpie Date: Tue, 14 May 2024 10:40:41 -0700 Subject: [PATCH 06/21] Empty matrix does not work comment the all thing --- .github/workflows/Tests.yml | 95 ++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 48 deletions(-) diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index 7c477e644..a58fdb3f7 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -108,51 +108,50 @@ jobs: - name: Clippy run: cargo clippy --all-targets -- -Dclippy::all -D warnings - tests: - needs: build - runs-on: oracles-20.04 - strategy: - fail-fast: false - matrix: - # package: [boost-manager,file-store,ingest,iot-config,iot-packet-verifier,iot-verifier,mobile-config,mobile-packet-verifier,mobile-verifier,reward-scheduler,task-manager] - package: [] - concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-tests-${{ matrix.package }} - cancel-in-progress: true - services: - postgres: - image: postgres - env: - POSTGRES_PASSWORD: postgres - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Rust install - uses: dtolnay/rust-toolchain@stable - - - name: Install protoc - run: sudo apt-get install -y protobuf-compiler - - - name: Cache - uses: actions/cache@v4 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - - name: Run unit and integration tests - env: - DATABASE_URL: "postgres://postgres:postgres@localhost:5432/postgres" - run: cargo test -p ${{ matrix.package }} -- --include-ignored \ No newline at end of file + # tests: + # needs: build + # runs-on: oracles-20.04 + # strategy: + # fail-fast: false + # matrix: + # package: [boost-manager,file-store,ingest,iot-config,iot-packet-verifier,iot-verifier,mobile-config,mobile-packet-verifier,mobile-verifier,reward-scheduler,task-manager] + # concurrency: + # group: ${{ github.workflow }}-${{ github.ref }}-tests-${{ matrix.package }} + # cancel-in-progress: true + # services: + # postgres: + # image: postgres + # env: + # POSTGRES_PASSWORD: postgres + # options: >- + # --health-cmd pg_isready + # --health-interval 10s + # --health-timeout 5s + # --health-retries 5 + # ports: + # - 5432:5432 + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + + # - name: Rust install + # uses: dtolnay/rust-toolchain@stable + + # - name: Install protoc + # run: sudo apt-get install -y protobuf-compiler + + # - name: Cache + # uses: actions/cache@v4 + # with: + # path: | + # ~/.cargo/bin/ + # ~/.cargo/registry/index/ + # ~/.cargo/registry/cache/ + # ~/.cargo/git/db/ + # target/ + # key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + # - name: Run unit and integration tests + # env: + # DATABASE_URL: "postgres://postgres:postgres@localhost:5432/postgres" + # run: cargo test -p ${{ matrix.package }} -- --include-ignored \ No newline at end of file From 3e8456aeed1eafd459cac9150bb553494162a6f9 Mon Sep 17 00:00:00 2001 From: Macpie Date: Tue, 14 May 2024 10:49:37 -0700 Subject: [PATCH 07/21] Change trigger --- .github/workflows/Package.yml | 16 ++++-- .github/workflows/Tests.yml | 94 +++++++++++++++++------------------ 2 files changed, 58 insertions(+), 52 deletions(-) diff --git a/.github/workflows/Package.yml b/.github/workflows/Package.yml index 725884984..d2afd4bcf 100644 --- a/.github/workflows/Package.yml +++ b/.github/workflows/Package.yml @@ -1,9 +1,16 @@ name: Package +# on: +# workflow_run: +# workflows: [Tests] +# types: [completed] + on: - workflow_run: - workflows: [Tests] - types: [completed] + pull_request: + branches: [main] + push: + branches: ["main"] + tags: ["*"] env: CARGO_INCREMENTAL: 0 # this setting is automatically applied by rust-cache but documented here for explicitness @@ -15,7 +22,7 @@ env: jobs: on-failure: runs-on: oracles-20.04 - if: ${{ github.event.workflow_run.conclusion == 'failure' }} + # if: ${{ github.event.workflow_run.conclusion == 'failure' }} steps: - run: echo 'The triggering workflow failed' on-success: @@ -24,7 +31,6 @@ jobs: group: ${{ github.workflow }}-${{ github.ref }}-build-release cancel-in-progress: true # if: ${{ github.event.workflow_run.conclusion == 'success' }} && contains(github.ref, 'refs/tags/') - if: ${{ github.event.workflow_run.conclusion == 'success' }} steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index a58fdb3f7..53fb79e4f 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -108,50 +108,50 @@ jobs: - name: Clippy run: cargo clippy --all-targets -- -Dclippy::all -D warnings - # tests: - # needs: build - # runs-on: oracles-20.04 - # strategy: - # fail-fast: false - # matrix: - # package: [boost-manager,file-store,ingest,iot-config,iot-packet-verifier,iot-verifier,mobile-config,mobile-packet-verifier,mobile-verifier,reward-scheduler,task-manager] - # concurrency: - # group: ${{ github.workflow }}-${{ github.ref }}-tests-${{ matrix.package }} - # cancel-in-progress: true - # services: - # postgres: - # image: postgres - # env: - # POSTGRES_PASSWORD: postgres - # options: >- - # --health-cmd pg_isready - # --health-interval 10s - # --health-timeout 5s - # --health-retries 5 - # ports: - # - 5432:5432 - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - - # - name: Rust install - # uses: dtolnay/rust-toolchain@stable - - # - name: Install protoc - # run: sudo apt-get install -y protobuf-compiler - - # - name: Cache - # uses: actions/cache@v4 - # with: - # path: | - # ~/.cargo/bin/ - # ~/.cargo/registry/index/ - # ~/.cargo/registry/cache/ - # ~/.cargo/git/db/ - # target/ - # key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - # - name: Run unit and integration tests - # env: - # DATABASE_URL: "postgres://postgres:postgres@localhost:5432/postgres" - # run: cargo test -p ${{ matrix.package }} -- --include-ignored \ No newline at end of file + tests: + needs: build + runs-on: oracles-20.04 + strategy: + fail-fast: false + matrix: + package: [boost-manager,file-store,ingest,iot-config,iot-packet-verifier,iot-verifier,mobile-config,mobile-packet-verifier,mobile-verifier,reward-scheduler,task-manager] + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-tests-${{ matrix.package }} + cancel-in-progress: true + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Rust install + uses: dtolnay/rust-toolchain@stable + + - name: Install protoc + run: sudo apt-get install -y protobuf-compiler + + - name: Cache + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Run unit and integration tests + env: + DATABASE_URL: "postgres://postgres:postgres@localhost:5432/postgres" + run: cargo test -p ${{ matrix.package }} -- --include-ignored \ No newline at end of file From 4eac50fda480d9f8032f584886bd4f702cd27b8e Mon Sep 17 00:00:00 2001 From: Macpie Date: Tue, 14 May 2024 11:11:25 -0700 Subject: [PATCH 08/21] Add some log line --- .github/scripts/make_debian.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/scripts/make_debian.sh b/.github/scripts/make_debian.sh index 10603f900..617988ad6 100644 --- a/.github/scripts/make_debian.sh +++ b/.github/scripts/make_debian.sh @@ -99,14 +99,16 @@ run_fpm() sudo apt update sudo apt install --yes ruby sudo gem install fpm -v 1.14.2 # current as of 2022-11-08 +echo "ruby deps installed" for config_path in $( find . -name 'settings-template.toml' ) do oracle=$(echo $config_path | sed -E 's!\./([^/]+)/.+$!\1!' | sed -E 's!_!-!g') - + write_unit_template $oracle write_prepost_template $oracle run_fpm $oracle $config_path $VERSION + echo "$oracle done" done for deb in /tmp/*.deb From 77a8b3b2c7c6b2c621ea8a46962559fca3452da4 Mon Sep 17 00:00:00 2001 From: Macpie Date: Tue, 14 May 2024 11:27:57 -0700 Subject: [PATCH 09/21] More logs --- .github/scripts/make_debian.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/scripts/make_debian.sh b/.github/scripts/make_debian.sh index 617988ad6..1cffe127b 100644 --- a/.github/scripts/make_debian.sh +++ b/.github/scripts/make_debian.sh @@ -105,10 +105,13 @@ for config_path in $( find . -name 'settings-template.toml' ) do oracle=$(echo $config_path | sed -E 's!\./([^/]+)/.+$!\1!' | sed -E 's!_!-!g') + echo "starting $oracle $config_path $VERSION" write_unit_template $oracle + echo "write_unit_template $oracle done" write_prepost_template $oracle + echo "write_prepost_template $oracle done" run_fpm $oracle $config_path $VERSION - echo "$oracle done" + echo "run_fpm $oracle done" done for deb in /tmp/*.deb From 3957057875b84fb3a7fa637e111f1010528f3c15 Mon Sep 17 00:00:00 2001 From: Macpie Date: Tue, 14 May 2024 11:50:17 -0700 Subject: [PATCH 10/21] Add random version --- .github/scripts/make_debian.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/make_debian.sh b/.github/scripts/make_debian.sh index 1cffe127b..c3214d5ac 100644 --- a/.github/scripts/make_debian.sh +++ b/.github/scripts/make_debian.sh @@ -110,7 +110,7 @@ do echo "write_unit_template $oracle done" write_prepost_template $oracle echo "write_prepost_template $oracle done" - run_fpm $oracle $config_path $VERSION + run_fpm $oracle $config_path 2.16.2-666-6 echo "run_fpm $oracle done" done From 6a105858020540d15e7e8b47edb4f6e026e2da84 Mon Sep 17 00:00:00 2001 From: Macpie Date: Tue, 14 May 2024 12:22:41 -0700 Subject: [PATCH 11/21] Revert comments --- .github/workflows/Package.yml | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/.github/workflows/Package.yml b/.github/workflows/Package.yml index d2afd4bcf..1cd7639c8 100644 --- a/.github/workflows/Package.yml +++ b/.github/workflows/Package.yml @@ -1,16 +1,9 @@ name: Package -# on: -# workflow_run: -# workflows: [Tests] -# types: [completed] - on: - pull_request: - branches: [main] - push: - branches: ["main"] - tags: ["*"] + workflow_run: + workflows: [Tests] + types: [completed] env: CARGO_INCREMENTAL: 0 # this setting is automatically applied by rust-cache but documented here for explicitness @@ -22,7 +15,7 @@ env: jobs: on-failure: runs-on: oracles-20.04 - # if: ${{ github.event.workflow_run.conclusion == 'failure' }} + if: ${{ github.event.workflow_run.conclusion == 'failure' }} steps: - run: echo 'The triggering workflow failed' on-success: @@ -30,9 +23,10 @@ jobs: concurrency: group: ${{ github.workflow }}-${{ github.ref }}-build-release cancel-in-progress: true - # if: ${{ github.event.workflow_run.conclusion == 'success' }} && contains(github.ref, 'refs/tags/') + if: ${{ github.event.workflow_run.conclusion == 'success' }} && contains(github.ref, 'refs/tags/') steps: - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable - name: Install protoc @@ -54,8 +48,7 @@ jobs: - name: Debian packaging env: - # PACKAGECLOUD_API_KEY: ${{ secrets.PACKAGECLOUD_API_KEY }} - PACKAGECLOUD_API_KEY: XXX + PACKAGECLOUD_API_KEY: ${{ secrets.PACKAGECLOUD_API_KEY }} run: | chmod +x ./.github/scripts/make_debian.sh ./.github/scripts/make_debian.sh From 736fe95cdd7f27e72d8715ae2bb5281d54ec04ab Mon Sep 17 00:00:00 2001 From: Macpie Date: Tue, 14 May 2024 12:24:33 -0700 Subject: [PATCH 12/21] Get back to using $VERSION --- .github/scripts/make_debian.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/make_debian.sh b/.github/scripts/make_debian.sh index c3214d5ac..1cffe127b 100644 --- a/.github/scripts/make_debian.sh +++ b/.github/scripts/make_debian.sh @@ -110,7 +110,7 @@ do echo "write_unit_template $oracle done" write_prepost_template $oracle echo "write_prepost_template $oracle done" - run_fpm $oracle $config_path 2.16.2-666-6 + run_fpm $oracle $config_path $VERSION echo "run_fpm $oracle done" done From 64ef69d386003fd3e315c8fe8aa3ee01b2be503f Mon Sep 17 00:00:00 2001 From: Macpie Date: Tue, 14 May 2024 12:44:32 -0700 Subject: [PATCH 13/21] Fix package --- .github/workflows/Package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/Package.yml b/.github/workflows/Package.yml index 1cd7639c8..bca7d03ad 100644 --- a/.github/workflows/Package.yml +++ b/.github/workflows/Package.yml @@ -2,7 +2,7 @@ name: Package on: workflow_run: - workflows: [Tests] + workflows: ["Tests"] types: [completed] env: From 27a6506e7a4689e7ea137e7099b9130d219d0b4a Mon Sep 17 00:00:00 2001 From: Macpie Date: Tue, 14 May 2024 13:09:53 -0700 Subject: [PATCH 14/21] Add build-release to main CI --- .github/workflows/Tests.yml | 38 ++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index 53fb79e4f..3b2dea398 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -154,4 +154,40 @@ jobs: - name: Run unit and integration tests env: DATABASE_URL: "postgres://postgres:postgres@localhost:5432/postgres" - run: cargo test -p ${{ matrix.package }} -- --include-ignored \ No newline at end of file + run: cargo test -p ${{ matrix.package }} -- --include-ignored + + build-release: + needs: [fmt, clippy, tests] + runs-on: oracles-20.04 + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-build-release + cancel-in-progress: true + if: contains(github.ref, 'refs/tags/') + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + + - name: Install protoc + run: sudo apt-get install -y protobuf-compiler + + - name: Cache + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Build Release + run: cargo build --all --release + + - name: Debian packaging + env: + PACKAGECLOUD_API_KEY: ${{ secrets.PACKAGECLOUD_API_KEY }} + run: | + chmod +x ./.github/scripts/make_debian.sh + ./.github/scripts/make_debian.sh \ No newline at end of file From 938a916ea3057b7b161ad991e3de13b09cfca6f5 Mon Sep 17 00:00:00 2001 From: Macpie Date: Tue, 14 May 2024 13:40:49 -0700 Subject: [PATCH 15/21] Make it into 1 workflow --- .github/workflows/{Tests.yml => CI.yml} | 2 +- .github/workflows/Package.yml | 54 ------------------------- 2 files changed, 1 insertion(+), 55 deletions(-) rename .github/workflows/{Tests.yml => CI.yml} (99%) delete mode 100644 .github/workflows/Package.yml diff --git a/.github/workflows/Tests.yml b/.github/workflows/CI.yml similarity index 99% rename from .github/workflows/Tests.yml rename to .github/workflows/CI.yml index 3b2dea398..0d177663f 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/CI.yml @@ -1,4 +1,4 @@ -name: Tests +name: CI on: pull_request: diff --git a/.github/workflows/Package.yml b/.github/workflows/Package.yml deleted file mode 100644 index bca7d03ad..000000000 --- a/.github/workflows/Package.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Package - -on: - workflow_run: - workflows: ["Tests"] - types: [completed] - -env: - CARGO_INCREMENTAL: 0 # this setting is automatically applied by rust-cache but documented here for explicitness - CARGO_NET_RETRY: 10 - RUST_BACKTRACE: short - RUSTFLAGS: "-D warnings" - RUSTUP_MAX_RETRIES: 10 - -jobs: - on-failure: - runs-on: oracles-20.04 - if: ${{ github.event.workflow_run.conclusion == 'failure' }} - steps: - - run: echo 'The triggering workflow failed' - on-success: - runs-on: oracles-20.04 - concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-build-release - cancel-in-progress: true - if: ${{ github.event.workflow_run.conclusion == 'success' }} && contains(github.ref, 'refs/tags/') - steps: - - uses: actions/checkout@v4 - - - uses: dtolnay/rust-toolchain@stable - - - name: Install protoc - run: sudo apt-get install -y protobuf-compiler - - - name: Cache - uses: actions/cache@v4 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - - name: Build Release - run: cargo build --all --release - - - name: Debian packaging - env: - PACKAGECLOUD_API_KEY: ${{ secrets.PACKAGECLOUD_API_KEY }} - run: | - chmod +x ./.github/scripts/make_debian.sh - ./.github/scripts/make_debian.sh From aec43bfb35f7bbb4264abf2270daff048a916b0e Mon Sep 17 00:00:00 2001 From: Macpie Date: Wed, 15 May 2024 14:47:37 -0700 Subject: [PATCH 16/21] Separate test with postgres --- .github/workflows/CI.yml | 44 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 0d177663f..de82d56e4 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -108,15 +108,15 @@ jobs: - name: Clippy run: cargo clippy --all-targets -- -Dclippy::all -D warnings - tests: + tests-postgres: needs: build runs-on: oracles-20.04 strategy: fail-fast: false matrix: - package: [boost-manager,file-store,ingest,iot-config,iot-packet-verifier,iot-verifier,mobile-config,mobile-packet-verifier,mobile-verifier,reward-scheduler,task-manager] + package: [boost-manager,iot-config,iot-packet-verifier,iot-verifier,mobile-config,mobile-verifier] concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-tests-${{ matrix.package }} + group: ${{ github.workflow }}-${{ github.ref }}-tests-postgres-${{ matrix.package }} cancel-in-progress: true services: postgres: @@ -156,8 +156,44 @@ jobs: DATABASE_URL: "postgres://postgres:postgres@localhost:5432/postgres" run: cargo test -p ${{ matrix.package }} -- --include-ignored + tests: + needs: build + runs-on: oracles-20.04 + strategy: + fail-fast: false + matrix: + package: [file-store,ingest,mobile-packet-verifier,reward-scheduler,task-manager] + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-tests-${{ matrix.package }} + cancel-in-progress: true + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Rust install + uses: dtolnay/rust-toolchain@stable + + - name: Install protoc + run: sudo apt-get install -y protobuf-compiler + + - name: Cache + uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Run unit and integration tests + env: + DATABASE_URL: "postgres://postgres:postgres@localhost:5432/postgres" + run: cargo test -p ${{ matrix.package }} -- --include-ignored + build-release: - needs: [fmt, clippy, tests] + needs: [fmt, clippy, tests, tests-postgres] runs-on: oracles-20.04 concurrency: group: ${{ github.workflow }}-${{ github.ref }}-build-release From 33c540e3816f76dbd1e4d2e8768ed79b0f78ca93 Mon Sep 17 00:00:00 2001 From: Macpie Date: Wed, 15 May 2024 14:49:16 -0700 Subject: [PATCH 17/21] remove env in tests --- .github/workflows/CI.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index de82d56e4..2e3680a6c 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -188,8 +188,6 @@ jobs: key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Run unit and integration tests - env: - DATABASE_URL: "postgres://postgres:postgres@localhost:5432/postgres" run: cargo test -p ${{ matrix.package }} -- --include-ignored build-release: From 7d9b738374e68795039622fb49288c98c50633a9 Mon Sep 17 00:00:00 2001 From: Macpie Date: Thu, 16 May 2024 10:00:10 -0700 Subject: [PATCH 18/21] Experiment --- .github/workflows/CI.yml | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 2e3680a6c..30dc48adf 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -46,7 +46,6 @@ jobs: run: cargo build --all --tests fmt: - needs: build runs-on: oracles-20.04 concurrency: group: ${{ github.workflow }}-${{ github.ref }}-fmt @@ -60,22 +59,26 @@ jobs: with: components: rustfmt - - name: Install protoc - run: sudo apt-get install -y protobuf-compiler + - name: Check formatting + run: cargo fmt -- --check - - name: Cache - uses: actions/cache@v4 + clippy2: + runs-on: oracles-20.04 + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-clippy2 + cancel-in-progress: true + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Rust install + uses: dtolnay/rust-toolchain@stable with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + components: clippy + + - name: Clippy + run: cargo clippy --all-targets -- -Dclippy::all -D warnings - - name: Check formatting - run: cargo fmt -- --check clippy: needs: build runs-on: oracles-20.04 From 7c849d41b1476995a0c010afd1d83fd35f348c68 Mon Sep 17 00:00:00 2001 From: Macpie Date: Thu, 16 May 2024 14:36:05 -0700 Subject: [PATCH 19/21] Update readme --- README.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5d741c250..eae2d542e 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ flowchart TD MPV("`**Mobile Packet Verifier** - Burns DC for data transfer (on solana) `") - MB("`**Mobile Price** + MP("`**Mobile Price** - Records Pyth price for MOBILE `") DB1[(Foundation owned db populated by helius)] @@ -31,12 +31,15 @@ flowchart TD - Writes rewards to foundation db `") DB2[(Foundation owned db that stores reward totals)] + S[(Solana)] MI -- S3 --> MV MI -- S3 --> MPV MPV -- S3 --> MV - MB -- S3 --> MV - DB1 --> MB - MB --> MC + MPV -- gRPC --> MC + MPV --> S + MP <--> S + MP -- S3 --> MV + DB1 --> MC MC -- gRPC --> MV MV -- S3 --> MRI MRI --> DB2 From a29adde86750177cb069aaaee50ce47c8df87ccf Mon Sep 17 00:00:00 2001 From: Macpie Date: Thu, 16 May 2024 14:36:38 -0700 Subject: [PATCH 20/21] Add protoc to clippy2 --- .github/workflows/CI.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 30dc48adf..2be3a125d 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -75,6 +75,9 @@ jobs: uses: dtolnay/rust-toolchain@stable with: components: clippy + + - name: Install protoc + run: sudo apt-get install -y protobuf-compiler - name: Clippy run: cargo clippy --all-targets -- -Dclippy::all -D warnings From 43d8d8bf17a2e3055a347c49306bad8d26b79a80 Mon Sep 17 00:00:00 2001 From: Macpie Date: Thu, 16 May 2024 14:59:34 -0700 Subject: [PATCH 21/21] Move clippy --- .github/workflows/CI.yml | 34 +--------------------------------- 1 file changed, 1 insertion(+), 33 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 2be3a125d..c6258f443 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -62,28 +62,7 @@ jobs: - name: Check formatting run: cargo fmt -- --check - clippy2: - runs-on: oracles-20.04 - concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-clippy2 - cancel-in-progress: true - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Rust install - uses: dtolnay/rust-toolchain@stable - with: - components: clippy - - - name: Install protoc - run: sudo apt-get install -y protobuf-compiler - - - name: Clippy - run: cargo clippy --all-targets -- -Dclippy::all -D warnings - clippy: - needs: build runs-on: oracles-20.04 concurrency: group: ${{ github.workflow }}-${{ github.ref }}-clippy @@ -96,21 +75,10 @@ jobs: uses: dtolnay/rust-toolchain@stable with: components: clippy - + - name: Install protoc run: sudo apt-get install -y protobuf-compiler - - name: Cache - uses: actions/cache@v4 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - name: Clippy run: cargo clippy --all-targets -- -Dclippy::all -D warnings