Skip to content

Commit

Permalink
Merge pull request #1760 from o1-labs/feat/speed-up-ci
Browse files Browse the repository at this point in the history 
Refactor GitHub Actions Workflows for Improved CI/CD Performance
  • Loading branch information
@MartinMinkov
MartinMinkov authored Jul 19, 2024
2 parents 046ec18 + 4b9e24d commit 9a1d10c
Show file tree
Hide file tree
Showing 2 changed files with 195 additions and 32 deletions.
218 changes: 195 additions & 23 deletions .github/workflows/build-action.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,53 @@ on:
workflow_dispatch: {}

jobs:
Prepare:
runs-on: ubuntu-latest
outputs:
test_count: ${{ steps.count_tests.outputs.test_count }}
chunk_count: 8 # This is hardcoded to 8, but it can be changed to any number.
steps:
- name: Checkout repository with submodules
uses: actions/checkout@v4
with:
submodules: recursive

- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: '18'

- name: Cache dependencies and build
uses: actions/cache@v4
id: cache
with:
path: |
~/.npm
node_modules
dist
key: ${{ runner.OS }}-node-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.ts', '**/*.js') }}

- name: Build o1js
if: steps.cache.outputs.cache-hit != 'true'
run: |
npm ci
npm run build
- name: Count tests
id: count_tests
run: |
TEST_COUNT=$(find ./dist/node -name "*.unit-test.js" | wc -l)
echo "test_count=${TEST_COUNT}" >> $GITHUB_OUTPUT
echo "Total test count: ${TEST_COUNT}"
- name: Cache repository
uses: actions/cache@v4
with:
path: .
key: repo-${{ github.sha }}

Build-And-Test-Server:
needs: Prepare
timeout-minutes: 210
runs-on: ubuntu-latest
strategy:
Expand All @@ -21,51 +67,167 @@ jobs:
'DEX integration tests',
'DEX integration test with proofs',
'Voting integration tests',
'Unit tests',
'Verification Key Regression Check',
'CommonJS test',
]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Restore repository
uses: actions/cache@v4
with:
path: .
key: repo-${{ github.sha }}

- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: '18'
- name: Build o1js and execute tests

- name: Restore cache
uses: actions/cache@v4
with:
path: |
~/.npm
node_modules
dist
key: ${{ runner.OS }}-node-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.ts', '**/*.js') }}

- name: Prepare for tests
run: touch profiling.md

- name: Execute tests
env:
TEST_TYPE: ${{ matrix.test_type }}
run: sh run-ci-tests.sh

- name: Add to job summary
if: always()
run: |
echo "### Test Results for ${{ matrix.test_type }}" >> $GITHUB_STEP_SUMMARY
cat profiling.md >> $GITHUB_STEP_SUMMARY
Run-Unit-Tests:
needs: Prepare
name: Run unit tests parallel
timeout-minutes: 210
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
chunk: [1, 2, 3, 4, 5, 6, 7, 8]
steps:
- name: Restore repository
uses: actions/cache@v4
with:
path: .
key: repo-${{ github.sha }}

- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: '18'

- name: Restore cache
uses: actions/cache@v4
with:
path: |
~/.npm
node_modules
dist
key: ${{ runner.OS }}-node-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/*.ts', '**/*.js') }}

- name: Prepare for tests
run: touch profiling.md

- name: Run unit tests
env:
TOTAL_TESTS: ${{ needs.Prepare.outputs.test_count }}
CHUNK: ${{ matrix.chunk }}
CHUNKS: 8
run: |
echo "Total tests: $TOTAL_TESTS"
echo "Current chunk: $CHUNK"
echo "Total chunks: $CHUNKS"
if [ -z "$TOTAL_TESTS" ] || [ "$TOTAL_TESTS" -eq 0 ]; then
echo "Error: TOTAL_TESTS is not set or is zero. Exiting."
exit 1
fi
start_index=$(( (TOTAL_TESTS * (CHUNK - 1) / CHUNKS) ))
end_index=$(( (TOTAL_TESTS * CHUNK / CHUNKS) ))
echo "Running tests from index $start_index to $end_index"
shopt -s globstar
test_files=(./dist/node/**/*.unit-test.js)
for ((i=start_index; i<end_index && i<${#test_files[@]}; i++)); do
echo "Running test: ${test_files[$i]}"
node --enable-source-maps "${test_files[$i]}" | tee -a profiling.md
done
continue-on-error: false

- name: Upload test results
if: always()
uses: actions/upload-artifact@v4
with:
name: test-results-${{ matrix.chunk }}
path: profiling.md

- name: Add to job summary
if: always()
run: |
git submodule update --init --recursive
npm ci
npm run build
touch profiling.md
sh run-ci-tests.sh
echo "### Test Results for Unit Tests Chunk ${{ matrix.chunk }}" >> $GITHUB_STEP_SUMMARY
cat profiling.md >> $GITHUB_STEP_SUMMARY
Build-And-Test-Server-Unit-Tests:
name: Build-And-Test-Server (Unit tests)
needs: [Run-Unit-Tests]
runs-on: ubuntu-latest
steps:
- run: echo "All unit tests completed successfully"

Build-And-Test-Web:
needs: Prepare
timeout-minutes: 90
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Restore repository
uses: actions/cache@v4
with:
path: .
key: repo-${{ github.sha }}

- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: '18'
- name: Install Node dependencies
run: |
git submodule update --init --recursive
npm ci

- name: Restore npm cache
uses: actions/cache@v4
with:
path: ~/.npm
key: ${{ runner.OS }}-node-${{ hashFiles('**/package-lock.json') }}

- name: Cache Playwright browsers
uses: actions/cache@v4
id: playwright-cache
with:
path: ~/.cache/ms-playwright
key: ${{ runner.OS }}-playwright-${{ hashFiles('**/package-lock.json') }}

- name: Install Playwright browsers
if: steps.playwright-cache.outputs.cache-hit != 'true'
run: npm run e2e:install

- name: Build o1js and prepare the web server
run: |
npm run build:web
npm run e2e:prepare-server
- name: Execute E2E tests
run: npm run test:e2e

- name: Upload E2E test artifacts
uses: actions/upload-artifact@v4
continue-on-error: true
Expand All @@ -80,19 +242,24 @@ jobs:
if: github.ref == 'refs/heads/main'
timeout-minutes: 180
runs-on: ubuntu-latest
needs: [Build-And-Test-Server, Build-And-Test-Web]
needs: [Build-And-Test-Server, Run-Unit-Tests, Build-And-Test-Web]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Restore repository
uses: actions/cache@v4
with:
path: .
key: repo-${{ github.sha }}

- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: '18'

- name: Build o1js
run: |
git submodule update --init --recursive
npm ci
npm run prepublishOnly
- name: Publish to NPM if version has changed
uses: JS-DevTools/npm-publish@v3
with:
Expand All @@ -105,21 +272,26 @@ jobs:
if: github.ref == 'refs/heads/main'
timeout-minutes: 180
runs-on: ubuntu-latest
needs: [Build-And-Test-Server, Build-And-Test-Web]
needs: [Build-And-Test-Server, Run-Unit-Tests, Build-And-Test-Web]
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Restore repository
uses: actions/cache@v4
with:
path: .
key: repo-${{ github.sha }}

- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: '18'

- name: Build mina-signer
run: |
git submodule update --init --recursive
npm ci
cd src/mina-signer
npm ci
npm run prepublishOnly
- name: Publish to NPM if version has changed
uses: JS-DevTools/npm-publish@v3
with:
Expand Down
9 changes: 0 additions & 9 deletions run-ci-tests.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,15 +34,6 @@ case $TEST_TYPE in
./run src/examples/zkapps/dex/happy-path-with-proofs.ts --bundle
;;

"Unit tests")
echo "Running unit tests"
cd src/mina-signer
npm run build
cd ../..
npm run test:unit
npm run test
;;

"Verification Key Regression Check")
echo "Running Regression checks"
./run ./tests/vk-regression/vk-regression.ts --bundle
Expand Down

0 comments on commit 9a1d10c

Please sign in to comment.