Schedule based benchmark of pallet-contracts and pallet-evm #110
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Schedule based benchmark of pallet-contracts and pallet-evm | |
| on: | |
| schedule: | |
| - cron: '0 0 * * *' | |
| workflow_dispatch: | |
| jobs: | |
| build_dev_moonbeam: | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash | |
| container: | |
| image: paritytech/ci-unified:bullseye-1.73.0 | |
| steps: | |
| - name: Download Moonbeam Release | |
| run: | | |
| API_URL="https://api.github.com/repos/moonbeam-foundation/moonbeam/releases/latest" | |
| RESPONSE=$(curl -s "${API_URL}") | |
| # Get the download URL of the release binary from Repo | |
| DOWNLOAD_URL=$(echo $RESPONSE | jq -r '.assets | map(select(.name == "moonbeam.gz")) | .[0].browser_download_url') | |
| RELEASE_TAG=$(echo $RESPONSE | jq -r '.tag_name') | |
| mkdir -p bin | |
| curl -L ${DOWNLOAD_URL} | tar -xzv -C bin && chmod +x bin/moonbeam | |
| echo "tag=$(echo ${RELEASE_TAG})" >> $GITHUB_OUTPUT | |
| smart_contract_benchmark: | |
| strategy: | |
| matrix: | |
| type: [ink-wasm, sol-wasm, evm] | |
| contract: [erc20] | |
| env: | |
| BENCHMARK_FILE: benchmark-${{ matrix.type }}.csv | |
| BENCHMARK_PATH: results | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Cache | |
| uses: Swatinem/rust-cache@v2 | |
| - name: Download Moonbeam Release | |
| id: moonbeam_release | |
| run: | | |
| API_URL="https://api.github.com/repos/karolk91/moonbeam/releases/latest" | |
| RESPONSE=$(curl -s "${API_URL}") | |
| # Get the download URL of the release binary from Repo | |
| DOWNLOAD_URL=$(echo $RESPONSE | jq -r '.assets | map(select(.name == "moonbeam.gz")) | .[0].browser_download_url') | |
| RELEASE_TAG=$(echo $RESPONSE | jq -r '.tag_name') | |
| mkdir -p bin | |
| curl -L ${DOWNLOAD_URL} | tar -xzv -C bin && chmod +x bin/moonbeam | |
| echo "tag=$(echo ${RELEASE_TAG})" >> $GITHUB_OUTPUT | |
| - name: Download Polkadot-Parachain Release | |
| id: polkadot_parachain_release | |
| run: | | |
| API_URL="https://api.github.com/repos/paritytech/cumulus/releases/latest" | |
| RESPONSE=$(curl -s "${API_URL}") | |
| # Get the download URL of the release binary from Repo | |
| DOWNLOAD_URL=$(echo $RESPONSE | jq -r '.assets | map(select(.name == "polkadot-parachain")) | .[0].browser_download_url') | |
| RELEASE_TAG=$(echo $RESPONSE | jq -r '.tag_name') | |
| mkdir -p bin | |
| curl -L -o bin/polkadot-parachain ${DOWNLOAD_URL} && chmod +x bin/polkadot-parachain | |
| echo "tag=$(echo ${RELEASE_TAG})" >> $GITHUB_OUTPUT | |
| - name: Get Ink contract language verision | |
| id: ink_version | |
| uses: ./.github/actions/get-contract-language | |
| with: | |
| contracts-directory: './contracts/ink' | |
| - name: Get Solang contract language verision | |
| id: solang_version | |
| uses: ./.github/actions/get-contract-language | |
| with: | |
| contracts-directory: './contracts/solidity/wasm' | |
| - name: Get Solc contract language verision | |
| id: solc_version | |
| run: | | |
| language="" | |
| # Iterate over each contract in the directory | |
| for contract in ./contracts/solidity/evm/contracts/*/*.dbg.json; do | |
| build_info=$(jq -r '.buildInfo' "$contract") | |
| build_info_path="./contracts/solidity/evm/contracts/${build_info#../}" | |
| solc_version=$(jq -r '.solcVersion' "$build_info_path") | |
| current_language="solc $solc_version" | |
| # Check if the current language is different from the previous one | |
| if [ -n "$language" ] && [ "$current_language" != "$language" ]; then | |
| echo "Error: Different language detected in contract $contract: $language != $current_language" | |
| exit 1 | |
| fi | |
| language="$current_language" | |
| echo "Contract: $contract, Language: $current_language" | |
| done | |
| echo "language=$language" >> $GITHUB_OUTPUT | |
| - name: Display variables | |
| run: | | |
| echo moonbeam_tag: ${{ steps.moonbeam_release.outputs.tag }} | |
| echo polkadot_parachain_tag: ${{ steps.polkadot_parachain_release.outputs.tag }} | |
| - name: Execute tests | |
| id: run_smart_bench | |
| env: | |
| TEST_PARAMS: --instance-count 1 --call-count 50 | |
| run: | | |
| cd launch | |
| ./download-bins.sh | |
| # overwrite parachains | |
| cp ../bin/* ./bin | |
| ./build.sh | |
| TPS=$(./run.sh -- ${{ matrix.type }} ${{ matrix.contract }} ${TEST_PARAMS} | grep -o 'TPS: [0-9]*' | awk '{print $2}') | |
| echo "tps=$(echo ${TPS})" >> $GITHUB_OUTPUT | |
| - name: Extract Ink benchmark stats | |
| if: matrix.type == 'ink-wasm' | |
| run: | | |
| CURRENT_DATE=$(date +"%s") | |
| # date, parachain tag, contract, contract language, TPS | |
| echo "${CURRENT_DATE}, ${{ matrix.type }}, ${{ steps.polkadot_parachain_release.outputs.tag }}, ${{ matrix.contract }}, \ | |
| ${{ steps.run_smart_bench.outputs.tps }}, ${{ steps.ink_version.outputs.language }}" > ${BENCHMARK_PATH}/${BENCHMARK_FILE} | |
| - name: Extract Solang benchmark stats | |
| if: matrix.type == 'sol-wasm' | |
| run: | | |
| CURRENT_DATE=$(date +"%s") | |
| # date, parachain tag, contract, contract language, TPS | |
| echo "${CURRENT_DATE}, ${{ matrix.type }}, ${{ steps.polkadot_parachain_release.outputs.tag }}, ${{ matrix.contract }}, \ | |
| ${{ steps.run_smart_bench.outputs.tps }}, ${{ steps.solang_version.outputs.language }}" > ${BENCHMARK_PATH}/${BENCHMARK_FILE} | |
| - name: Extract Solidity benchmark stats | |
| if: matrix.type == 'evm' | |
| run: | | |
| CURRENT_DATE=$(date +"%s") | |
| # date, parachain tag, contract, contract language, TPS | |
| echo "${CURRENT_DATE}, ${{ matrix.type }}, ${{ steps.moonbeam_release.outputs.tag }}, ${{ matrix.contract }}, \ | |
| ${{ steps.run_smart_bench.outputs.tps }}, ${{ steps.solc_version.outputs.language }}" > ${BENCHMARK_PATH}/${BENCHMARK_FILE} | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ env.BENCHMARK_FILE }} | |
| path: ${{ env.BENCHMARK_PATH }} | |
| retention-days: 1 | |
| collect: | |
| runs-on: ubuntu-latest | |
| needs: [smart_contract_benchmark] | |
| env: | |
| BENCHMARK_FILE: benchmark-result.csv | |
| steps: | |
| - name: Download artifact | |
| uses: actions/download-artifact@v3 | |
| - name: Merge CSV | |
| run: | | |
| cat ./*/*.csv > ${{ env.BENCHMARK_FILE }} | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ env.BENCHMARK_FILE }} | |
| path: ./ | |
| retention-days: 14 | |
| # - name: Commit benchmark stats | |
| # run: | | |
| # CURRENT_DATE=$(date +"%Y%m%d") | |
| # git config --global user.email "actions@example.com" | |
| # git config --global user.name "GitHub Action" | |
| # git add ${BENCHMARK_PATH}/${BENCHMARK_FILE} | |
| # git commit -m "Add benchmark results on ${CURRENT_DATE}" | |
| # git push | |