wip: new rate limiter #7110
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and publish application binaries | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| debugInfo: | |
| description: whether to create binaries with debugging info (dwarf debugging symbols) | |
| default: 'False' | |
| optimizationLevel: | |
| description: optimization level that is used to compile pact and chainweb-node | |
| default: '1' | |
| profiling: | |
| description: enable profiling runtime | |
| default: 'False' | |
| debug: | |
| description: compile and link with -debug | |
| default: 'False' | |
| eventlog: | |
| description: link with -eventlog | |
| default: 'False' | |
| merge_group: | |
| push: | |
| branches-ignore: | |
| - 'gh-readonly-queue/**' | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.kadena_cabal_cache_aws_access_key_id }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.kadena_cabal_cache_aws_secret_access_key }} | |
| AWS_DEFAULT_REGION: us-east-1 | |
| S3_ARTIFACT_BUCKET: kadena-cabal-cache | |
| S3_ARTIFACT_FOLDER: chainweb-node | |
| USE_CABAL_CACHE: true | |
| jobs: | |
| # ########################################################################## # | |
| # Configure Workflow | |
| config: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| git-sha-short: ${{ steps.git-info.outputs.git-sha-short }} | |
| git-ref-short: ${{ steps.git-info.outputs.git-ref-short }} | |
| is-release: ${{ steps.git-info.outputs.is-release }} | |
| debug-info: ${{ steps.compile-flags.outputs.debug-info }} | |
| stripping: ${{ steps.compile-flags.outputs.stripping }} | |
| opt-level: ${{ steps.compile-flags.outputs.opt-level }} | |
| profiling: ${{ steps.compile-flags.outputs.profiling }} | |
| debug: ${{ steps.compile-flags.outputs.debug }} | |
| eventlog: ${{ steps.compile-flags.outputs.eventlog }} | |
| tag-suffix: ${{ steps.compile-flags.outputs.tag-suffix }} | |
| chainweb-network-version: ${{ steps.compile-flags.outputs.chainweb-network-version }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Git revision infos | |
| id: git-info | |
| run: | | |
| echo "git-sha-short=${GITHUB_SHA:0:7}" >> $GITHUB_OUTPUT | |
| echo "is-release=${{ contains(github.event_name, 'release') }}" >> $GITHUB_OUTPUT | |
| echo "git-ref-short=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT | |
| - name: Set optimization level and debugging info | |
| id: compile-flags | |
| run: | | |
| TAG_SUFFIX= | |
| echo "chainweb-network-version=development" >> $GITHUB_OUTPUT | |
| # Optimization | |
| OPT_LEVEL=${{ github.event.inputs.optimizationLevel }} | |
| if [[ -n "$OPT_LEVEL" && "$OPT_LEVEL" != "1" ]] ; then | |
| echo "opt-level=$OPT_LEVEL" >> $GITHUB_OUTPUT | |
| TAG_SUFFIX+=-opt$OPT_LEVEL | |
| else | |
| echo "opt-level=1" >> $GITHUB_OUTPUT | |
| fi | |
| # Debug info | |
| DEBUG_INFO=${{ github.event.inputs.debugInfo }} | |
| if [[ -n "$DEBUG_INFO" && "${DEBUG_INFO^^}" = "TRUE" ]] ; then | |
| echo "building binaries with dwarf debug symbols" | |
| echo "debug-info=True" >> $GITHUB_OUTPUT | |
| echo "stripping=False" >> $GITHUB_OUTPUT | |
| TAG_SUFFIX+=-dwarf | |
| else | |
| echo "building binaries without dwarf debug symbols" | |
| echo "debug-info=False" >> $GITHUB_OUTPUT | |
| echo "stripping=True" >> $GITHUB_OUTPUT | |
| fi | |
| # Profiling | |
| PROFILING=${{ github.event.inputs.profiling }} | |
| if [[ -n "$PROFILING" && "${PROFILING^^}" = "TRUE" ]] ; then | |
| echo "building binaries with profiling runtime" | |
| echo "profiling=True" >> $GITHUB_OUTPUT | |
| TAG_SUFFIX+=-prof | |
| else | |
| echo "building binaries without profiling runtime" | |
| echo "profiling=False" >> $GITHUB_OUTPUT | |
| fi | |
| # Debug | |
| DEBUG=${{ github.event.inputs.debug }} | |
| if [[ -n "$DEBUG" && "${DEBUG^^}" = "TRUE" ]] ; then | |
| echo "building binaries with -debug" | |
| echo "debug=True" >> $GITHUB_OUTPUT | |
| TAG_SUFFIX+=-debug | |
| else | |
| echo "building binaries without -debug" | |
| echo "debug=False" >> $GITHUB_OUTPUT | |
| fi | |
| # Eventlog | |
| EVENTLOG=${{ github.event.inputs.eventlog }} | |
| if [[ -n "$EVENTLOG" && "${EVENTLOG^^}" = "TRUE" ]] ; then | |
| echo "building binaries with -eventlog" | |
| echo "eventlog=True" >> $GITHUB_OUTPUT | |
| TAG_SUFFIX+=-eventlog | |
| else | |
| echo "building binaries without -eventlog" | |
| echo "eventlog=False" >> $GITHUB_OUTPUT | |
| fi | |
| echo "tag-suffix=$TAG_SUFFIX" >> $GITHUB_OUTPUT | |
| # BUILD MATRIX | |
| # | |
| - name: Create matrix | |
| id: set-matrix | |
| run: | | |
| if "${{ github.event_name == 'schedule' }}" == "true"; then | |
| MATRIX="$(jq -c '.' <<EOF | |
| { | |
| # The first version in this list is the version to be used for release | |
| "ghc": ["9.8.2", "9.10.1", "9.6.6"], | |
| "cabal": ["3.12"], | |
| "os": ["ubuntu-22.04"], | |
| "use-freeze-file": ["false"], | |
| "include" : [ | |
| { | |
| "ghc": "9.8.2", | |
| "cabal": "3.12", | |
| "os": "ubuntu-22.04", | |
| "use-freeze-file": "true" | |
| } | |
| ] | |
| } | |
| EOF | |
| )" | |
| else | |
| MATRIX="$(jq -c '.' <<EOF | |
| { | |
| "ghc": ["9.8.2", "9.10.1", "9.6.6"], | |
| "cabal": ["3.12"], | |
| "os": ["ubuntu-22.04"], | |
| "use-freeze-file": ["false"], | |
| "include" : [ | |
| { | |
| "ghc": "9.8.2", | |
| "cabal": "3.12", | |
| "os": "ubuntu-22.04", | |
| "use-freeze-file": "true" | |
| } | |
| ] | |
| } | |
| EOF | |
| )" | |
| fi | |
| echo "matrix=$MATRIX" >> $GITHUB_OUTPUT | |
| - name: Log config | |
| run: | | |
| echo "::group::context" | |
| echo "GITHUB_SHA=${GITHUB_SHA}" | |
| echo "github.event_name=${{ github.event_name }}" | |
| echo "GITHUB_REF=${GITHUB_REF}" | |
| echo "::endgroup::" | |
| echo "::group::config" | |
| echo "config.outputs.git-sha-short=${{ steps.git-info.outputs.git-sha-short }}" | |
| echo "config.outputs.git-ref-short=${{ steps.git-info.outputs.git-ref-short }}" | |
| echo "config.outputs.is-release=${{ steps.git-info.outputs.is-release }}" | |
| echo "config.outputs.debug-info=${{ steps.compile-flags.outputs.debug-info }}" | |
| echo "config.outputs.stripping=${{ steps.compile-flags.outputs.stripping }}" | |
| echo "config.outputs.debug=${{ steps.compile-flags.outputs.debug }}" | |
| echo "config.outputs.eventlog=${{ steps.compile-flags.outputs.eventlog }}" | |
| echo "config.outputs.profiling=${{ steps.compile-flags.outputs.profiling }}" | |
| echo "config.outputs.opt-level=${{ steps.compile-flags.outputs.opt-level }}" | |
| echo "config.outputs.tag-suffix=${{ steps.compile-flags.outputs.tag-suffix }}" | |
| echo "config.outputs.matrix=${{ steps.set-matrix.outputs.matrix }}" | |
| echo "::endgroup::" | |
| # ########################################################################## # | |
| # Download Development Database for Testing | |
| # | |
| # Uploading artifacts is slow, but downloading is fast. | |
| # For the development database that pays off. | |
| sync-chain-db: | |
| name: Download test chain database | |
| needs: [config] | |
| runs-on: 'ubuntu-latest' | |
| strategy: | |
| fail-fast: false | |
| env: | |
| DB_SNAPSHOT_URI: 's3://chainweb-chain-db/${{ needs.config.outputs.chainweb-network-version }}/pipeline-db/rocksDb' | |
| steps: | |
| - name: Sync chain database from S3 | |
| run: aws s3 sync "$DB_SNAPSHOT_URI" db/0/rocksDb --delete --exclude=LOCK | |
| - name: Store chain database as artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: chain-db-${{ needs.config.outputs.chainweb-network-version }} | |
| path: db | |
| # ########################################################################## # | |
| # Build and Upload Artifacts | |
| build: | |
| name: Build | |
| needs: [config] | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.config.outputs.matrix) }} | |
| env: | |
| ARTIFACTS_ARCHIVE: chainweb.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }}.${{ needs.config.outputs.git-sha-short }}${{ needs.config.outputs.tag-suffix }}.tar.gz | |
| ARTIFACTS_NAME: chainweb-applications.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }} | |
| steps: | |
| # Setup | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # This only works if also the times of the cache are adjusted accordingly | |
| # to the time of the commit for the respective revision. This can be done | |
| # by adjusting the time of all files in the cache before the end of the task | |
| # | |
| # - name: Set all tracked file modification times to the time of their last commit | |
| # run: | | |
| # rev=HEAD | |
| # for f in $(git ls-tree -r -t --full-name --name-only "$rev") ; do | |
| # touch -d $(git log --pretty=format:%cI -1 "$rev" -- "$f") "$f"; | |
| # done | |
| # work around for https://github.com/haskell/actions/issues/187 | |
| - name: Set permissions for .ghcup | |
| if: startsWith(matrix.os, 'ubuntu-') | |
| run: sudo chown -R $USER /usr/local/.ghcup | |
| - name: Install GHC and Cabal | |
| id: setup | |
| uses: haskell-actions/setup@v2 | |
| with: | |
| ghc-version: ${{ matrix.ghc }} | |
| cabal-version: ${{ matrix.cabal }} | |
| - name: Confirm GHC and Cabal installation | |
| run: | | |
| echo "setup ghc-version: ${{ steps.setup.outputs.ghc-version }}" | |
| echo "setup cabal-version: ${{ steps.setup.outputs.cabal-version }}" | |
| echo "setup cabal-store: ${{ steps.setup.outputs.cabal-store }}" | |
| ghc --version | |
| cabal --version | |
| - name: Install non-Haskell dependencies (ubuntu) | |
| if: contains(matrix.os, 'ubuntu') | |
| run: sudo apt-get install -y libgflags-dev liblz4-dev libzstd-dev libsnappy-dev libbz2-dev | |
| # Project Configuration | |
| - name: Create cabal.project.local | |
| run: | | |
| cat > cabal.project.local <<EOF | |
| package * | |
| documentation: False | |
| package chainweb | |
| documentation: False | |
| benchmarks: True | |
| tests: True | |
| ghc-options: -Wunused-packages | |
| package pact | |
| documentation: False | |
| EOF | |
| - name: Configure Optimization Level | |
| run: | | |
| cat >> cabal.project.local <<EOF | |
| package pact | |
| optimization: ${{ needs.config.outputs.opt-level }} | |
| package chainweb | |
| optimization: ${{ needs.config.outputs.opt-level }} | |
| EOF | |
| - name: Configure Profiling | |
| run: | | |
| cat >> cabal.project.local <<EOF | |
| package * | |
| profiling: ${{ needs.config.outputs.profiling }} | |
| EOF | |
| - name: Configure Debug Info and Symbol Stripping | |
| run: | | |
| cat >> cabal.project.local <<EOF | |
| package * | |
| debug-info: ${{ needs.config.outputs.debug-info }} | |
| executable-stripping: ${{ needs.config.outputs.stripping }} | |
| library-stripping: ${{ needs.config.outputs.stripping }} | |
| EOF | |
| - name: Set GHC Debug Flag | |
| if: needs.config.outputs.debug == 'True' | |
| run: | | |
| cat >> cabal.project.local <<EOF | |
| package chainweb | |
| ghc-options: -debug | |
| EOF | |
| # dist cache | |
| # the cache-key forces uploading of cache at least once a day, which ensures that | |
| # upstream dependency changes are captured regularly. | |
| - uses: actions/cache@v4 | |
| name: Cache dist-newstyle | |
| with: | |
| path: | | |
| dist-newstyle | |
| key: ${{ matrix.os }}-${{ matrix.ghc }}-${{ matrix.use-freeze-file }}-2-${{ hashFiles('cabal.*', '*.cabal', 'src/**', 'test/**') }} | |
| restore-keys: | | |
| ${{ matrix.os }}-${{ matrix.ghc }}-${{ matrix.use-freeze-file }}-2- | |
| ${{ matrix.os }}-${{ matrix.ghc }}- | |
| # Build | |
| - name: Delete Freeze file if it exists | |
| if: matrix.use-freeze-file != 'true' | |
| # we generate and archive a new freeze file during the build | |
| run: rm -f cabal.project.freeze | |
| - name: Update package database | |
| run: cabal update | |
| - name: Display outdated packages | |
| run: cabal outdated | |
| - name: Configure build | |
| run: cabal build all --dry-run | |
| - name: Generate new freeze file | |
| if: matrix.use-freeze-file != 'true' | |
| run: cabal freeze | |
| - name: Display Dependency Changes | |
| run: | | |
| diff -w <(git show HEAD:cabal.project.freeze) cabal.project.freeze || true | |
| - name: Sync from cabal cache | |
| if: env.USE_CABAL_CACHE == 'true' | |
| uses: larskuhtz/cabal-cache-action@4b537195b33898fcd9adc62cee2a44986fd7b1b6 | |
| with: | |
| bucket: "kadena-cabal-cache" | |
| region: "us-east-1" | |
| folder: "packages/${{ matrix.os }}" | |
| store_path: ${{ steps.setup.outputs.cabal-store }} | |
| aws_access_key_id: "${{ secrets.kadena_cabal_cache_aws_access_key_id }}" | |
| aws_secret_access_key: "${{ secrets.kadena_cabal_cache_aws_secret_access_key }}" | |
| - name: Install build dependencies | |
| run: cabal build chainweb --only-dependencies | |
| - name: Build chainweb library | |
| run: cabal build --ghc-options=-j2 lib:chainweb | |
| - name: Build chainweb applications | |
| run: | | |
| cabal build -j2 --ghc-options=-j2 \ | |
| exe:compact \ | |
| test:chainweb-tests \ | |
| test:multi-node-network-tests \ | |
| test:compaction-tests \ | |
| test:remote-tests \ | |
| exe:cwtool \ | |
| chainweb:bench:bench | |
| - name: Build chainweb-node application | |
| run: cabal build -j2 --ghc-options=-j2 chainweb-node:exe:chainweb-node | |
| # Checks | |
| - name: Check that working directory tree is clean | |
| run: | | |
| if ! git diff --exit-code -- :^cabal.project.freeze ; then | |
| echo "Git working tree is not clean. The build changed some file that is checked into git." 1>&2 | |
| exit 1 | |
| fi | |
| - name: Run ea and verify consistency of genesis headers | |
| run: | | |
| cabal run cwtool -- ea | |
| if ! git diff --exit-code -- src/Chainweb/BlockHeader/Genesis/ src/Chainweb/Pact/Transactions/ ; then | |
| echo "Inconsistent genesis headers detected. Did you forget to run ea?" 1>&2 | |
| exit 1 | |
| fi | |
| # Archive Artifacts | |
| - name: Prepare artifacts | |
| run: | | |
| mkdir -p artifacts/chainweb | |
| cp $(cabal list-bin chainweb-node) artifacts/chainweb | |
| cp $(cabal list-bin compact) artifacts/chainweb | |
| cp $(cabal list-bin chainweb-tests) artifacts/chainweb | |
| cp $(cabal list-bin multi-node-network-tests) artifacts/chainweb | |
| cp $(cabal list-bin remote-tests) artifacts/chainweb | |
| cp $(cabal list-bin compaction-tests) artifacts/chainweb | |
| cp $(cabal list-bin cwtool) artifacts/chainweb | |
| cp $(cabal list-bin bench) artifacts/chainweb | |
| cp README.md artifacts/chainweb | |
| cp CHANGELOG.md artifacts/chainweb | |
| cp LICENSE artifacts/chainweb | |
| cp chainweb.cabal artifacts/chainweb | |
| cp cabal.project artifacts/chainweb | |
| cp cabal.project.local artifacts/chainweb | |
| cp cabal.project.freeze artifacts/chainweb | |
| - name: Create artifacts archive | |
| run: | | |
| echo "tar -C ./artifacts/ -czf $ARTIFACTS_ARCHIVE chainweb" | |
| tar -C ./artifacts/ -czf "$ARTIFACTS_ARCHIVE" chainweb | |
| ls ./artifacts | |
| - name: Safe artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ env.ARTIFACTS_NAME }} | |
| path: ${{ env.ARTIFACTS_ARCHIVE }} | |
| if-no-files-found: error | |
| # ########################################################################## # | |
| # Run misc test suites | |
| remote-tests: | |
| name: Remote tests | |
| needs: [config, build] | |
| runs-on: ${{ matrix.os }} | |
| env: | |
| ARTIFACTS_NAME: chainweb-applications.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.config.outputs.matrix) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download build artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ env.ARTIFACTS_NAME }} | |
| path: . | |
| - name: Install chainweb from artifacts | |
| uses: ./.github/actions/install-chainweb-from-artifacts | |
| with: | |
| os: ${{ matrix.os }} | |
| ghc: ${{ matrix.ghc }} | |
| use-freeze-file: ${{ matrix.use-freeze-file }} | |
| - name: Run remote-tests | |
| timeout-minutes: 2 | |
| run: remote-tests | |
| compaction-tests: | |
| name: Compaction tests | |
| needs: [config, build] | |
| runs-on: ${{ matrix.os }} | |
| env: | |
| ARTIFACTS_NAME: chainweb-applications.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.config.outputs.matrix) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download build artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ env.ARTIFACTS_NAME }} | |
| path: . | |
| - name: Install chainweb from artifacts | |
| uses: ./.github/actions/install-chainweb-from-artifacts | |
| with: | |
| os: ${{ matrix.os }} | |
| ghc: ${{ matrix.ghc }} | |
| use-freeze-file: ${{ matrix.use-freeze-file }} | |
| - name: Run compaction-tests | |
| timeout-minutes: 7 | |
| run: | | |
| ulimit -n 10000 | |
| compaction-tests | |
| multi-node-network-tests: | |
| name: Multi-node network tests | |
| needs: [config, build] | |
| runs-on: ${{ matrix.os }} | |
| env: | |
| ARTIFACTS_NAME: chainweb-applications.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.config.outputs.matrix) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download build artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ env.ARTIFACTS_NAME }} | |
| path: . | |
| - name: Install chainweb from artifacts | |
| uses: ./.github/actions/install-chainweb-from-artifacts | |
| with: | |
| os: ${{ matrix.os }} | |
| ghc: ${{ matrix.ghc }} | |
| use-freeze-file: ${{ matrix.use-freeze-file }} | |
| - name: Run multi-node-network-tests | |
| timeout-minutes: 7 | |
| run: | | |
| ulimit -n 10000 | |
| multi-node-network-tests | |
| # ########################################################################## # | |
| # Run Benchmarks | |
| benchmarks: | |
| name: Benchmarks | |
| needs: [config, build] | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.config.outputs.matrix) }} | |
| runs-on: ${{ matrix.os }} | |
| env: | |
| BENCH_FILE: results.${{ github.run_id }}.${{ github.run_number }}.json | |
| BENCH_BUCKET: kadena-cabal-cache | |
| BENCH_FOLDER: chainweb-benchmark-results/${{ matrix.ghc }}/${{ matrix.os }} | |
| ARTIFACTS_NAME: chainweb-applications.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download build artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ env.ARTIFACTS_NAME }} | |
| path: . | |
| - name: Install chainweb from artifacts | |
| uses: ./.github/actions/install-chainweb-from-artifacts | |
| with: | |
| os: ${{ matrix.os }} | |
| ghc: ${{ matrix.ghc }} | |
| use-freeze-file: ${{ matrix.use-freeze-file }} | |
| - name: Run all benchmarks | |
| timeout-minutes: 12 | |
| run: bench --json $BENCH_FILE +RTS -N | |
| - name: Upload results | |
| run: | | |
| aws s3 cp $BENCH_FILE s3://$BENCH_BUCKET/$BENCH_FOLDER/ | |
| aws s3api put-object-acl --bucket $BENCH_BUCKET --key=$BENCH_FOLDER/$BENCH_FILE --acl public-read | |
| # ########################################################################## # | |
| # Validate Devnet Database and Replay Pact History | |
| validate-devnet-database: | |
| name: Validate Devnet Database Snapshot and Replay Pact History | |
| needs: [config, sync-chain-db, build] | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.config.outputs.matrix) }} | |
| runs-on: ${{ matrix.os }} | |
| env: | |
| ARTIFACTS_NAME: chainweb-applications.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download ${{ needs.config.outputs.chainweb-network-version }} chain database artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: chain-db-${{ needs.config.outputs.chainweb-network-version }} | |
| path: db | |
| - name: Download build artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ env.ARTIFACTS_NAME }} | |
| path: . | |
| - name: Install chainweb from artifacts | |
| uses: ./.github/actions/install-chainweb-from-artifacts | |
| with: | |
| os: ${{ matrix.os }} | |
| ghc: ${{ matrix.ghc }} | |
| use-freeze-file: ${{ matrix.use-freeze-file }} | |
| - name: Write chainweb-node configuration | |
| run: | | |
| cat >> config.yaml <<EOF | |
| databaseDirectory: "db" | |
| chainweb: | |
| onlySyncPact: true | |
| logGas: true | |
| chainwebVersion: ${{ needs.config.outputs.chainweb-network-version }} | |
| validateHashesOnReplay: true | |
| p2p: | |
| peer: | |
| interface: localhost | |
| hostaddress: | |
| port: 4445 | |
| private: true | |
| ignoreBootstrapNodes: true | |
| transactionIndex: | |
| enabled: false | |
| mempoolP2p: | |
| enabled: false | |
| cuts: | |
| pruneChainDatabase: headers-checked | |
| logging: | |
| telemetryBackend: | |
| enabled: false | |
| configuration: | |
| color: 'true' | |
| backend: | |
| color: 'true' | |
| filter: | |
| rules: | |
| - value: mempool | |
| key: component | |
| level: warn | |
| - value: connection-manager | |
| key: component | |
| level: warn | |
| - value: sync | |
| key: sub-component | |
| level: warn | |
| - value: MempoolAccess | |
| key: sub-component | |
| level: warn | |
| - value: PactService | |
| key: logger | |
| level: warn | |
| - value: Checkpointer | |
| key: logger | |
| level: warn | |
| default: info | |
| EOF | |
| - name: Validate database and replay pact transactions | |
| timeout-minutes: 6 | |
| run: | | |
| chainweb-node +RTS -T -H1G -A64M -RTS \ | |
| --config-file=config.yaml \ | |
| --log-level=info \ | |
| --disable-pow | |
| # ########################################################################## # | |
| # Run Unit Tests | |
| unit-tests: | |
| name: Unit tests | |
| needs: [config, build] | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.config.outputs.matrix) }} | |
| env: | |
| TEST_RESULT_FILE: test-results.${{ github.run_id }}.${{ github.run_number }}.json | |
| TEST_RESULT_BUCKET: kadena-cabal-cache | |
| TEST_RESULT_FOLDER: chainweb-test-results/ghc-${{ matrix.ghc }}/${{ matrix.os }} | |
| ARTIFACTS_NAME: chainweb-applications.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download build artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ env.ARTIFACTS_NAME }} | |
| path: . | |
| - name: Install chainweb from artifacts | |
| uses: ./.github/actions/install-chainweb-from-artifacts | |
| with: | |
| os: ${{ matrix.os }} | |
| ghc: ${{ matrix.ghc }} | |
| use-freeze-file: ${{ matrix.use-freeze-file }} | |
| - name: Run Tests | |
| id: tests | |
| timeout-minutes: 10 | |
| run: | | |
| ulimit -n 10000 | |
| chainweb-tests --hide-successes --results-json test-results.json | |
| # Publish test results to S3 | |
| - name: Upload test results | |
| if: always() && steps.tests.outcome != 'skipped' && steps.tests.outcome != 'cancelled' | |
| run: | | |
| cat > filter.jq <<EOF | |
| .gitRevision = "${{ github.sha }}" | | |
| .gitRef = "${{ needs.config.outputs.git-short-ref }}" | | |
| .githubRunId = ${{ github.run_id }} | | |
| .githubRunNumber = ${{ github.run_number }} | |
| EOF | |
| jq -f filter.jq test-results.json > $TEST_RESULT_FILE | |
| aws s3 cp $TEST_RESULT_FILE s3://$TEST_RESULT_BUCKET/$TEST_RESULT_FOLDER/ | |
| aws s3api put-object-acl --bucket $TEST_RESULT_BUCKET --key=$TEST_RESULT_FOLDER/$TEST_RESULT_FILE --acl public-read | |
| # ########################################################################## # | |
| # Publish Artifiacts to S3 | |
| publish: | |
| name: Publish | |
| needs: [config, build] | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJson(needs.config.outputs.matrix) }} | |
| env: | |
| ARTIFACTS_ARCHIVE: chainweb.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }}.${{ needs.config.outputs.git-sha-short }}${{ needs.config.outputs.tag-suffix }}.tar.gz | |
| ARTIFACTS_NAME: chainweb-applications.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }} | |
| steps: | |
| - name: Download build artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ env.ARTIFACTS_NAME }} | |
| path: . | |
| - name: Publish applications to S3 | |
| run: | | |
| aws s3 cp $ARTIFACTS_ARCHIVE s3://$S3_ARTIFACT_BUCKET/$S3_ARTIFACT_FOLDER/ | |
| aws s3api put-object-acl --bucket $S3_ARTIFACT_BUCKET --key=$S3_ARTIFACT_FOLDER/$ARTIFACTS_ARCHIVE --acl public-read | |
| # ########################################################################## # | |
| # Build and publish docker image | |
| docker-image: | |
| name: Build and publish docker image | |
| needs: [config, build] | |
| runs-on: ${{ matrix.os }} | |
| outputs: | |
| docker-suffix: ${{ steps.frozen-tag.outputs.value }}${{ needs.config.outputs.tag-suffix }} | |
| if: "${{ github.event_name != 'schedule' }}" | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # when adding more than one build, use a different package name or | |
| # different tags | |
| include: | |
| - ghc: "9.8.2" | |
| os: "ubuntu-22.04" | |
| use-freeze-file: "true" | |
| - ghc: "9.10.1" | |
| os: "ubuntu-22.04" | |
| use-freeze-file: "false" | |
| env: | |
| OS: ${{ matrix.os }} | |
| ARTIFACTS_NAME: chainweb-applications.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }} | |
| ARTIFACTS_ARCHIVE: chainweb.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }}.${{ needs.config.outputs.git-sha-short }}${{ needs.config.outputs.tag-suffix }}.tar.gz | |
| steps: | |
| - name: Get build artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ env.ARTIFACTS_NAME }} | |
| path: . | |
| - name: Extract artifacts | |
| run: | | |
| tar -xzf "$ARTIFACTS_ARCHIVE" | |
| # ubuntu-22.04 only include libssl3. ubuntu-20.04 uses libssl1.1 | |
| - name: Create Dockerfile | |
| run: | | |
| cat > Dockerfile <<DEOF | |
| FROM ubuntu:${OS#ubuntu-} | |
| LABEL com.chainweb.docker.image.compiler="ghc-${{ matrix.ghc }}" | |
| LABEL com.chainweb.docker.image.os="${OS}" | |
| LABEL com.chainweb.docker.image.optimization="${{ needs.config.outputs.opt-level }}" | |
| LABEL com.chainweb.docker.image.debug-info="${{ needs.config.outputs.debug-info }}" | |
| LABEL com.chainweb.docker.image.profiling="${{ needs.config.outputs.profiling }}" | |
| LABEL com.chainweb.docker.image.debug="${{ needs.config.outputs.debug }}" | |
| LABEL com.chainweb.docker.image.eventlog="${{ needs.config.outputs.eventlog }}" | |
| LABEL com.chainweb.docker.image.revision="${{ needs.config.outputs.git-sha-short }}" | |
| RUN <<EOF | |
| apt-get update && | |
| apt-get install -y \ | |
| ca-certificates \ | |
| libgmp10 \ | |
| libssl3 \ | |
| libsnappy1v5 \ | |
| zlib1g \ | |
| liblz4-1 \ | |
| libbz2-1.0 \ | |
| libgflags2.2 \ | |
| zstd \ | |
| locales && | |
| rm -rf /var/lib/apt/lists/* && | |
| locale-gen en_US.UTF-8 && | |
| update-locale LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 | |
| EOF | |
| ENV LANG=en_US.UTF-8 | |
| WORKDIR /chainweb | |
| COPY chainweb/* . | |
| STOPSIGNAL SIGTERM | |
| ENTRYPOINT [ "/chainweb/chainweb-node" ] | |
| DEOF | |
| - name: Set frozen tag | |
| id: frozen-tag | |
| run: | | |
| if [ "${{ matrix.use-freeze-file }}" == "true" ] ; then | |
| echo "value=-frozen" >> $GITHUB_OUTPUT | |
| else | |
| echo "value=" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v4 | |
| with: | |
| images: ghcr.io/kadena-io/chainweb-node | |
| flavor: | | |
| suffix=${{ steps.frozen-tag.outputs.value }}${{ needs.config.outputs.tag-suffix }} | |
| tags: | | |
| type=sha | |
| type=ref,event=branch | |
| type=ref,event=tag | |
| type=ref,event=pr | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Cache Docker layers | |
| uses: actions/cache@v4 | |
| with: | |
| path: /tmp/.buildx-cache | |
| key: ${{ runner.os }}-buildx-${{ github.sha }} | |
| restore-keys: | | |
| ${{ runner.os }}-buildx- | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: kadena-build | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and push | |
| id: docker_build | |
| uses: docker/build-push-action@v4 | |
| with: | |
| push: true | |
| context: . | |
| file: ./Dockerfile | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| cache-from: type=local,src=/tmp/.buildx-cache | |
| cache-to: type=local,dest=/tmp/.buildx-cache | |
| # ########################################################################## # | |
| # Trigger Integration Tests Run | |
| integration-tests: | |
| name: Run Integration Tests from remote repo | |
| needs: [ config, docker-image ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Get auth token | |
| uses: Nastaliss/get-github-app-pat@683b0b370911354c411dbbc586b7c64f32b2b850 | |
| id: githubAppAuth | |
| with: | |
| app-id: ${{ secrets.GH_KADENAINFRA_APP_ID }} | |
| app-installation-id: ${{ secrets.GH_KADENAINFRA_APP_INSTALLATION_ID }} | |
| app-private-key: ${{ secrets.GH_KADENAINFRA_APP_PRIVATE_KEY }} | |
| - name: Start remote integration tests | |
| uses: the-actions-org/workflow-dispatch@v4.0.0 | |
| with: | |
| workflow: Integration Tests on devnet chain | |
| token: ${{ steps.githubAppAuth.outputs.access-token }} | |
| inputs: '{ "chainweb_node_container_id" : "sha-${{ needs.config.outputs.git-sha-short }}${{ needs.docker-image.outputs.docker-suffix }}" }' | |
| ref: refs/heads/master | |
| repo: kadena-io/integration-tests | |
| display-workflow-run-url: true | |
| wait-for-completion: true # here you could make this pipeline wait them out |