applications.yml

name: Build and publish application binaries

on:
  workflow_dispatch:
    inputs:
      debugInfo:
         description: whether to create binaries with debugging info (dwarf debugging symbols)
         default: 'False'
      optimizationLevel:
        description: optimization level that is used to compile pact and chainweb-node
        default: '1'
      profiling:
        description: enable profiling runtime
        default: 'False'
      debug:
        description: compile and link with -debug
        default: 'False'
      eventlog:
        description: link with -eventlog
        default: 'False'
  merge_group:
  push:
    branches-ignore:
      - 'gh-readonly-queue/**'

env:
  AWS_ACCESS_KEY_ID: ${{ secrets.kadena_cabal_cache_aws_access_key_id }}
  AWS_SECRET_ACCESS_KEY: ${{ secrets.kadena_cabal_cache_aws_secret_access_key }}
  AWS_DEFAULT_REGION: us-east-1

  S3_ARTIFACT_BUCKET: kadena-cabal-cache
  S3_ARTIFACT_FOLDER: chainweb-node

  USE_CABAL_CACHE: true

jobs:

  # ########################################################################## #
  # Configure Workflow

  config:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
      git-sha-short: ${{ steps.git-info.outputs.git-sha-short }}
      git-ref-short: ${{ steps.git-info.outputs.git-ref-short }}
      is-release: ${{ steps.git-info.outputs.is-release }}
      debug-info: ${{ steps.compile-flags.outputs.debug-info }}
      stripping: ${{ steps.compile-flags.outputs.stripping }}
      opt-level: ${{ steps.compile-flags.outputs.opt-level }}
      profiling: ${{ steps.compile-flags.outputs.profiling }}
      debug: ${{ steps.compile-flags.outputs.debug }}
      eventlog: ${{ steps.compile-flags.outputs.eventlog }}
      tag-suffix: ${{ steps.compile-flags.outputs.tag-suffix }}
      chainweb-network-version: ${{ steps.compile-flags.outputs.chainweb-network-version }}
    steps:
    - name: Checkout repository
      uses: actions/checkout@v4
    - name: Git revision infos
      id: git-info
      run: |
        echo "git-sha-short=${GITHUB_SHA:0:7}" >> $GITHUB_OUTPUT
        echo "is-release=${{ contains(github.event_name, 'release') }}" >> $GITHUB_OUTPUT
        echo "git-ref-short=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT
    - name: Set optimization level and debugging info
      id: compile-flags
      run: |
        TAG_SUFFIX=

        echo "chainweb-network-version=development" >> $GITHUB_OUTPUT

        # Optimization
        OPT_LEVEL=${{ github.event.inputs.optimizationLevel }}
        if [[ -n "$OPT_LEVEL" && "$OPT_LEVEL" != "1" ]] ; then
          echo "opt-level=$OPT_LEVEL" >> $GITHUB_OUTPUT
          TAG_SUFFIX+=-opt$OPT_LEVEL
        else
          echo "opt-level=1" >> $GITHUB_OUTPUT
        fi

        # Debug info
        DEBUG_INFO=${{ github.event.inputs.debugInfo }}
        if [[ -n "$DEBUG_INFO" && "${DEBUG_INFO^^}" = "TRUE" ]] ; then
          echo "building binaries with dwarf debug symbols"
          echo "debug-info=True" >> $GITHUB_OUTPUT
          echo "stripping=False" >> $GITHUB_OUTPUT
          TAG_SUFFIX+=-dwarf
        else
          echo "building binaries without dwarf debug symbols"
          echo "debug-info=False" >> $GITHUB_OUTPUT
          echo "stripping=True" >> $GITHUB_OUTPUT
        fi

        # Profiling
        PROFILING=${{ github.event.inputs.profiling }}
        if [[ -n "$PROFILING" && "${PROFILING^^}" = "TRUE" ]] ; then
          echo "building binaries with profiling runtime"
          echo "profiling=True" >> $GITHUB_OUTPUT
          TAG_SUFFIX+=-prof
        else
          echo "building binaries without profiling runtime"
          echo "profiling=False" >> $GITHUB_OUTPUT
        fi

        # Debug
        DEBUG=${{ github.event.inputs.debug }}
        if [[ -n "$DEBUG" && "${DEBUG^^}" = "TRUE" ]] ; then
          echo "building binaries with -debug"
          echo "debug=True" >> $GITHUB_OUTPUT
          TAG_SUFFIX+=-debug
        else
          echo "building binaries without -debug"
          echo "debug=False" >> $GITHUB_OUTPUT
        fi

        # Eventlog
        EVENTLOG=${{ github.event.inputs.eventlog }}
        if [[ -n "$EVENTLOG" && "${EVENTLOG^^}" = "TRUE" ]] ; then
          echo "building binaries with -eventlog"
          echo "eventlog=True" >> $GITHUB_OUTPUT
          TAG_SUFFIX+=-eventlog
        else
          echo "building binaries without -eventlog"
          echo "eventlog=False" >> $GITHUB_OUTPUT
        fi

        echo "tag-suffix=$TAG_SUFFIX" >> $GITHUB_OUTPUT

    # BUILD MATRIX
    #
    - name: Create matrix
      id: set-matrix
      run: |
        if "${{ github.event_name == 'schedule' }}" == "true"; then
        MATRIX="$(jq -c '.' <<EOF
        {
          # The first version in this list is the version to be used for release
          "ghc": ["9.8.2", "9.10.1", "9.6.6"],
          "cabal": ["3.12"],
          "os": ["ubuntu-22.04"],
          "use-freeze-file": ["false"],
          "include" : [
            {
              "ghc": "9.8.2",
              "cabal": "3.12",
              "os": "ubuntu-22.04",
              "use-freeze-file": "true"
            }
          ]
        }
        EOF
        )"
        else
        MATRIX="$(jq -c '.' <<EOF
        {
          "ghc": ["9.8.2", "9.10.1", "9.6.6"],
          "cabal": ["3.12"],
          "os": ["ubuntu-22.04"],
          "use-freeze-file": ["false"],
          "include" : [
            {
              "ghc": "9.8.2",
              "cabal": "3.12",
              "os": "ubuntu-22.04",
              "use-freeze-file": "true"
            }
          ]
        }
        EOF
        )"
        fi
        echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
    - name: Log config
      run: |
        echo "::group::context"
        echo "GITHUB_SHA=${GITHUB_SHA}"
        echo "github.event_name=${{ github.event_name }}"
        echo "GITHUB_REF=${GITHUB_REF}"
        echo "::endgroup::"
        echo "::group::config"
        echo "config.outputs.git-sha-short=${{ steps.git-info.outputs.git-sha-short }}"
        echo "config.outputs.git-ref-short=${{ steps.git-info.outputs.git-ref-short }}"
        echo "config.outputs.is-release=${{ steps.git-info.outputs.is-release }}"
        echo "config.outputs.debug-info=${{ steps.compile-flags.outputs.debug-info }}"
        echo "config.outputs.stripping=${{ steps.compile-flags.outputs.stripping }}"
        echo "config.outputs.debug=${{ steps.compile-flags.outputs.debug }}"
        echo "config.outputs.eventlog=${{ steps.compile-flags.outputs.eventlog }}"
        echo "config.outputs.profiling=${{ steps.compile-flags.outputs.profiling }}"
        echo "config.outputs.opt-level=${{ steps.compile-flags.outputs.opt-level }}"
        echo "config.outputs.tag-suffix=${{ steps.compile-flags.outputs.tag-suffix }}"
        echo "config.outputs.matrix=${{ steps.set-matrix.outputs.matrix }}"
        echo "::endgroup::"

  # ########################################################################## #
  # Download Development Database for Testing
  #
  # Uploading artifacts is slow, but downloading is fast.
  # For the development database that pays off.

  sync-chain-db:
    name: Download test chain database
    needs: [config]
    runs-on: 'ubuntu-latest'
    strategy:
      fail-fast: false
    env:
      DB_SNAPSHOT_URI: 's3://chainweb-chain-db/${{ needs.config.outputs.chainweb-network-version }}/pipeline-db/rocksDb'
    steps:
    - name: Sync chain database from S3
      run: aws s3 sync "$DB_SNAPSHOT_URI" db/0/rocksDb --delete --exclude=LOCK
    - name: Store chain database as artifact
      uses: actions/upload-artifact@v4
      with:
        name: chain-db-${{ needs.config.outputs.chainweb-network-version }}
        path: db

  # ########################################################################## #
  # Build and Upload Artifacts

  build:
    name: Build
    needs: [config]
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.config.outputs.matrix) }}

    env:
      ARTIFACTS_ARCHIVE: chainweb.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }}.${{ needs.config.outputs.git-sha-short }}${{ needs.config.outputs.tag-suffix }}.tar.gz
      ARTIFACTS_NAME: chainweb-applications.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }}

    steps:
    # Setup
    - name: Checkout repository
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
    # This only works if also the times of the cache are adjusted accordingly
    # to the time of the commit for the respective revision. This can be done
    # by adjusting the time of all files in the cache before the end of the task
    #
    # - name: Set all tracked file modification times to the time of their last commit
    #   run: |
    #     rev=HEAD
    #     for f in $(git ls-tree -r -t --full-name --name-only "$rev") ; do
    #       touch -d $(git log --pretty=format:%cI -1 "$rev" -- "$f") "$f";
    #     done

    # work around for https://github.com/haskell/actions/issues/187
    - name: Set permissions for .ghcup
      if: startsWith(matrix.os, 'ubuntu-')
      run: sudo chown -R $USER /usr/local/.ghcup
    - name: Install GHC and Cabal
      id: setup
      uses: haskell-actions/setup@v2
      with:
         ghc-version: ${{ matrix.ghc }}
         cabal-version: ${{ matrix.cabal }}
    - name: Confirm GHC and Cabal installation
      run: |
        echo "setup ghc-version: ${{ steps.setup.outputs.ghc-version }}"
        echo "setup cabal-version: ${{ steps.setup.outputs.cabal-version }}"
        echo "setup cabal-store: ${{ steps.setup.outputs.cabal-store }}"
        ghc --version
        cabal --version
    - name: Install non-Haskell dependencies (ubuntu)
      if: contains(matrix.os, 'ubuntu')
      run: sudo apt-get install -y libgflags-dev liblz4-dev libzstd-dev libsnappy-dev libbz2-dev
    # Project Configuration
    - name: Create cabal.project.local
      run: |
        cat > cabal.project.local <<EOF
        package *
          documentation: False
        package chainweb
          documentation: False
          benchmarks: True
          tests: True
          ghc-options: -Wunused-packages
        package pact
          documentation: False
        EOF

    - name: Configure Optimization Level
      run: |
        cat >> cabal.project.local <<EOF
        package pact
          optimization: ${{ needs.config.outputs.opt-level }}
        package chainweb
          optimization: ${{ needs.config.outputs.opt-level }}
        EOF

    - name: Configure Profiling
      run: |
        cat >> cabal.project.local <<EOF
        package *
          profiling: ${{ needs.config.outputs.profiling }}
        EOF

    - name: Configure Debug Info and Symbol Stripping
      run: |
        cat >> cabal.project.local <<EOF
        package *
          debug-info: ${{ needs.config.outputs.debug-info }}
          executable-stripping: ${{ needs.config.outputs.stripping }}
          library-stripping: ${{ needs.config.outputs.stripping }}
        EOF

    - name: Set GHC Debug Flag
      if: needs.config.outputs.debug == 'True'
      run: |
        cat >> cabal.project.local <<EOF
        package chainweb
          ghc-options: -debug
        EOF

    # dist cache
    # the cache-key forces uploading of cache at least once a day, which ensures that
    # upstream dependency changes are captured regularly.
    - uses: actions/cache@v4
      name: Cache dist-newstyle
      with:
        path: |
          dist-newstyle
        key: ${{ matrix.os }}-${{ matrix.ghc }}-${{ matrix.use-freeze-file }}-2-${{ hashFiles('cabal.*', '*.cabal', 'src/**', 'test/**') }}
        restore-keys: |
          ${{ matrix.os }}-${{ matrix.ghc }}-${{ matrix.use-freeze-file }}-2-
          ${{ matrix.os }}-${{ matrix.ghc }}-

    # Build
    - name: Delete Freeze file if it exists
      if: matrix.use-freeze-file != 'true'
      # we generate and archive a new freeze file during the build
      run: rm -f cabal.project.freeze
    - name: Update package database
      run: cabal update
    - name: Display outdated packages
      run: cabal outdated
    - name: Configure build
      run: cabal build all --dry-run
    - name: Generate new freeze file
      if: matrix.use-freeze-file != 'true'
      run: cabal freeze
    - name: Display Dependency Changes
      run: |
        diff -w <(git show HEAD:cabal.project.freeze) cabal.project.freeze || true
    - name: Sync from cabal cache
      if: env.USE_CABAL_CACHE == 'true'
      uses: larskuhtz/cabal-cache-action@4b537195b33898fcd9adc62cee2a44986fd7b1b6
      with:
        bucket: "kadena-cabal-cache"
        region: "us-east-1"
        folder: "packages/${{ matrix.os }}"
        store_path: ${{ steps.setup.outputs.cabal-store }}
        aws_access_key_id: "${{ secrets.kadena_cabal_cache_aws_access_key_id }}"
        aws_secret_access_key: "${{ secrets.kadena_cabal_cache_aws_secret_access_key }}"
    - name: Install build dependencies
      run: cabal build chainweb --only-dependencies
    - name: Build chainweb library
      run: cabal build --ghc-options=-j2 lib:chainweb
    - name: Build chainweb applications
      run: | 
        cabal build -j2 --ghc-options=-j2 \
          exe:compact \
          test:chainweb-tests \
          test:multi-node-network-tests \
          test:compaction-tests \
          test:remote-tests \
          exe:cwtool \
          chainweb:bench:bench
    - name: Build chainweb-node application
      run: cabal build -j2 --ghc-options=-j2 chainweb-node:exe:chainweb-node

    # Checks
    - name: Check that working directory tree is clean
      run: |
        if ! git diff --exit-code -- :^cabal.project.freeze ; then
          echo "Git working tree is not clean. The build changed some file that is checked into git." 1>&2
          exit 1
        fi
    - name: Run ea and verify consistency of genesis headers
      run: |
        cabal run cwtool -- ea
        if ! git diff --exit-code -- src/Chainweb/BlockHeader/Genesis/ src/Chainweb/Pact/Transactions/ ; then
          echo "Inconsistent genesis headers detected. Did you forget to run ea?" 1>&2
          exit 1
        fi

    # Archive Artifacts
    - name: Prepare artifacts
      run: |
        mkdir -p artifacts/chainweb
        cp $(cabal list-bin chainweb-node) artifacts/chainweb
        cp $(cabal list-bin compact) artifacts/chainweb
        cp $(cabal list-bin chainweb-tests) artifacts/chainweb
        cp $(cabal list-bin multi-node-network-tests) artifacts/chainweb
        cp $(cabal list-bin remote-tests) artifacts/chainweb
        cp $(cabal list-bin compaction-tests) artifacts/chainweb
        cp $(cabal list-bin cwtool) artifacts/chainweb
        cp $(cabal list-bin bench) artifacts/chainweb
        cp README.md artifacts/chainweb
        cp CHANGELOG.md artifacts/chainweb
        cp LICENSE artifacts/chainweb
        cp chainweb.cabal artifacts/chainweb
        cp cabal.project artifacts/chainweb
        cp cabal.project.local artifacts/chainweb
        cp cabal.project.freeze artifacts/chainweb
    - name: Create artifacts archive
      run: |
        echo "tar -C ./artifacts/ -czf $ARTIFACTS_ARCHIVE chainweb"
        tar -C ./artifacts/ -czf "$ARTIFACTS_ARCHIVE" chainweb
        ls ./artifacts
    - name: Safe artifacts
      uses: actions/upload-artifact@v4
      with:
        name: ${{ env.ARTIFACTS_NAME }}
        path: ${{ env.ARTIFACTS_ARCHIVE }}
        if-no-files-found: error

  # ########################################################################## #
  # Run misc test suites

  remote-tests:
    name: Remote tests
    needs: [config, build]
    runs-on: ${{ matrix.os }}
    env:
      ARTIFACTS_NAME: chainweb-applications.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.config.outputs.matrix) }}
    steps:
    - uses: actions/checkout@v4
    - name: Download build artifacts
      uses: actions/download-artifact@v4
      with:
        name: ${{ env.ARTIFACTS_NAME }}
        path: .
    - name: Install chainweb from artifacts
      uses: ./.github/actions/install-chainweb-from-artifacts
      with:
        os: ${{ matrix.os }}
        ghc: ${{ matrix.ghc }}
        use-freeze-file: ${{ matrix.use-freeze-file }}
    - name: Run remote-tests
      timeout-minutes: 2
      run: remote-tests

  compaction-tests:
    name: Compaction tests
    needs: [config, build]
    runs-on: ${{ matrix.os }}
    env:
      ARTIFACTS_NAME: chainweb-applications.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.config.outputs.matrix) }}
    steps:
    - uses: actions/checkout@v4
    - name: Download build artifacts
      uses: actions/download-artifact@v4
      with:
        name: ${{ env.ARTIFACTS_NAME }}
        path: .
    - name: Install chainweb from artifacts
      uses: ./.github/actions/install-chainweb-from-artifacts
      with:
        os: ${{ matrix.os }}
        ghc: ${{ matrix.ghc }}
        use-freeze-file: ${{ matrix.use-freeze-file }}
    - name: Run compaction-tests
      timeout-minutes: 7
      run: |
          ulimit -n 10000
          compaction-tests

  multi-node-network-tests:
    name: Multi-node network tests
    needs: [config, build]
    runs-on: ${{ matrix.os }}
    env:
      ARTIFACTS_NAME: chainweb-applications.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.config.outputs.matrix) }}
    steps:
    - uses: actions/checkout@v4
    - name: Download build artifacts
      uses: actions/download-artifact@v4
      with:
        name: ${{ env.ARTIFACTS_NAME }}
        path: .
    - name: Install chainweb from artifacts
      uses: ./.github/actions/install-chainweb-from-artifacts
      with:
        os: ${{ matrix.os }}
        ghc: ${{ matrix.ghc }}
        use-freeze-file: ${{ matrix.use-freeze-file }}
    - name: Run multi-node-network-tests
      timeout-minutes: 7
      run: |
          ulimit -n 10000
          multi-node-network-tests

  # ########################################################################## #
  # Run Benchmarks

  benchmarks:
    name: Benchmarks
    needs: [config, build]
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.config.outputs.matrix) }}
    runs-on: ${{ matrix.os }}
    env:
      BENCH_FILE: results.${{ github.run_id }}.${{ github.run_number }}.json
      BENCH_BUCKET: kadena-cabal-cache
      BENCH_FOLDER: chainweb-benchmark-results/${{ matrix.ghc }}/${{ matrix.os }}
      ARTIFACTS_NAME: chainweb-applications.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }}
    steps:
    - uses: actions/checkout@v4
    - name: Download build artifacts
      uses: actions/download-artifact@v4
      with:
        name: ${{ env.ARTIFACTS_NAME }}
        path: .
    - name: Install chainweb from artifacts
      uses: ./.github/actions/install-chainweb-from-artifacts
      with:
        os: ${{ matrix.os }}
        ghc: ${{ matrix.ghc }}
        use-freeze-file: ${{ matrix.use-freeze-file }}
    - name: Run all benchmarks
      timeout-minutes: 12
      run: bench --json $BENCH_FILE +RTS -N
    - name: Upload results
      run: |
        aws s3 cp $BENCH_FILE s3://$BENCH_BUCKET/$BENCH_FOLDER/
        aws s3api put-object-acl --bucket $BENCH_BUCKET --key=$BENCH_FOLDER/$BENCH_FILE --acl public-read

  # ########################################################################## #
  # Validate Devnet Database and Replay Pact History

  validate-devnet-database:
    name: Validate Devnet Database Snapshot and Replay Pact History
    needs: [config, sync-chain-db, build]
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.config.outputs.matrix) }}
    runs-on: ${{ matrix.os }}
    env:
      ARTIFACTS_NAME: chainweb-applications.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }}
    steps:
    - uses: actions/checkout@v4
    - name: Download ${{ needs.config.outputs.chainweb-network-version }} chain database artifact
      uses: actions/download-artifact@v4
      with:
        name: chain-db-${{ needs.config.outputs.chainweb-network-version }}
        path: db
    - name: Download build artifacts
      uses: actions/download-artifact@v4
      with:
        name: ${{ env.ARTIFACTS_NAME }}
        path: .
    - name: Install chainweb from artifacts
      uses: ./.github/actions/install-chainweb-from-artifacts
      with:
        os: ${{ matrix.os }}
        ghc: ${{ matrix.ghc }}
        use-freeze-file: ${{ matrix.use-freeze-file }}
    - name: Write chainweb-node configuration
      run: |
        cat >> config.yaml <<EOF
          databaseDirectory: "db"
          chainweb:
            onlySyncPact: true
            logGas: true
            chainwebVersion: ${{ needs.config.outputs.chainweb-network-version }}
            validateHashesOnReplay: true
            p2p:
              peer:
                interface: localhost
                hostaddress:
                  port: 4445
              private: true
              ignoreBootstrapNodes: true
            transactionIndex:
              enabled: false
            mempoolP2p:
              enabled: false
            cuts:
              pruneChainDatabase: headers-checked
          logging:
            telemetryBackend:
              enabled: false
              configuration:
                color: 'true'
            backend:
              color: 'true'
            filter:
              rules:
              - value: mempool
                key: component
                level: warn
              - value: connection-manager
                key: component
                level: warn
              - value: sync
                key: sub-component
                level: warn
              - value: MempoolAccess
                key: sub-component
                level: warn
              - value: PactService
                key: logger
                level: warn
              - value: Checkpointer
                key: logger
                level: warn
              default: info
        EOF
    - name: Validate database and replay pact transactions
      timeout-minutes: 6
      run: |
        chainweb-node +RTS -T -H1G -A64M -RTS \
          --config-file=config.yaml \
          --log-level=info \
          --disable-pow

  # ########################################################################## #
  # Run Unit Tests

  unit-tests:
    name: Unit tests
    needs: [config, build]
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.config.outputs.matrix) }}
    env:
      TEST_RESULT_FILE: test-results.${{ github.run_id }}.${{ github.run_number }}.json
      TEST_RESULT_BUCKET: kadena-cabal-cache
      TEST_RESULT_FOLDER: chainweb-test-results/ghc-${{ matrix.ghc }}/${{ matrix.os }}
      ARTIFACTS_NAME: chainweb-applications.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }}
    steps:
    - uses: actions/checkout@v4
    - name: Download build artifacts
      uses: actions/download-artifact@v4
      with:
        name: ${{ env.ARTIFACTS_NAME }}
        path: .
    - name: Install chainweb from artifacts
      uses: ./.github/actions/install-chainweb-from-artifacts
      with:
        os: ${{ matrix.os }}
        ghc: ${{ matrix.ghc }}
        use-freeze-file: ${{ matrix.use-freeze-file }}
    - name: Run Tests
      id: tests
      timeout-minutes: 10
      run: |
          ulimit -n 10000
          chainweb-tests --hide-successes --results-json test-results.json

    # Publish test results to S3
    - name: Upload test results
      if: always() && steps.tests.outcome != 'skipped' && steps.tests.outcome != 'cancelled'
      run: |
        cat > filter.jq <<EOF
            .gitRevision = "${{ github.sha }}" |
            .gitRef = "${{ needs.config.outputs.git-short-ref }}" |
            .githubRunId = ${{ github.run_id }} |
            .githubRunNumber = ${{ github.run_number }}
        EOF
        jq -f filter.jq test-results.json > $TEST_RESULT_FILE
        aws s3 cp $TEST_RESULT_FILE s3://$TEST_RESULT_BUCKET/$TEST_RESULT_FOLDER/
        aws s3api put-object-acl --bucket $TEST_RESULT_BUCKET --key=$TEST_RESULT_FOLDER/$TEST_RESULT_FILE --acl public-read

  # ########################################################################## #
  # Publish Artifiacts to S3

  publish:
    name: Publish
    needs: [config, build]
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.config.outputs.matrix) }}
    env:
      ARTIFACTS_ARCHIVE: chainweb.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }}.${{ needs.config.outputs.git-sha-short }}${{ needs.config.outputs.tag-suffix }}.tar.gz
      ARTIFACTS_NAME: chainweb-applications.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }}
    steps:
    - name: Download build artifacts
      uses: actions/download-artifact@v4
      with:
        name: ${{ env.ARTIFACTS_NAME }}
        path: .
    - name: Publish applications to S3
      run: |
        aws s3 cp $ARTIFACTS_ARCHIVE s3://$S3_ARTIFACT_BUCKET/$S3_ARTIFACT_FOLDER/
        aws s3api put-object-acl --bucket $S3_ARTIFACT_BUCKET --key=$S3_ARTIFACT_FOLDER/$ARTIFACTS_ARCHIVE --acl public-read

  # ########################################################################## #
  # Build and publish docker image

  docker-image:
    name: Build and publish docker image
    needs: [config, build]
    runs-on: ${{ matrix.os }}
    outputs:
      docker-suffix: ${{ steps.frozen-tag.outputs.value }}${{ needs.config.outputs.tag-suffix }}
    if: "${{ github.event_name != 'schedule' }}"
    strategy:
      fail-fast: false
      matrix:
        # when adding more than one build, use a different package name or
        # different tags
        include:
        - ghc: "9.8.2"
          os: "ubuntu-22.04"
          use-freeze-file: "true"
        - ghc: "9.10.1"
          os: "ubuntu-22.04"
          use-freeze-file: "false"
    env:
      OS: ${{ matrix.os }}
      ARTIFACTS_NAME: chainweb-applications.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }}
      ARTIFACTS_ARCHIVE: chainweb.${{ matrix.use-freeze-file }}.${{ matrix.ghc }}.${{ matrix.os }}.${{ needs.config.outputs.git-sha-short }}${{ needs.config.outputs.tag-suffix }}.tar.gz

    steps:
    - name: Get build artifacts
      uses: actions/download-artifact@v4
      with:
        name: ${{ env.ARTIFACTS_NAME }}
        path: .

    - name: Extract artifacts
      run: |
        tar -xzf "$ARTIFACTS_ARCHIVE"

    # ubuntu-22.04 only include libssl3. ubuntu-20.04 uses libssl1.1
    - name: Create Dockerfile
      run: |
        cat > Dockerfile <<DEOF
        FROM ubuntu:${OS#ubuntu-}
        LABEL com.chainweb.docker.image.compiler="ghc-${{ matrix.ghc }}"
        LABEL com.chainweb.docker.image.os="${OS}"
        LABEL com.chainweb.docker.image.optimization="${{ needs.config.outputs.opt-level }}"
        LABEL com.chainweb.docker.image.debug-info="${{ needs.config.outputs.debug-info }}"
        LABEL com.chainweb.docker.image.profiling="${{ needs.config.outputs.profiling }}"
        LABEL com.chainweb.docker.image.debug="${{ needs.config.outputs.debug }}"
        LABEL com.chainweb.docker.image.eventlog="${{ needs.config.outputs.eventlog }}"
        LABEL com.chainweb.docker.image.revision="${{ needs.config.outputs.git-sha-short }}"
        RUN <<EOF
        apt-get update &&
        apt-get install -y \
          ca-certificates \
          libgmp10 \
          libssl3 \
          libsnappy1v5 \
          zlib1g \
          liblz4-1 \
          libbz2-1.0 \
          libgflags2.2 \
          zstd \
          locales &&
          rm -rf /var/lib/apt/lists/* &&
          locale-gen en_US.UTF-8 &&
          update-locale LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8
        EOF
        ENV LANG=en_US.UTF-8
        WORKDIR /chainweb
        COPY chainweb/* .
        STOPSIGNAL SIGTERM
        ENTRYPOINT [ "/chainweb/chainweb-node" ]
        DEOF

    - name: Set frozen tag
      id: frozen-tag
      run: |
        if [ "${{ matrix.use-freeze-file }}" == "true" ] ; then
          echo "value=-frozen" >> $GITHUB_OUTPUT
        else
          echo "value=" >> $GITHUB_OUTPUT
        fi

    - name: Docker meta
      id: meta
      uses: docker/metadata-action@v4
      with:
        images: ghcr.io/kadena-io/chainweb-node
        flavor: |
          suffix=${{ steps.frozen-tag.outputs.value }}${{ needs.config.outputs.tag-suffix }}
        tags: |
          type=sha
          type=ref,event=branch
          type=ref,event=tag
          type=ref,event=pr

    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v2

    - name: Cache Docker layers
      uses: actions/cache@v4
      with:
        path: /tmp/.buildx-cache
        key: ${{ runner.os }}-buildx-${{ github.sha }}
        restore-keys: |
          ${{ runner.os }}-buildx-

    - name: Login to GitHub Container Registry
      uses: docker/login-action@v2
      with:
        registry: ghcr.io
        username: kadena-build
        password: ${{ secrets.GITHUB_TOKEN }}

    - name: Build and push
      id: docker_build
      uses: docker/build-push-action@v4
      with:
        push: true
        context: .
        file: ./Dockerfile
        tags: ${{ steps.meta.outputs.tags }}
        labels: ${{ steps.meta.outputs.labels }}
        cache-from: type=local,src=/tmp/.buildx-cache
        cache-to: type=local,dest=/tmp/.buildx-cache

  # ########################################################################## #
  # Trigger Integration Tests Run

  integration-tests:
    name: Run Integration Tests from remote repo
    needs: [ config, docker-image ]
    runs-on: ubuntu-latest
    steps:
    - name: Get auth token
      uses: Nastaliss/get-github-app-pat@683b0b370911354c411dbbc586b7c64f32b2b850
      id: githubAppAuth
      with:
        app-id: ${{ secrets.GH_KADENAINFRA_APP_ID }}
        app-installation-id: ${{ secrets.GH_KADENAINFRA_APP_INSTALLATION_ID }}
        app-private-key: ${{ secrets.GH_KADENAINFRA_APP_PRIVATE_KEY }}
    - name: Start remote integration tests
      uses: the-actions-org/workflow-dispatch@v4.0.0
      with:
        workflow: Integration Tests on devnet chain
        token: ${{ steps.githubAppAuth.outputs.access-token }}
        inputs: '{ "chainweb_node_container_id" : "sha-${{ needs.config.outputs.git-sha-short }}${{ needs.docker-image.outputs.docker-suffix }}" }'
        ref: refs/heads/master
        repo: kadena-io/integration-tests
        display-workflow-run-url: true
        wait-for-completion: true # here you could make this pipeline wait them out