diff --git a/.github/workflows/devx.yml b/.github/workflows/devx.yml new file mode 100644 index 00000000000..24bcbd6aec1 --- /dev/null +++ b/.github/workflows/devx.yml @@ -0,0 +1,267 @@ +name: Haskell CI (DevX Shell) + +on: + pull_request: # Required for workflows to be able to be approved from forks + merge_group: + + # DO NOT DELETE. + # This is required for nightly builds and is invoked by nightly-trigger.yml + # on a schedule trigger. + workflow_dispatch: + inputs: + reason: + description: 'Reason' + required: false + default: manual + tests: + description: 'Tests' + required: false + default: some + +jobs: + build: + defaults: + run: + shell: devx {0} + env: + CABAL_AWS_ACCESS_KEY_ID: ${{ secrets.CABAL_AWS_ACCESS_KEY_ID }} + CABAL_AWS_SECRET_ACCESS_KEY: ${{ secrets.CABAL_AWS_SECRET_ACCESS_KEY }} + # Modify this value to "invalidate" the cabal cache. + CABAL_CACHE_VERSION: "2024-04-24" + + runs-on: ${{ matrix.platform == 'x86_64-linux' && 'ubuntu-latest' || 'macos-latest' }} + + strategy: + fail-fast: false + matrix: + platform: [ x86_64-linux, x86_64-darwin, aarch64-darwin ] + compiler-nix-name: [ ghc96, ghc98 ] # , ghc810 ] + + concurrency: + group: > + a+${{ github.event_name }} + b+${{ github.workflow_ref }} + c+${{ github.job }} + d+${{ matrix.compiler-nix-name }} + e+${{ matrix.platform }} + f+${{ (startsWith(github.ref, 'refs/heads/gh-readonly-queue/') && github.run_id) || github.event.pull_request.number || github.ref }} + cancel-in-progress: true + + steps: + - name: Install GHC and Cabal + uses: input-output-hk/actions/devx@latest + with: + platform: ${{ matrix.platform }} + target-platform: ${{ matrix.target-platform }} + compiler-nix-name: ${{ matrix.compiler-nix-name }} + # for now we'll set minimal to false, as minimal-iog images don't exist. + minimal: false + iog: true + + - name: Concurrency group + run: > + echo + a+${{ github.event_name }} + b+${{ github.workflow_ref }} + c+${{ github.job }} + d+${{ matrix.compiler-nix-name }} + e+${{ matrix.platform }} + f+${{ (startsWith(github.ref, 'refs/heads/gh-readonly-queue/') && github.run_id) || github.event.pull_request.number || github.ref }} + + - uses: actions/checkout@v4 + + - name: add patched cabal + run: | + mkdir ~/bin + case "$(uname -m)-$(uname -o)" in + arm64-Darwin) + curl -L https://ci.zw3rk.com/job/input-output-hk-haskell-nix-example/pullrequest-2/aarch64-darwin.cabal-install-static/latest/download/1 | gunzip > ~/bin/cabal + ;; + x86_64-Darwin) + curl -L https://ci.zw3rk.com/job/input-output-hk-haskell-nix-example/pullrequest-2/x86_64-darwin.cabal-install-static/latest/download/1 | gunzip > ~/bin/cabal + ;; + x86_64-GNU/Linux) + curl -L https://ci.zw3rk.com/job/input-output-hk-haskell-nix-example/pullrequest-2/x86_64-linux.cabal-install-static/latest/download/1 | gunzip > ~/bin/cabal + ;; + esac + chmod +x ~/bin/cabal + + - name: Cabal update + run: ~/bin/cabal update + + - name: Configure build + run: | + cp .github/workflows/cabal.project.local.ci cabal.project.local + echo "# cabal.project.local" + cat cabal.project.local + + # A dry run `build all` operation does *NOT* downlaod anything, it just looks at the package + # indices to generate an install plan. + - name: Build dry run + run: ~/bin/cabal build all --enable-tests --dry-run --minimize-conflict-set + + # From the install plan we generate a dependency list. + - name: Record dependencies + id: record-deps + run: | + # The tests call out to msys2 commands. We generally do not want to mix toolchains, so + # we are very deliberate about only adding msys64 to the path where absolutely necessary. + ${{ (runner.os == 'Windows' && '$env:PATH=("C:\msys64\mingw64\bin;{0}" -f $env:PATH)') || '' }} + cat dist-newstyle/cache/plan.json | jq -r '."install-plan"[] | select(.style != "local") | .id' | sort | uniq > dependencies.txt + + # From the dependency list we restore the cached dependencies. + # We use the hash of `dependencies.txt` as part of the cache key because that will be stable + # until the `index-state` values in the `cabal.project` file changes. + - name: Restore cached dependencies + uses: actions/cache/restore@v4 + id: cache + with: + path: | + ${{ steps.setup-haskell.outputs.cabal-store }} + dist-newstyle + key: cache-${{ env.CABAL_CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles('dependencies.txt') }} + + # Now we install the dependencies. If the cache was found and restored in the previous step, + # this should be a no-op, but if the cache key was not found we need to build stuff so we can + # cache it for the next step. + - name: Install dependencies + run: ~/bin/cabal build all --enable-tests --only-dependencies -j --ghc-option=-j4 + + # Always store the cabal cache. + # This can fail (benign failure) if there is already a hash at that key. + - name: Cache Cabal store + uses: actions/cache/save@v4 + with: + path: | + ${{ steps.setup-haskell.outputs.cabal-store }} + dist-newstyle + key: cache-${{ env.CABAL_CACHE_VERSION }}-${{ runner.os }}-${{ matrix.ghc }}-${{ hashFiles('dependencies.txt') }} + + # Now we build. + - name: Build all + run: ~/bin/cabal build all --enable-tests + + - name: Run tests + env: + KEEP_WORKSPACE: 1 + run: | + export TMPDIR="${{ runner.temp }}" + export TMP="${{ runner.temp }}" + ~/bin/cabal test cardano-testnet cardano-node cardano-node-chairman cardano-submit-api + + - name: Tar failed tests workspaces + if: ${{ failure() }} + env: + TMP: ${{ runner.temp }} + run: | + cd $TMP + find . -name 'module' -type f -exec dirname {} \; | xargs -L1 basename | sort -u | xargs tar -czvf workspaces.tgz + + - name: Upload workspaces on tests failure + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: failed-test-workspaces-${{ matrix.os }}-ghc${{ matrix.ghc }}-cabal${{ matrix.cabal }}.tgz + path: ${{ runner.temp }}/workspaces.tgz + + - name: "Tar artifacts" + run: | + mkdir -p artifacts + + for exe in $(cat dist-newstyle/cache/plan.json | jq -r '."install-plan"[] | select(.style == "local" and (."component-name" | startswith("exe:"))) | ."bin-file"'); do + if [ -f $exe ]; then + echo "Including artifact $exe" + + ( cd artifacts + tar -C "$(dirname $exe)" -czf "$(basename $exe).tar.gz" "$(basename $exe)" + ) + else + echo "Skipping artifact $exe" + fi + done + + - name: Delete socket files in chairman tests in preparation for uploading artifacts + if: ${{ always() }} + run: | + if [ -d "${{ runner.temp }}/chairman" ]; then + find "${{ runner.temp }}/chairman" -type s -exec rm -f {} \; + fi + + - name: Save Artifact + uses: actions/upload-artifact@v4 + if: ${{ always() }} + continue-on-error: true + with: + name: chairman-test-artifacts-${{ matrix.os }}-${{ matrix.ghc }} + path: ${{ runner.temp }}/chairman/ + + # Uncomment the following back in for debugging. Remember to launch a `pwsh` from + # the tmux session to debug `pwsh` issues. And be reminded that the `/msys2` and + # `/msys2/mingw64` paths are not in PATH by default for the workflow, but tmate + # will put them in. + # You may also want to run + # + # $env:PATH=("C:\Program Files\PowerShell\7;{0}" -f $env:ORIGINAL_PATH) + # + # to restore the original path. Do note that some test might need msys2 + # and will silently fail if msys2 is not in path. See the "Run tests" step. + # + # - name: Setup tmate session + # if: ${{ failure() }} + # uses: mxschmitt/action-tmate@v3 + # with: + # limit-access-to-actor: true + + build-complete: + needs: [build] + if: ${{ always() }} + runs-on: ubuntu-latest + steps: + - name: Check if any previous job failed + run: | + if [[ "${{ needs.build.result }}" == "failure" ]]; then + # this ignores skipped dependencies + echo 'Required jobs failed to build.' + exit 1 + else + echo 'Build complete' + fi + + release: + needs: [build] + if: ${{ startsWith(github.ref, 'refs/tags') }} + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Create Release Tag + id: create_release_tag + run: | + echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT + + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: Release ${{ github.ref }} + draft: true + prerelease: false + + - name: Download Artifact + uses: actions/download-artifact@v1 + with: + name: artifacts-ubuntu-latest + + - name: Upload Release Asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./artifacts-ubuntu-latest/cardano-submit-api.tar.gz + asset_name: cardano-submit-api_${{ steps.create_release_tag.outputs.TAG }}-linux.tar.gz + asset_content_type: application/gzip diff --git a/.github/workflows/haskell.yml b/.github/workflows/haskell.yaml similarity index 96% rename from .github/workflows/haskell.yml rename to .github/workflows/haskell.yaml index 3b9bf800118..2c2c194f470 100644 --- a/.github/workflows/haskell.yml +++ b/.github/workflows/haskell.yaml @@ -26,15 +26,10 @@ jobs: fail-fast: false matrix: # If you edit these versions, make sure the version in the lonely macos-latest job below is updated accordingly + # Linux / macOS are in devx.yaml, please ensure the GHC versions line up. ghc: ["9.6.4", "9.8.1"] cabal: ["3.10.2.1"] - os: [windows-latest, ubuntu-latest] - include: - # Using include, to make sure there will only be one macOS job, even if the matrix gets expanded later on. - # We want a single job, because macOS runners are scarce. - - os: macos-latest - cabal: "3.10.2.1" - ghc: "9.6.4" + os: [windows-latest] env: # Modify this value to "invalidate" the cabal cache. diff --git a/.github/workflows/nightly-trigger.yml b/.github/workflows/nightly-trigger.yml index 452077f08e3..0843e5e20c3 100644 --- a/.github/workflows/nightly-trigger.yml +++ b/.github/workflows/nightly-trigger.yml @@ -23,10 +23,18 @@ jobs: git checkout -b nightly git push origin nightly --force - - name: Invoke workflow + - name: Invoke workflow (Windows) uses: input-output-hk/workflow-dispatch@v1 with: workflow: .github/workflows/haskell.yml ref: nightly token: ${{ secrets.MACHINE_TOKEN }} inputs: '{ "reason": "nightly", "tests": "all" }' + + - name: Invoke workflow (macOS & Linux) + uses: input-output-hk/workflow-dispatch@v1 + with: + workflow: .github/workflows/devx.yml + ref: nightly + token: ${{ secrets.MACHINE_TOKEN }} + inputs: '{ "reason": "nightly", "tests": "all" }' diff --git a/cabalHooks/postBuildHook b/cabalHooks/postBuildHook new file mode 100755 index 00000000000..1c0dd11ad47 --- /dev/null +++ b/cabalHooks/postBuildHook @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +#./postBuildHook.local "$@" +$(dirname "${BASH_SOURCE[0]}")/postBuildHook.s3 "$@" diff --git a/cabalHooks/postBuildHook.local b/cabalHooks/postBuildHook.local new file mode 100755 index 00000000000..f206383991c --- /dev/null +++ b/cabalHooks/postBuildHook.local @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +CACHE=$HOME/.cabal-build-cache + +mkdir -p $CACHE + +echo "Uploading $1 to $CACHE" +tar -czf $CACHE/$1.tar.gz --exclude=$3/setup-config $3 +exit 0 + diff --git a/cabalHooks/postBuildHook.s3 b/cabalHooks/postBuildHook.s3 new file mode 100755 index 00000000000..d3465643003 --- /dev/null +++ b/cabalHooks/postBuildHook.s3 @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC1091,SC2046 +. "$(dirname "${BASH_SOURCE[0]}")/s3-credentials.bash" + +CACHE_KEY="$1.tar.gz" + +if ! test -f "$CACHE_KEY" ; then + echo "S3 upload $1" + tar -czf "$CACHE_KEY" --exclude="$3/setup-config" "$3" + aws s3 cp "$CACHE_KEY" s3://"$CACHE_BUCKET"/devx-shell/"$CACHE_KEY" --endpoint-url "$AWS_ENDPOINT" + rm "$CACHE_KEY" # Optional: Cleanup local file +fi diff --git a/cabalHooks/preBuildHook b/cabalHooks/preBuildHook new file mode 100755 index 00000000000..874201d5380 --- /dev/null +++ b/cabalHooks/preBuildHook @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +#./preBuildHook.local "$@" +$(dirname "${BASH_SOURCE[0]}")/preBuildHook.s3 "$@" diff --git a/cabalHooks/preBuildHook.local b/cabalHooks/preBuildHook.local new file mode 100755 index 00000000000..d01c7dbb4ba --- /dev/null +++ b/cabalHooks/preBuildHook.local @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +CACHE=$HOME/.cabal-build-cache + +mkdir -p $CACHE + +if [ -f $CACHE/$1.tar.gz ]; then + echo "Cache hit" + tar -xzf $CACHE/$1.tar.gz + exit 0 +else + echo "Cache miss" + exit 1 +fi \ No newline at end of file diff --git a/cabalHooks/preBuildHook.s3 b/cabalHooks/preBuildHook.s3 new file mode 100755 index 00000000000..7b3da382c1c --- /dev/null +++ b/cabalHooks/preBuildHook.s3 @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC1091,SC2046 +. $(dirname "${BASH_SOURCE[0]}")/s3-credentials.bash + +CACHE_KEY="$1.tar.gz" + +# Check if artifact exists in S3 (with endpoint and credentials) +aws s3 ls s3://"$CACHE_BUCKET"/devx-shell/"$CACHE_KEY" --endpoint-url "$AWS_ENDPOINT" > /dev/null 2>&1 + +# shellcheck disable=SC2181 +if [ $? -eq 0 ]; then + echo "S3 hit $1" + aws s3 cp s3://"$CACHE_BUCKET"/devx-shell/"$CACHE_KEY" - --endpoint-url "$AWS_ENDPOINT" | tar -xz + # leave an empty $CACHE_KEY file for the postBuildHook to see and not re-upload. + touch "$CACHE_KEY" +else + echo "S3 miss $1" + exit 1 +fi diff --git a/cabalHooks/s3-credentials.bash b/cabalHooks/s3-credentials.bash new file mode 100644 index 00000000000..577c7d7067c --- /dev/null +++ b/cabalHooks/s3-credentials.bash @@ -0,0 +1,5 @@ +CACHE_BUCKET="cabal-cache" # Replace with your bucket name +export AWS_ACCESS_KEY_ID=$CABAL_AWS_ACCESS_KEY_ID +export AWS_SECRET_ACCESS_KEY=$CABAL_AWS_SECRET_ACCESS_KEY +export AWS_DEFAULT_REGION=auto +AWS_ENDPOINT="https://c1fe99e5a421ac98af478e51dda5db4f.r2.cloudflarestorage.com"