build_test_release_eudsl.yml

#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
#  See https://llvm.org/LICENSE.txt for license information.
#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#  Copyright (c) 2024.

name: "Build, test, release EUDSL"

on:
  workflow_dispatch:
    inputs:
      force_debug_with_tmate:
        type: boolean
        description: 'Run the build with tmate session'
        required: false
        default: false
      debug_with_tmate:
        type: boolean
        description: 'Run the build with a tmate session ONLY in case of failure'
        required: false
        default: false
  pull_request:
    paths:
      - ".github/workflows/build_test_release_eudsl.yml"
      - "projects/eudsl-tblgen/**"
      - "projects/eudsl-nbgen/**"
      - "projects/eudsl-py/**"
      - "projects/eudsl-llvmpy/**"
      - "third_party/llvm-project"
  merge_group:
  push:
    branches:
      - main

concurrency:
  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
  cancel-in-progress: true

jobs:
  build-eudsl:

    strategy:
      fail-fast: false
      matrix:
        include:
          - name: "ubuntu_x86_64"
            runs-on: "ubuntu-22.04"
            os: "ubuntu"
            arch: "x86_64"
          - name: "ubuntu_aarch64"
            runs-on: "ubuntu-22.04-arm"
            os: "ubuntu"
            arch: "aarch64"
          - name: "windows_x86_64"
            runs-on: "windows-2019"
            os: "windows"
          - name: "macos_arm64"
            runs-on: "macos-14"
            os: "macos"
            arch: "aarch64"
          - name: "macos_x86_64"
            runs-on: "macos-13"
            os: "macos"
            arch: "x86_64"

    runs-on: ${{ matrix.runs-on }}

    name: "Build eudsl ${{ matrix.name }}"

    defaults:
      run:
        shell: bash

    permissions:
      id-token: write
      contents: write

    env:
      # either the PR number or `branch-N` where N always increments
      cache-key: eudsl_${{ matrix.name }}_clang_${{ format('{0}-{1}', github.ref_name, github.run_number) }}

    container:
      image: ${{ matrix.container }}

    steps:
      - name: "Check out repository"
        uses: actions/checkout@v4.2.2

      - name: "Setup base"
        uses: ./.github/actions/setup_base
        id: setup_base
        with:
          cache-key: ${{ env.cache-key }}
          restore-key: "eudsl_${{ matrix.name }}_clang"
          os: ${{ matrix.os }}
          arch: ${{ matrix.arch }}

      - name: "Download MLIR and configure build"
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          
          if [[ "${{ matrix.os }}" == "almalinux" ]] || [[ "${{ matrix.os }}" == "ubuntu" ]]; then
            export RELEASE_PREFIX="mlir_manylinux_${{ matrix.arch }}"
          else
            export RELEASE_PREFIX="mlir_${{ matrix.name }}"
          fi
          RELEASE_URL=$(gh release view llvm --json assets -q '.assets |= sort_by(.createdAt) | .assets | map(select(.name | contains(env.RELEASE_PREFIX))) | .[-1] | .url')
          curl -sLO $RELEASE_URL
          tar xf $RELEASE_PREFIX*.tar.gz
          
          if [[ "${{ matrix.os }}" == "ubuntu" ]]; then
            echo "LLVM_DIR=/host/$PWD/llvm-install/lib/cmake/llvm" >> $GITHUB_ENV
            echo "MLIR_DIR=/host/$PWD/llvm-install/lib/cmake/mlir" >> $GITHUB_ENV
            echo "Clang_DIR=/host/$PWD/llvm-install/lib/cmake/clang" >> $GITHUB_ENV
            echo "PIP_FIND_LINKS=/host/$PWD/wheelhouse" >> $GITHUB_ENV
          else
            echo "LLVM_DIR=$PWD/llvm-install/lib/cmake/llvm" >> $GITHUB_ENV
            echo "MLIR_DIR=$PWD/llvm-install/lib/cmake/mlir" >> $GITHUB_ENV
            echo "Clang_DIR=$PWD/llvm-install/lib/cmake/clang" >> $GITHUB_ENV
            echo "PIP_FIND_LINKS=$PWD/wheelhouse" >> $GITHUB_ENV
          fi
          
          # these have to match the deployment targets of the LLVM build runners
          # ld: warning: object file (libLLVMTableGen.a[3](Error.cpp.o)) was built for newer 'macOS' version (13.7) than being linked (10.13)
          if [[ "${{ matrix.runs-on }}" == "macos-14" ]]; then
            echo MACOSX_DEPLOYMENT_TARGET=14.0 >> $GITHUB_ENV
          fi
          # https://github.com/wjakob/nanobind/issues/433
          # i don't know why i need this here but the llvm build doesn't?
          if [[ "${{ matrix.runs-on }}" == "macos-13" ]]; then
            echo CMAKE_CXX_FLAGS="-fno-aligned-allocation" >> $GITHUB_ENV
          fi
          
          $python3_command -m pip install cibuildwheel
          if [[ "${{ matrix.os }}" == "ubuntu" ]] && [[ "${{ matrix.arch }}" == "aarch64" ]]; then
            sudo apt-get install libarchive-dev antlr libxml2-dev libxslt-dev libcurl4-openssl-dev
          fi
          if [[ "${{ matrix.os }}" == "macos" ]]; then
            brew install antlr boost
          fi
          
          $python3_command -m pip install -r requirements-dev.txt

      - name: "Build eudsl-tblgen"
        run: |
          
          # this one is different from the rest as a smoketest re cibuildwheel
          if [[ "${{ matrix.os }}" == "ubuntu" ]]; then
            export CCACHE_DIR=/host/$CCACHE_DIR
            export CC=clang
            export CXX=clang++
          fi
          $python3_command -m cibuildwheel "$PWD/projects/eudsl-tblgen" --output-dir wheelhouse

      - name: "Build eudsl-llvmpy"
        run: |
          
          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
            if [[ "${{ matrix.os }}" == "ubuntu" ]]; then
              export CCACHE_DIR=/host/$CCACHE_DIR
              export CC=clang
              export CXX=clang++
            fi
            $python3_command -m cibuildwheel "$PWD/projects/eudsl-llvmpy" --output-dir wheelhouse
          else
            export CMAKE_PREFIX_PATH=$PWD/llvm-install
            export PIP_FIND_LINKS=$PWD/wheelhouse
            $python3_command -m pip install eudsl-tblgen -f wheelhouse
            $python3_command -m pip wheel "$PWD/projects/eudsl-llvmpy" -w wheelhouse -v --no-build-isolation
          fi

      - name: "Build eudsl-nbgen"
        run: |

          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
            if [[ "${{ matrix.os }}" == "ubuntu" ]]; then
              export CCACHE_DIR=/host/$CCACHE_DIR
              export CC=clang
              export CXX=clang++
            fi
            $python3_command -m cibuildwheel "$PWD/projects/eudsl-nbgen" --output-dir wheelhouse
          else
            export CMAKE_PREFIX_PATH=$PWD/llvm-install
            export PIP_FIND_LINKS=$PWD/wheelhouse
            $python3_command -m pip wheel "$PWD/projects/eudsl-nbgen" -w wheelhouse -v --no-build-isolation
          fi

#      - name: "Build eudsl-py"
#        if: ${{ ! startsWith(matrix.os, 'windows') }}
#        run: |
#
#          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
#            if [[ "${{ matrix.os }}" == "ubuntu" ]]; then
#              export CCACHE_DIR=/host/$CCACHE_DIR
#            fi
#            $python3_command -m cibuildwheel "$PWD/projects/eudsl-py" --output-dir wheelhouse
#          else
#            export CMAKE_PREFIX_PATH=$PWD/llvm-install
#            export PIP_FIND_LINKS=$PWD/wheelhouse
#            $python3_command -m pip install eudsl-nbgen -f wheelhouse
#            $python3_command -m pip wheel "$PWD/projects/eudsl-py" -w wheelhouse -v --no-build-isolation
#          fi

      # just to/make sure total build continues to work
#      - name: "Build all of eudsl"V
#        if: ${{ github.event_name != 'workflow_dispatch' }}
#        run: |
#
#          $python3_command -m pip install eudsl-tblgen -f wheelhouse
#          cmake -B $PWD/eudsl-build -S $PWD \
#            -DCMAKE_PREFIX_PATH=$PWD/llvm-install \
#            -DCMAKE_BUILD_TYPE=Release \
#            -DCMAKE_INSTALL_PREFIX=$PWD/eudsl-install \
#            -DPython3_EXECUTABLE=$(which $python3_command)
#          cmake --build "$PWD/eudsl-build" --target install

      - name: "Save cache"
        uses: actions/cache/save@v3
        if: ${{ !cancelled() && github.event_name == 'push' && github.ref_name == 'main' }}
        with:
          path: ${{ steps.setup_base.outputs.cache-dir }}
          key: ${{ env.cache-key }}

      - name: Upload artifacts
        uses: actions/upload-artifact@v4
        with:
          name: eudsl_${{ matrix.name }}_artifact
          path: wheelhouse/*.whl
          if-no-files-found: warn

      - name: "Setup tmate session"
        if: ${{ (failure() && inputs.debug_with_tmate) || inputs.force_debug_with_tmate }}
        uses: mxschmitt/action-tmate@v3.18
        with:
          limit-access-to-actor: true
          install-dependencies: ${{ startsWith(matrix.os, 'macos') || startsWith(matrix.os, 'windows') || startsWith(matrix.os, 'ubuntu') }}

  test-eudsl-tblgen:

    needs: [build-eudsl]

    strategy:
      fail-fast: false
      matrix:
        runs-on: ["ubuntu-22.04", "ubuntu-22.04-arm", "macos-13", "macos-14", "windows-2019"]
        python-version: [
          # "3.9", "3.10", "3.11",
          "3.12"
        ]
        include: [
          {runs-on: "ubuntu-22.04", name: "ubuntu_x86_64", os: "ubuntu"},
          {runs-on: "ubuntu-22.04-arm", name: "ubuntu_aarch64", os: "ubuntu"},
          {runs-on: "windows-2019", name: "windows_x86_64", os: "windows"},
          {runs-on: "macos-14", name: "macos_arm64", os: "macos"},
          {runs-on: "macos-13", name: "macos_x86_64", os: "macos"}
        ]

    runs-on: ${{ matrix.runs-on }}

    name: "Test eudsl-tblgen ${{ matrix.name }}"

    defaults:
      run:
        shell: bash

    steps:
      - name: "Check out repository"
        uses: actions/checkout@v4.2.2
        with:
          submodules: false

      - name: "Install Python"
        uses: actions/setup-python@v4
        with:
          python-version: "${{ matrix.python-version }}"

      - uses: actions/download-artifact@v4
        with:
          name: eudsl_${{ matrix.name }}_artifact
          path: wheelhouse

      - name: "Install eudsl-tblgen"
        run: |
          
          python -m pip install pytest
          python -m pip install eudsl-tblgen -f wheelhouse

      - name: "Test eudsl-tblgen"
        run: |
          
          export TESTS_DIR="$PWD/projects/eudsl-tblgen/tests"
          if [ ${{ matrix.os }} == 'windows' ]; then
            python -m pytest -s -rA $TESTS_DIR
          else
            python -m pytest -rA --capture=tee-sys $TESTS_DIR
          fi

#  test-eudsl-py:
#
#    needs: [build-eudsl]
#
#    strategy:
#      fail-fast: false
#      matrix:
#        runs-on: [
#          "ubuntu-22.04",
#          "ubuntu-22.04-arm",
#          "macos-14"
#          # "windows-2019"
#        ]
#        python-version: [
#          # "3.9", "3.10", "3.11",
#          "3.12"]
#        include: [
#          {runs-on: "ubuntu-22.04", name: "ubuntu_x86_64", os: "ubuntu"},
#          {runs-on: "ubuntu-22.04-arm", name: "ubuntu_aarch64", os: "ubuntu"},
#          # TODO(max): enable on windows by statically linking
#          # {runs-on: "windows-2019", name: "windows_x86_64", os: "windows"},
#          {runs-on: "macos-14", name: "macos_arm64", os: "macos"}
#        ]
#
#    runs-on: ${{ matrix.runs-on }}
#
#    name: "Test eudsl-py ${{ matrix.name }}"
#
#    defaults:
#      run:
#        shell: bash
#
#    steps:
#      - name: "Check out repository"
#        uses: actions/checkout@v4.2.2
#        with:
#          submodules: false
#
#      - name: "Install Python"
#        uses: actions/setup-python@v4
#        with:
#          python-version: "${{ matrix.python-version }}"
#
#      - uses: actions/download-artifact@v4
#        with:
#          name: eudsl_${{ matrix.name }}_artifact
#          path: wheelhouse
#
#      - name: "Install eudsl-py"
#        run: |
#
#          python -m pip install pytest
#          python -m pip install eudsl-py -f wheelhouse
#
#      - name: "Test eudsl-py"
#        run: |
#
#          export TESTS_DIR="$PWD/projects/eudsl-py/tests"
#          python -m pytest -rA --capture=tee-sys $TESTS_DIR

  test-eudsl-llvmpy:

    needs: [build-eudsl]

    strategy:
      fail-fast: false
      matrix:
        runs-on: [
          "ubuntu-22.04",
          "ubuntu-22.04-arm",
          "macos-14",
          "macos-13",
          "windows-2019"
        ]
        python-version: [
          # "3.10", "3.11",
          "3.12"
        ]
        include: [
          {runs-on: "ubuntu-22.04", name: "ubuntu_x86_64", os: "ubuntu"},
          {runs-on: "ubuntu-22.04-arm", name: "ubuntu_aarch64", os: "ubuntu"},
          {runs-on: "windows-2019", name: "windows_x86_64", os: "windows"},
          {runs-on: "macos-14", name: "macos_arm64", os: "macos"},
          {runs-on: "macos-13", name: "macos_x86_64", os: "macos"}
        ]

    runs-on: ${{ matrix.runs-on }}

    name: "Test eudsl-llvmpy ${{ matrix.name }}"

    defaults:
      run:
        shell: bash

    steps:
      - name: "Check out repository"
        uses: actions/checkout@v4.2.2
        with:
          submodules: false

      - name: "Install Python"
        uses: actions/setup-python@v4
        with:
          python-version: "${{ matrix.python-version }}"

      - uses: actions/download-artifact@v4
        with:
          name: eudsl_${{ matrix.name }}_artifact
          path: wheelhouse

      - name: "Install eudsl-llvmpy"
        run: |
          
          python -m pip install pytest
          python -m pip install eudsl-llvmpy -f wheelhouse

      - name: "Test eudsl-llvmpy"
        run: |
          
          export TESTS_DIR="$PWD/projects/eudsl-llvmpy/tests"
          python -m pytest -rA --capture=tee-sys $TESTS_DIR

  release-eudsl:

    if: ${{ github.event_name == 'workflow_dispatch' && github.ref_name == 'main' }}
    needs: [
      test-eudsl-tblgen,
      # test-eudsl-py,
      test-eudsl-llvmpy
    ]
    runs-on: "ubuntu-22.04"
    name: "Release eudsl"

    permissions:
      id-token: write
      contents: write

    strategy:
      fail-fast: false
      matrix:
        name: [
         "ubuntu_x86_64",
         "ubuntu_aarch64",
         "macos_arm64",
         "macos_x86_64",
         "windows_x86_64"
        ]

    steps:

      - uses: actions/download-artifact@v4
        with:
          name: eudsl_${{ matrix.name }}_artifact
          path: wheelhouse

      - name: Release current commit
        uses: ncipollo/release-action@v1.12.0
        with:
          artifacts: "wheelhouse/eudsl*.whl"
          token: "${{ secrets.GITHUB_TOKEN }}"
          tag: "latest"
          name: "latest"
          removeArtifacts: false
          allowUpdates: true
          replacesArtifacts: true
          makeLatest: true
          omitBody: true