test.yml

name: Test

on:
  workflow_dispatch:
    inputs:
      os:
        type: choice
        description: OS to test with
        default: 'ubuntu-latest'
        options:
        - all
        - ubuntu-latest
        - macos-13  # TODO(matt): switch to 14 once this is fixed: https://github.com/actions/setup-python/issues/825
        - windows-latest
      python_version:
        type: choice
        description: Python version to test with
        default: '3.11'
        options:
        - 'all'
        - '3.8'
        - '3.9'
        - '3.10'
        - '3.11'
        - '3.12'
        - 'pypy3.9'
        - 'pypy3.10'
      test_specification:
        type: string
        description: Specification for the tests to run
        default: 'tests/test_import_hook/'
      fail_fast:
        type: boolean
        default: true
        description: Fail fast
  pull_request:
  merge_group:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number }}
  cancel-in-progress: true

jobs:
  generate-matrix:
    name: Generate Matrix
    runs-on: ubuntu-latest
    outputs:
      os: ${{ steps.generate-matrix.outputs.os }}
      python-version: ${{ steps.generate-matrix.outputs.python-version }}
      fail-fast: ${{ steps.generate-matrix.outputs.fail-fast }}
    steps:
      - uses: actions/setup-node@v4
        with:
          node-version: 16
      - run: npm install js-yaml
      - name: Generate matrix
        id: generate-matrix
        uses: actions/github-script@v7
        with:
          script: |
            const yaml = require('js-yaml')
            const OS = yaml.load(process.env.OS_MATRIX)
            const PYTHON_VERSIONS = yaml.load(process.env.PYTHON_VERSION)

            if (context.eventName == 'workflow_dispatch') {
              const input_os = "${{ github.event.inputs.os }}";
              const input_python_version = "${{ github.event.inputs.python_version }}";
              core.setOutput('os', input_os == "all" ? OS : [input_os]);
              core.setOutput('python-version', input_python_version == "all" ? PYTHON_VERSIONS : [input_python_version]);
              core.setOutput('fail-fast', "${{ github.event.inputs.fail_fast }}");

            } else if (context.eventName == 'merge_group') {
              core.setOutput('os', OS)
              core.setOutput('python-version', PYTHON_VERSIONS)
              core.setOutput('fail-fast', 'false')

            } else if (context.eventName == 'pull_request') {
              const { data: { labels: labels } } = await github.rest.pulls.get({
                owner: context.repo.owner,
                repo: context.repo.repo,
                pull_number: context.payload.pull_request.number
              })
              const labelNames = labels.map(label => label.name)
              if (labelNames.includes('CI-no-fail-fast')) {
                core.setOutput('fail-fast', 'false')
              }
              // Only run latest CPython and PyPy tests on pull requests
              const firstPyPy = PYTHON_VERSIONS.findIndex(version => version.startsWith('pypy'))
              const pythonVersions = [PYTHON_VERSIONS[firstPyPy - 1], PYTHON_VERSIONS[PYTHON_VERSIONS.length - 1]]
              core.setOutput('python-version', pythonVersions)
            }
    env:
      OS_MATRIX: |
        - ubuntu-latest
        - macos-13
        - windows-latest
      PYTHON_VERSION: |
        - '3.8'
        - '3.9'
        - '3.10'
        - '3.11'
        - '3.12'
        - 'pypy3.9'
        - 'pypy3.10'


  test:
    name: Test
    needs: [generate-matrix]
    strategy:
      fail-fast: ${{ needs.generate-matrix.outputs.fail-fast != 'false' }}
      matrix:
        os: ${{ fromJson(needs.generate-matrix.outputs.os) }}
        python-version: ${{ fromJson(needs.generate-matrix.outputs.python-version) }}
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: recursive
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          architecture: "x64"
          cache: "pip"
      - uses: dtolnay/rust-toolchain@stable
        id: rustup
      - name: Install aarch64-apple-darwin Rust target
        if: startsWith(matrix.os, 'macos')
        run: rustup target add aarch64-apple-darwin
      - name: Setup Xcode env
        if: startsWith(matrix.os, 'macos')
        shell: bash
        run: |
          set -ex
          sudo xcode-select -s /Applications/Xcode.app
          bindir="$(xcode-select --print-path)/Toolchains/XcodeDefault.xctoolchain/usr/bin"
          echo "CC=${bindir}/clang" >> "${GITHUB_ENV}"
          echo "CXX=${bindir}/clang++" >> "${GITHUB_ENV}"
          echo "SDKROOT=$(xcrun --sdk macosx --show-sdk-path)" >> "${GITHUB_ENV}"
      # To save disk space
      - name: Disable debuginfo on Windows
        if: startsWith(matrix.os, 'windows')
        run: echo "RUSTFLAGS="-C debuginfo=0"" >> $GITHUB_ENV
      - name: Install test requirements
        run: cd tests && pip install --disable-pip-version-check -r requirements.txt
      - name: Run tests
        shell: bash
        run: |
          EXTRA_ARGS=""
          if [ "${{ needs.generate-matrix.outputs.fail-fast }}" != "false" ]; then
            EXTRA_ARGS="$EXTRA_ARGS --max-failures 4"
          fi

          python tests/runner.py \
            --workspace ./test_workspace \
            --name "${{ matrix.os }}_${{ matrix.python-version }}" \
            ${EXTRA_ARGS} \
            "${{ github.event.inputs.test_specification || 'tests/test_import_hook' }}"
      - name: Upload HTML test report
        uses: actions/upload-artifact@v4
        if: failure()
        with:
          name: ${{ matrix.os }}-${{ matrix.python-version }}-test-report.html
          path: './test_workspace/report.html'
      - name: Publish Test Report
        uses: mikepenz/action-junit-report@v4
        if: success() || failure()
        with:
          report_paths: './test_workspace/reports/*.xml'
          test_files_prefix: "${{ matrix.os }}_${{ matrix.python-version }}"


  conclusion:
    needs:
      - test
    if: always()
    runs-on: ubuntu-latest
    steps:
      - name: Result
        run: |
          jq -C <<< "${needs}"
          # Check if all needs were successful or skipped.
          "$(jq -r 'all(.result as $result | (["success", "skipped"] | contains([$result])))' <<< "${needs}")"
        env:
          needs: ${{ toJson(needs) }}