python-tests.yaml

name: Unit tests

env:
  # enable colored output
  # https://github.com/pytest-dev/pytest/issues/7443
  PY_COLORS: 1

on:
  pull_request:
    paths:
      - .github/workflows/python-tests.yaml
      - "**/*.py"
      - requirements.txt
      - requirements-client.txt
      - requirements-dev.txt
      - setup.cfg
      - Dockerfile
  push:
    branches:
      - main
    paths:
      - .github/workflows/python-tests.yaml
      - "**/*.py"
      - requirements.txt
      - requirements-client.txt
      - requirements-dev.txt
      - setup.cfg
      - Dockerfile

permissions:
  contents: read

# Limit concurrency by workflow/branch combination.
#
# For pull request builds, pushing additional changes to the
# branch will cancel prior in-progress and pending builds.
#
# For builds triggered on a branch push, additional changes
# will wait for prior builds to complete before starting.
#
# https://docs.github.com/en/actions/using-jobs/using-concurrency
concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  run-tests:
    runs-on:
      group: oss-larger-runners
    name: python:${{ matrix.python-version }}, ${{ matrix.database }}
    strategy:
      matrix:
        database:
          - "postgres:14"
          - "sqlite"
        python-version:
          - "3.8"
          - "3.9"
          - "3.10"
          - "3.11"
          - "3.12"

      fail-fast: false

    timeout-minutes: 45

    steps:
      - name: Display current test matrix
        run: echo '${{ toJSON(matrix) }}'

      - uses: actions/checkout@v4
        with:
          persist-credentials: false
          fetch-depth: 0

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"
          cache-dependency-path: "requirements*.txt"

      - name: Get image tag
        id: get_image_tag
        run: |
          SHORT_SHA=$(git rev-parse --short=7 HEAD)
          tmp="sha-$SHORT_SHA-python${{ matrix.python-version }}"
          echo "image_tag=${tmp}" >> $GITHUB_OUTPUT

      - name: Build test image
        uses: docker/build-push-action@v5
        with:
          context: .
          # TODO: We do not need the UI in these tests and we may want to add a build-arg to disable building it
          #       so that CI test runs are faster
          build-args: |
            PYTHON_VERSION=${{ matrix.python-version }}
            PREFECT_EXTRAS=[dev]
          tags: prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }}
          outputs: type=docker,dest=/tmp/image.tar
          cache-from: type=gha
          cache-to: type=gha,mode=max,ignore-error=true

      - name: Test Docker image
        run: |
          docker load --input /tmp/image.tar
          docker run --rm prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }} prefect version

      - name: Build Conda flavored test image
        uses: docker/build-push-action@v5
        with:
          context: .
          build-args: |
            PYTHON_VERSION=${{ matrix.python-version }}
            BASE_IMAGE=prefect-conda
            PREFECT_EXTRAS=[dev]
          tags: prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }}-conda
          outputs: type=docker,dest=/tmp/image-conda.tar
          cache-from: type=gha
          # We do not cache Conda image layers because they very big and slow to upload
          # cache-to: type=gha,mode=max

      - name: Test Conda flavored Docker image
        # Not yet supported for 3.11, see note at top
        if: ${{ matrix.python-version != '3.11' }}
        run: |
          docker load --input /tmp/image-conda.tar
          docker run --rm prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }}-conda prefect version
          docker run --rm prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }}-conda conda --version

      - name: Install packages
        run: |
          python -m pip install -U pip
          pip install --upgrade --upgrade-strategy eager -e .[dev]

      - name: Start database container
        if: ${{ startsWith(matrix.database, 'postgres') }}
        run: >
          docker run
          --name "postgres"
          --detach
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
          --publish 5432:5432
          --tmpfs /var/lib/postgresql/data
          --env POSTGRES_USER="prefect"
          --env POSTGRES_PASSWORD="prefect"
          --env POSTGRES_DB="prefect"
          --env LANG="C.UTF-8"
          --env LANGUAGE="C.UTF-8"
          --env LC_ALL="C.UTF-8"
          --env LC_COLLATE="C.UTF-8"
          --env LC_CTYPE="C.UTF-8"
          ${{ matrix.database }}
          -c max_connections=250

          ./scripts/wait-for-healthy-container.sh postgres 30

          echo "PREFECT_API_DATABASE_CONNECTION_URL=postgresql+asyncpg://prefect:prefect@localhost/prefect" >> $GITHUB_ENV

        # Parallelize tests by scope to reduce expensive service fixture duplication
        # Do not allow the test suite to build images, as we want the prebuilt images to be tested
        # Do not run Kubernetes service tests, we do not have a cluster available
        # maxprocesses 6 is based on empirical testing; higher than 6 sees diminishing returns
      - name: Run tests
        run: >
          pytest tests
          --numprocesses auto
          --maxprocesses 6
          --dist loadscope
          --disable-docker-image-builds
          --exclude-service kubernetes
          --durations 26
          --no-cov

      - name: Create and Upload failure flag
        if: ${{ failure() }}
        id: create_failure_flag
        run: |
          sanitized_name="${{ matrix.python-version }}-${{ matrix.database }}"
          sanitized_name="${sanitized_name//:/-}"
          echo "Failure in $sanitized_name" > "${sanitized_name}-failure.txt"
          echo "artifact_name=${sanitized_name}-failure" >> $GITHUB_OUTPUT

      - name: Upload failure flag
        if: ${{ failure() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ steps.create_failure_flag.outputs.artifact_name }}
          path: "${{ steps.create_failure_flag.outputs.artifact_name }}.txt"

      - name: Check database container
        # Only applicable for Postgres, but we want this to run even when tests fail
        if: always()
        run: >
          docker container inspect postgres
          && docker container logs postgres
          || echo "Ignoring bad exit code"

  run-pydantic-v1-tests:
    runs-on:
      group: oss-larger-runners
    name: pydantic v1, python:${{ matrix.python-version }}
    strategy:
      matrix:
        database:
          - "postgres:14"
        python-version:
          - "3.8"
          - "3.12"

      fail-fast: false

    timeout-minutes: 45

    steps:
      - name: Display current test matrix
        run: echo '${{ toJSON(matrix) }}'

      - uses: actions/checkout@v4
        with:
          persist-credentials: false
          fetch-depth: 0

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"
          cache-dependency-path: "requirements*.txt"

      - name: Get image tag
        id: get_image_tag
        run: |
          SHORT_SHA=$(git rev-parse --short=7 HEAD)
          tmp="sha-$SHORT_SHA-python${{ matrix.python-version }}"
          echo "image_tag=${tmp}" >> $GITHUB_OUTPUT

      - name: Force install pydantic in docker images
        run: |
          sed -i 's/RUN prefect version/RUN pip install "pydantic<2"\nRUN prefect version/' Dockerfile

      - name: Build test image
        uses: docker/build-push-action@v5
        with:
          context: .
          # TODO: We do not need the UI in these tests and we may want to add a build-arg to disable building it
          #       so that CI test runs are faster
          build-args: |
            PYTHON_VERSION=${{ matrix.python-version }}
            PREFECT_EXTRAS=[dev]
          tags: prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }}
          outputs: type=docker,dest=/tmp/image.tar
          cache-from: type=gha
          cache-to: type=gha,mode=max,ignore-error=true

      - name: Test Docker image
        run: |
          docker load --input /tmp/image.tar
          docker run --rm prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }} prefect version

      - name: Build Conda flavored test image
        uses: docker/build-push-action@v5
        with:
          context: .
          build-args: |
            PYTHON_VERSION=${{ matrix.python-version }}
            BASE_IMAGE=prefect-conda
            PREFECT_EXTRAS=[dev]
          tags: prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }}-conda
          outputs: type=docker,dest=/tmp/image-conda.tar
          cache-from: type=gha
          # We do not cache Conda image layers because they very big and slow to upload
          # cache-to: type=gha,mode=max

      - name: Test Conda flavored Docker image
        run: |
          docker load --input /tmp/image-conda.tar
          docker run --rm prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }}-conda prefect version
          docker run --rm prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }}-conda conda --version

      - name: Install packages
        run: |
          python -m pip install -U pip
          pip install --upgrade --upgrade-strategy eager -e .[dev]

      - name: Force install pydantic for unit tests
        run: pip install "pydantic<2"

      - name: Start database container
        if: ${{ startsWith(matrix.database, 'postgres') }}
        run: >
          docker run
          --name "postgres"
          --detach
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
          --publish 5432:5432
          --tmpfs /var/lib/postgresql/data
          --env POSTGRES_USER="prefect"
          --env POSTGRES_PASSWORD="prefect"
          --env POSTGRES_DB="prefect"
          --env LANG="C.UTF-8"
          --env LANGUAGE="C.UTF-8"
          --env LC_ALL="C.UTF-8"
          --env LC_COLLATE="C.UTF-8"
          --env LC_CTYPE="C.UTF-8"
          ${{ matrix.database }}
          -c max_connections=250

          ./scripts/wait-for-healthy-container.sh postgres 30

          echo "PREFECT_API_DATABASE_CONNECTION_URL=postgresql+asyncpg://prefect:prefect@localhost/prefect" >> $GITHUB_ENV

        # Parallelize tests by scope to reduce expensive service fixture duplication
        # Do not allow the test suite to build images, as we want the prebuilt images to be tested
        # Do not run Kubernetes service tests, we do not have a cluster available
        # maxprocesses 6 is based on empirical testing; higher than 6 sees diminishing returns
      - name: Run tests
        run: >
          pytest tests
          --numprocesses auto
          --maxprocesses 6
          --dist loadscope
          --disable-docker-image-builds
          --exclude-service kubernetes
          --durations 26
          --no-cov

      - name: Create and Upload failure flag
        if: ${{ failure() }}
        id: create_failure_flag
        run: |
          sanitized_name="pydantic-v1-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.database }}-${{ matrix.pytest-options }}"
          sanitized_name="${sanitized_name//:/-}"
          echo "Failure in $sanitized_name" > "${sanitized_name}-failure.txt"
          echo "artifact_name=${sanitized_name}-failure" >> $GITHUB_OUTPUT

      - name: Upload failure flag
        if: ${{ failure() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ steps.create_failure_flag.outputs.artifact_name }}
          path: "${{ steps.create_failure_flag.outputs.artifact_name }}.txt"

      - name: Check database container
        # Only applicable for Postgres, but we want this to run even when tests fail
        if: always()
        run: >
          docker container inspect postgres
          && docker container logs postgres
          || echo "Ignoring bad exit code"

  run-sqlalchemy-v1-tests:
    runs-on:
      group: oss-larger-runners
    name: sqlalchemy v1, python:${{ matrix.python-version }}
    strategy:
      matrix:
        database:
          - "postgres:14"
        python-version:
          - "3.8"
          - "3.12"

      fail-fast: false

    timeout-minutes: 45

    steps:
      - name: Display current test matrix
        run: echo '${{ toJSON(matrix) }}'

      - uses: actions/checkout@v4
        with:
          persist-credentials: false
          fetch-depth: 0

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"
          cache-dependency-path: "requirements*.txt"

      - name: Get image tag
        id: get_image_tag
        run: |
          SHORT_SHA=$(git rev-parse --short=7 HEAD)
          tmp="sha-$SHORT_SHA-python${{ matrix.python-version }}"
          echo "image_tag=${tmp}" >> $GITHUB_OUTPUT

      - name: Force install sqlalchemy in docker images
        run: |
          sed -i 's/RUN prefect version/RUN pip install "sqlalchemy[asyncio]<2"\nRUN prefect version/' Dockerfile

      - name: Build test image
        uses: docker/build-push-action@v5
        with:
          context: .
          # TODO: We do not need the UI in these tests and we may want to add a build-arg to disable building it
          #       so that CI test runs are faster
          build-args: |
            PYTHON_VERSION=${{ matrix.python-version }}
            PREFECT_EXTRAS=[dev]
          tags: prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }}
          outputs: type=docker,dest=/tmp/image.tar
          cache-from: type=gha
          cache-to: type=gha,mode=max,ignore-error=true

      - name: Test Docker image
        run: |
          docker load --input /tmp/image.tar
          docker run --rm prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }} prefect version

      - name: Build Conda flavored test image
        uses: docker/build-push-action@v5
        with:
          context: .
          build-args: |
            PYTHON_VERSION=${{ matrix.python-version }}
            BASE_IMAGE=prefect-conda
            PREFECT_EXTRAS=[dev]
          tags: prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }}-conda
          outputs: type=docker,dest=/tmp/image-conda.tar
          cache-from: type=gha
          # We do not cache Conda image layers because they very big and slow to upload
          # cache-to: type=gha,mode=max

      - name: Test Conda flavored Docker image
        run: |
          docker load --input /tmp/image-conda.tar
          docker run --rm prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }}-conda prefect version
          docker run --rm prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }}-conda conda --version

      - name: Install packages
        run: |
          python -m pip install -U pip
          pip install --upgrade --upgrade-strategy eager -e .[dev]

      - name: Force install sqlalchemy for unit tests
        run: pip install "sqlalchemy[asyncio]<2"

      - name: Start database container
        if: ${{ startsWith(matrix.database, 'postgres') }}
        run: >
          docker run
          --name "postgres"
          --detach
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
          --publish 5432:5432
          --tmpfs /var/lib/postgresql/data
          --env POSTGRES_USER="prefect"
          --env POSTGRES_PASSWORD="prefect"
          --env POSTGRES_DB="prefect"
          --env LANG="C.UTF-8"
          --env LANGUAGE="C.UTF-8"
          --env LC_ALL="C.UTF-8"
          --env LC_COLLATE="C.UTF-8"
          --env LC_CTYPE="C.UTF-8"
          ${{ matrix.database }}
          -c max_connections=250

          ./scripts/wait-for-healthy-container.sh postgres 30

          echo "PREFECT_API_DATABASE_CONNECTION_URL=postgresql+asyncpg://prefect:prefect@localhost/prefect" >> $GITHUB_ENV

        # Parallelize tests by scope to reduce expensive service fixture duplication
        # Do not allow the test suite to build images, as we want the prebuilt images to be tested
        # Do not run Kubernetes service tests, we do not have a cluster available
        # maxprocesses 6 is based on empirical testing; higher than 6 sees diminishing returns
      - name: Run tests
        run: >
          pytest tests
          --numprocesses auto
          --maxprocesses 6
          --dist loadscope
          --disable-docker-image-builds
          --exclude-service kubernetes
          --durations 26
          --no-cov

      - name: Create and Upload failure flag
        if: ${{ failure() }}
        id: create_failure_flag
        run: |
          sanitized_name="sqlalchemy-v1-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.database }}-${{ matrix.pytest-options }}"
          sanitized_name="${sanitized_name//:/-}"
          echo "Failure in $sanitized_name" > "${sanitized_name}-failure.txt"
          echo "artifact_name=${sanitized_name}-failure" >> $GITHUB_OUTPUT

      - name: Upload failure flag
        if: ${{ failure() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ steps.create_failure_flag.outputs.artifact_name }}
          path: "${{ steps.create_failure_flag.outputs.artifact_name }}.txt"

      - name: Check database container
        # Only applicable for Postgres, but we want this to run even when tests fail
        if: always()
        run: >
          docker container inspect postgres
          && docker container logs postgres
          || echo "Ignoring bad exit code"

  notify-tests-failing-on-main:
    needs: run-tests
    if: github.ref == 'refs/heads/main' && failure()
    runs-on: ubuntu-latest
    env:
      FAILURE_THRESHOLD: 1
    steps:
      - name: Download all failure flags
        uses: actions/download-artifact@v4
        with:
          path: failure-flags/

      - name: Check for failure flags
        id: check_failure
        run: |
          failure_count=$(ls -1q failure-flags/*/*.txt | wc -l)

          if [ $failure_count -gt $FAILURE_THRESHOLD ]; then
            too_many_tests_failed="true"
          else
            too_many_tests_failed="false"
          fi
          echo "failure_count=$failure_count" >> $GITHUB_OUTPUT
          echo "too_many_tests_failed=$too_many_tests_failed" >> $GITHUB_OUTPUT

      - name: Send Slack Notification
        if: ${{ steps.check_failure.outputs.too_many_tests_failed == 'true' }}
        uses: 8398a7/action-slack@v3
        with:
          author_name: Prefect OSS Tests Failing on Main
          channel: CBH18KG8G # This is #engineering
          fields: message,commit,author,workflowRun
          status: failure
          text: ":warning: Unit tests are failing in Prefect's main branch. Commit author: please either fix or remove the failing tests. If you remove the failing tests create a GitHub issue with the details."
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.ENGINEERING_REVIEW_SLACK_WEBHOOK_URL }}

  run-tests-for-datadog:
    name: DataDog CI Visibility
    runs-on:
      group: oss-larger-runners
    strategy:
      matrix:
        database:
          - "postgres:14"
        python-version:
          - "3.12"

      fail-fast: false

    timeout-minutes: 45

    steps:
      - name: Display current test matrix
        run: echo '${{ toJSON(matrix) }}'

      - uses: actions/checkout@v4
        with:
          persist-credentials: false
          fetch-depth: 0

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"
          cache-dependency-path: "requirements*.txt"

      - name: Get image tag
        id: get_image_tag
        run: |
          SHORT_SHA=$(git rev-parse --short=7 HEAD)
          tmp="sha-$SHORT_SHA-python${{ matrix.python-version }}"
          echo "image_tag=${tmp}" >> $GITHUB_OUTPUT

      - name: Build test image
        uses: docker/build-push-action@v5
        with:
          context: .
          # TODO: We do not need the UI in these tests and we may want to add a build-arg to disable building it
          #       so that CI test runs are faster
          build-args: |
            PYTHON_VERSION=${{ matrix.python-version }}
            PREFECT_EXTRAS=[dev]
          tags: prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }}
          outputs: type=docker,dest=/tmp/image.tar
          cache-from: type=gha
          cache-to: type=gha,mode=max,ignore-error=true

      - name: Test Docker image
        run: |
          docker load --input /tmp/image.tar
          docker run --rm prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }} prefect version

      - name: Build Conda flavored test image
        uses: docker/build-push-action@v5
        with:
          context: .
          build-args: |
            PYTHON_VERSION=${{ matrix.python-version }}
            BASE_IMAGE=prefect-conda
            PREFECT_EXTRAS=[dev]
          tags: prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }}-conda
          outputs: type=docker,dest=/tmp/image-conda.tar
          cache-from: type=gha
          # We do not cache Conda image layers because they very big and slow to upload
          # cache-to: type=gha,mode=max

      - name: Test Conda flavored Docker image
        # Not yet supported for 3.11, see note at top
        run: |
          docker load --input /tmp/image-conda.tar
          docker run --rm prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }}-conda prefect version
          docker run --rm prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }}-conda conda --version

      - name: Install packages
        run: |
          python -m pip install -U pip
          pip install --upgrade --upgrade-strategy eager -e .[dev]

      - name: Start database container
        run: >
          docker run
          --name "postgres"
          --detach
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
          --publish 5432:5432
          --tmpfs /var/lib/postgresql/data
          --env POSTGRES_USER="prefect"
          --env POSTGRES_PASSWORD="prefect"
          --env POSTGRES_DB="prefect"
          --env LANG="C.UTF-8"
          --env LANGUAGE="C.UTF-8"
          --env LC_ALL="C.UTF-8"
          --env LC_COLLATE="C.UTF-8"
          --env LC_CTYPE="C.UTF-8"
          ${{ matrix.database }}
          -c max_connections=250

          ./scripts/wait-for-healthy-container.sh postgres 30

          echo "PREFECT_API_DATABASE_CONNECTION_URL=postgresql+asyncpg://prefect:prefect@localhost/prefect" >> $GITHUB_ENV

      - name: Run tests
        run: >
          pytest tests
          --numprocesses auto
          --maxprocesses 6
          --ddtrace
          --dist loadscope
          --disable-docker-image-builds
          --exclude-service kubernetes
          --durations 26
          --cov
          --cov-config setup.cfg
        env:
          DD_CIVISIBILITY_AGENTLESS_ENABLED: true
          DD_API_KEY: ${{ secrets.DD_API_KEY_CI_VISIBILITY }}
          DD_SITE: datadoghq.com
          DD_ENV: ci
          DD_SERVICE: prefect