connector-tests.yml

name: Test Connectors
on:
  workflow_dispatch:
  pull_request:
    types:
      - opened
      - synchronize
  # TODO: Consider moving to run these only after the "PyTest (Fast)" workflow is successful.
  # workflow_run:
  #   workflows: [PyTest (Fast)]
  #   types:
  #     - completed

concurrency:
  # This is the name of the concurrency group. It is used to prevent concurrent runs of the same workflow.
  #
  # - github.head_ref is only defined on PR runs, it makes sure that the concurrency group is unique for pull requests
  #  ensuring that only one run per pull request is active at a time.
  #
  # - github.run_id is defined on all runs, it makes sure that the concurrency group is unique for workflow dispatches.
  # This allows us to run multiple workflow dispatches in parallel.
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  cdk_changes:
    name: Get Changes
    runs-on: ubuntu-latest
    permissions:
      statuses: write
      pull-requests: read
    steps:
      - name: Checkout Airbyte
        if: github.event_name != 'pull_request'
        uses: actions/checkout@v4
      - id: changes
        uses: dorny/paths-filter@v3.0.2
        with:
          filters: |
            src:
              - 'airbyte_cdk/**'
              - 'bin/**'
              - 'poetry.lock'
              - 'pyproject.toml'
            file-based:
              - 'airbyte_cdk/sources/file_based/**'
            vector-db-based:
              - 'airbyte_cdk/destinations/vector_db_based/**'
            sql:
              - 'airbyte_cdk/sql/**'
    outputs:
      # Source code modified:
      src: ${{ steps.changes.outputs.src }}
      # Extras modified:
      file-based: ${{ steps.changes.outputs.file-based }}
      vector-db-based: ${{ steps.changes.outputs.vector-db-based }}
      sql: ${{ steps.changes.outputs.sql }}

  connectors_ci:
    needs: cdk_changes
    # We only run the Connectors CI job if there are changes to the connectors on a non-forked PR
    # Forked PRs are handled by the community_ci.yml workflow
    # If the condition is not met the job will be skipped (it will not fail)
    # runs-on: connector-test-large
    runs-on: ubuntu-latest
    timeout-minutes: 360 # 6 hours
    strategy:
      fail-fast: false
      matrix:
        include:
          - connector: source-faker
            cdk_extra: n/a
          - connector: source-shopify
            cdk_extra: n/a
          # Currently not passing CI (unrelated)
          # - connector: source-zendesk-support
          #   cdk_extra: n/a
          - connector: source-s3
            cdk_extra: file-based
          - connector: destination-pinecone
            cdk_extra: vector-db-based
          - connector: destination-motherduck
            cdk_extra: sql

    name: "Check: '${{matrix.connector}}' (skip=${{needs.cdk_changes.outputs[matrix.cdk_extra] == 'false'}})"
    steps:
      - name: Abort if extra not changed (${{matrix.cdk_extra}})
        id: no_changes
        if: ${{ matrix.cdk_extra != 'n/a' && needs.cdk_changes.outputs[matrix.cdk_extra] == 'false' }}
        run: |
          echo "Aborting job as specified extra not changed: ${{matrix.cdk_extra}} = ${{ needs.cdk_changes.outputs[matrix.cdk_extra] }}"
          echo "> Skipped '${{matrix.connector}}' (no relevant changes)" >> $GITHUB_STEP_SUMMARY
          echo "status=cancelled" >> $GITHUB_OUTPUT
          exit 0
        continue-on-error: true
      # Get the monorepo so we can test the connectors
      - name: Checkout CDK
        if: steps.no_changes.outputs.status != 'cancelled'
        uses: actions/checkout@v4
        with:
          path: airbyte-python-cdk
      - name: Checkout Airbyte Monorepo
        uses: actions/checkout@v4
        if: steps.no_changes.outputs.status != 'cancelled'
        with:
          repository: airbytehq/airbyte
          # TODO: Revert to `master` after Airbyte CI released:
          ref: aj/airbyte-ci/update-python-local-cdk-code
          path: airbyte
      - name: Test Connector
        if: steps.no_changes.outputs.status != 'cancelled'
        timeout-minutes: 90
        env:
          GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
          POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0"
        # TODO: Revert below to use `tools.airbyte-ci-binary.install` after Airbyte CI released:
        run: |
          cd airbyte
          make tools.airbyte-ci-dev.install
          airbyte-ci-dev connectors \
          --name ${{matrix.connector}} \
          --use-local-cdk \
          test \
          --fail-fast \
          --skip-step qa_checks \
          --skip-step connector_live_tests

      # Upload the job output to the artifacts
      - name: Upload Job Output
        id: upload_job_output
        if: always() && steps.no_changes.outputs.status != 'cancelled'
        uses: actions/upload-artifact@v4
        with:
          name: ${{matrix.connector}}-job-output
          path: airbyte/airbyte-ci/connectors/pipelines/pipeline_reports

      - name: Evaluate Job Output
        if: always() && steps.no_changes.outputs.status != 'cancelled'
        run: |
          # save job output json file as ci step output
          json_output_file=$(find airbyte/airbyte-ci/connectors/pipelines/pipeline_reports -name 'output.json' -print -quit)
          job_output=$(cat ${json_output_file})
          success=$(echo ${job_output} | jq -r '.success')
          failed_jobs=$(echo ${job_output} | jq -r '.failed_steps')
          run_duration=$(echo ${job_output} | jq -r '.run_duration')
          echo "## Job Output for ${{matrix.connector}}" >> $GITHUB_STEP_SUMMARY
          echo "- Success: ${success}" >> $GITHUB_STEP_SUMMARY
          echo "- Test Duration: $(printf "%.0f" ${run_duration})s" >> $GITHUB_STEP_SUMMARY
          echo "- Failed Checks: ${failed_jobs}" >> $GITHUB_STEP_SUMMARY
          echo -e "\n[Download Job Output](${{steps.upload_job_output.outputs.artifact-url}})" >> $GITHUB_STEP_SUMMARY
          if [ "${success}" != "true" ]; then
            # Throw failure if tests failed
            exit 1
          fi