diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index d0bc02e3..57052332 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -1,4 +1,15 @@ -name: Connectors Tests +name: Test Connectors +on: + workflow_dispatch: + pull_request: + types: + - opened + - synchronize + # TODO: Consider moving to run these only after the "PyTest (Fast)" workflow is successful. + # workflow_run: + # workflows: [PyTest (Fast)] + # types: + # - completed concurrency: # This is the name of the concurrency group. It is used to prevent concurrent runs of the same workflow. @@ -11,12 +22,6 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true -on: - workflow_dispatch: - pull_request: - types: - - opened - - synchronize jobs: cdk_changes: name: Get Changes @@ -51,21 +56,6 @@ jobs: vector-db-based: ${{ steps.changes.outputs.vector-db-based }} sql: ${{ steps.changes.outputs.sql }} - # # The Connector CI Tests is a status check emitted by airbyte-ci - # # We make it pass once we have determined that there are no changes to the connectors - # - name: "Skip Connectors CI tests" - # if: steps.changes.outputs.src != 'true' && github.event_name == 'pull_request' - # run: | - # curl --request POST \ - # --url https://api.github.com/repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }} \ - # --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \ - # --header 'content-type: application/json' \ - # --data '{ - # "state": "success", - # "context": "CDK Changes - Connectors Tests", - # "target_url": "${{ github.event.workflow_run.html_url }}" - # }' \ - connectors_ci: needs: cdk_changes # We only run the Connectors CI job if there are changes to the connectors on a non-forked PR @@ -89,9 +79,7 @@ jobs: cdk_extra: vector-db-based - connector: destination-motherduck cdk_extra: sql - if: > - ( github.event_name == 'pull_request' && needs.cdk_changes.outputs.src == 'true' && github.event.pull_request.head.repo.fork != true - ) || github.event_name == 'workflow_dispatch' + name: "Check: '${{matrix.connector}}' (skip=${{needs.cdk_changes.outputs[matrix.cdk_extra] == 'false'}})" steps: - name: Abort if extra not changed (${{matrix.cdk_extra}}) @@ -103,18 +91,25 @@ jobs: exit 1 continue-on-error: true # Get the monorepo so we can test the connectors + - name: Checkout CDK + if: steps.no_changes.outcome != 'failure' + uses: actions/checkout@v4 + with: + path: airbyte-python-cdk - name: Checkout Airbyte Monorepo uses: actions/checkout@v4 if: steps.no_changes.outcome != 'failure' with: repository: airbytehq/airbyte ref: master + path: airbyte - name: Test Connector if: steps.no_changes.outcome != 'failure' timeout-minutes: 90 env: GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} run: | + cd airbyte make tools.airbyte-ci-binary.install airbyte-ci connectors \ --name ${{matrix.connector}} \ diff --git a/.github/workflows/pytest_fast.yml b/.github/workflows/pytest_fast.yml new file mode 100644 index 00000000..c2511644 --- /dev/null +++ b/.github/workflows/pytest_fast.yml @@ -0,0 +1,59 @@ +name: Pytest (Fast) + +on: + push: + branches: + - main + paths: + - 'airbyte_cdk/**' + - 'poetry.lock' + - 'pyproject.toml' + pull_request: + paths: + - 'airbyte_cdk/**' + - 'poetry.lock' + - 'pyproject.toml' + +jobs: + pytest-fast: + name: Pytest (Fast) + runs-on: ubuntu-latest + steps: + # Common steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Set up Poetry + uses: Gr1N/setup-poetry@v9 + with: + poetry-version: "1.7.1" + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: "poetry" + - name: Install dependencies + run: poetry install --all-extras + + - name: Run Pytest with Coverage (Fast Tests Only) + timeout-minutes: 20 + run: > + poetry run coverage run -m pytest + --durations=5 --exitfirst + -m "not slow" + + - name: Print Coverage Report + if: always() + run: poetry run coverage report + + - name: Create Coverage Artifacts + if: always() + run: | + poetry run coverage html -d htmlcov + poetry run coverage xml -o htmlcov/coverage.xml + + - name: Upload coverage to GitHub Artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: fasttest-coverage + path: htmlcov/ diff --git a/.github/workflows/python_pytest.yml b/.github/workflows/pytest_matrix.yml similarity index 60% rename from .github/workflows/python_pytest.yml rename to .github/workflows/pytest_matrix.yml index ce5edbdc..8417a08e 100644 --- a/.github/workflows/python_pytest.yml +++ b/.github/workflows/pytest_matrix.yml @@ -5,70 +5,23 @@ # 2. `pytest`: Run all tests, across multiple Python versions. # # Note that `pytest-fast` also skips tests that require credentials, allowing it to run on forks. -name: PyTest +name: PyTest Matrix on: push: branches: - main - pull_request: {} + paths: + - 'airbyte_cdk/**' + - 'poetry.lock' + - 'pyproject.toml' + pull_request: + paths: + - 'airbyte_cdk/**' + - 'poetry.lock' + - 'pyproject.toml' jobs: - pytest-fast: - name: Pytest (Fast) - runs-on: ubuntu-latest - steps: - # Common steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Set up Poetry - uses: Gr1N/setup-poetry@v9 - with: - poetry-version: "1.7.1" - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.10" - cache: "poetry" - - name: Install dependencies - run: poetry install --all-extras - - - name: Run Pytest with Coverage (Fast Tests Only) - timeout-minutes: 60 - env: - GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - run: > - poetry run coverage run -m pytest - --durations=5 --exitfirst - -m "not slow and not requires_creds and not linting and not flaky" - - - name: Run Pytest with Coverage (Flaky Tests Only) - timeout-minutes: 60 - continue-on-error: true - env: - GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - run: > - poetry run coverage run -m pytest - --durations=5 --exitfirst - -m "flaky and not slow and not requires_creds" - - - name: Print Coverage Report - if: always() - run: poetry run coverage report - - - name: Create Coverage Artifacts - if: always() - run: | - poetry run coverage html -d htmlcov - poetry run coverage xml -o htmlcov/coverage.xml - - - name: Upload coverage to GitHub Artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: fasttest-coverage - path: htmlcov/ - pytest: name: Pytest (All, Python ${{ matrix.python-version }}, ${{ matrix.os }}) # Don't run on forks. Run on pushes to main, and on PRs that are not from forks. diff --git a/pyproject.toml b/pyproject.toml index a0eb6325..104d5c32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,7 @@ requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning>=1.0.0,<2.0.0"] build-backend = "poetry_dynamic_versioning.backend" + [tool.poetry] name = "airbyte-cdk" description = "A framework for writing Airbyte Connectors."