From 0d4bb4249fac6897b883a2593808a11e287721c0 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Mon, 2 Dec 2024 11:29:11 -0800 Subject: [PATCH 1/4] ci: allow test matrix to always run, reporting success when skipped (#100) --- .github/workflows/pytest_matrix.yml | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pytest_matrix.yml b/.github/workflows/pytest_matrix.yml index c0a9e7bb7..740bc06c4 100644 --- a/.github/workflows/pytest_matrix.yml +++ b/.github/workflows/pytest_matrix.yml @@ -17,11 +17,6 @@ on: - 'poetry.lock' - 'pyproject.toml' pull_request: - paths: - - 'airbyte_cdk/**' - - 'unit_tests/**' - - 'poetry.lock' - - 'pyproject.toml' jobs: pytest: @@ -52,21 +47,35 @@ jobs: # Common steps: - name: Checkout code uses: actions/checkout@v4 + - id: changes + uses: dorny/paths-filter@v3.0.2 + with: + filters: | + src: + - 'airbyte_cdk/**' + - 'unit_tests/**' + - 'bin/**' + - 'poetry.lock' + - 'pyproject.toml' - name: Set up Poetry uses: Gr1N/setup-poetry@v9 + if: steps.changes.outputs.src == 'true' with: poetry-version: "1.7.1" - name: Set up Python uses: actions/setup-python@v5 + if: steps.changes.outputs.src == 'true' with: python-version: ${{ matrix.python-version }} cache: "poetry" - name: Install dependencies + if: steps.changes.outputs.src == 'true' run: poetry install --all-extras # Job-specific step(s): - name: Run Pytest timeout-minutes: 60 + if: steps.changes.outputs.src == 'true' env: GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} run: > @@ -75,17 +84,17 @@ jobs: -m "not linting and not super_slow and not flaky" - name: Print Coverage Report - if: always() + if: always() && steps.changes.outputs.src == 'true' run: poetry run coverage report - name: Create Coverage Artifacts - if: always() + if: always() && steps.changes.outputs.src == 'true' run: | poetry run coverage html -d htmlcov poetry run coverage xml -o htmlcov/coverage.xml - name: Upload coverage to GitHub Artifacts - if: always() + if: always() && steps.changes.outputs.src == 'true' uses: actions/upload-artifact@v4 with: name: py${{ matrix.python-version }}-${{ matrix.os }}-test-coverage From 024cf2604fceaae019bc832c51b1fc2cb6433181 Mon Sep 17 00:00:00 2001 From: Christo Grabowski <108154848+ChristoGrab@users.noreply.github.com> Date: Mon, 2 Dec 2024 11:34:31 -0800 Subject: [PATCH 2/4] fix: ensure SDM image maintains airbyte-ci compatibility (#90) --- Dockerfile | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 79d2880d0..3b0f34a19 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,12 +10,19 @@ COPY dist/*.whl ./dist/ RUN poetry config virtualenvs.create false \ && poetry install --only main --no-interaction --no-ansi || true -# Copy source code -COPY airbyte_cdk ./airbyte_cdk - # Build and install the package RUN pip install dist/*.whl +# Recreate the original structure +RUN mkdir -p source_declarative_manifest \ + && echo 'from source_declarative_manifest.run import run\n\nif __name__ == "__main__":\n run()' > main.py \ + && touch source_declarative_manifest/__init__.py \ + && cp /usr/local/lib/python3.10/site-packages/airbyte_cdk/cli/source_declarative_manifest/_run.py source_declarative_manifest/run.py \ + && cp /usr/local/lib/python3.10/site-packages/airbyte_cdk/cli/source_declarative_manifest/spec.json source_declarative_manifest/ + +# Remove unnecessary build files +RUN rm -rf dist/ pyproject.toml poetry.lock README.md + # Set the entrypoint -ENV AIRBYTE_ENTRYPOINT="source-declarative-manifest" -ENTRYPOINT ["source-declarative-manifest"] +ENV AIRBYTE_ENTRYPOINT="python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] From cf86dedd3df8efbd8f16166cca83e216015facb1 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Mon, 2 Dec 2024 11:41:23 -0800 Subject: [PATCH 3/4] ci: skip downstream connector tests if no source code is changed (#101) --- .github/workflows/connector-tests.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index fb25bc72d..b2b056026 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -94,9 +94,13 @@ jobs: steps: - name: Abort if extra not changed (${{matrix.cdk_extra}}) id: no_changes - if: ${{ matrix.cdk_extra != 'n/a' && needs.cdk_changes.outputs[matrix.cdk_extra] == 'false' }} + if: ${{ needs.cdk_changes.outputs['src'] == 'false' || matrix.cdk_extra != 'n/a' && needs.cdk_changes.outputs[matrix.cdk_extra] == 'false' }} run: | - echo "Aborting job as specified extra not changed: ${{matrix.cdk_extra}} = ${{ needs.cdk_changes.outputs[matrix.cdk_extra] }}" + echo "Aborting job." + echo "Source code changed: ${{ needs.cdk_changes.outputs['src'] }}" + if [ "${{ matrix.cdk_extra }}" != "n/a" ]; then + echo "Extra not changed: ${{ matrix.cdk_extra }} = ${{ needs.cdk_changes.outputs[matrix.cdk_extra] }}" + fi echo "> Skipped '${{matrix.connector}}' (no relevant changes)" >> $GITHUB_STEP_SUMMARY echo "status=cancelled" >> $GITHUB_OUTPUT exit 0 From 6237fc3e0307dbd31b630dae1a3c3af93e8f63f2 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Mon, 2 Dec 2024 11:47:06 -0800 Subject: [PATCH 4/4] ci: run airbyte connector tests from master branch (#99) --- .github/workflows/connector-tests.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index b2b056026..0a082a389 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -74,21 +74,22 @@ jobs: cdk_extra: n/a - connector: source-chargebee cdk_extra: n/a - # Currently not passing CI (unrelated) - # - connector: source-zendesk-support - # cdk_extra: n/a - connector: source-s3 cdk_extra: file-based - connector: destination-pinecone cdk_extra: vector-db-based - connector: destination-motherduck cdk_extra: sql + # ZenDesk currently failing (as of 2024-12-02) + # TODO: Re-enable once fixed + # - connector: source-zendesk-support + # cdk_extra: n/a # TODO: These are manifest connectors and won't work as expected until we # add `--use-local-cdk` support for manifest connectors. - - connector: source-the-guardian-api - cdk_extra: n/a - - connector: source-pokeapi - cdk_extra: n/a + # - connector: source-the-guardian-api + # cdk_extra: n/a + # - connector: source-pokeapi + # cdk_extra: n/a name: "Check: '${{matrix.connector}}' (skip=${{needs.cdk_changes.outputs[matrix.cdk_extra] == 'false'}})" steps: @@ -116,8 +117,7 @@ jobs: if: steps.no_changes.outputs.status != 'cancelled' with: repository: airbytehq/airbyte - # TODO: Revert to `master` after Airbyte CI released: - ref: aj/airbyte-ci/update-python-local-cdk-code + ref: master path: airbyte - name: Test Connector if: steps.no_changes.outputs.status != 'cancelled'