diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index fb25bc72d..0a082a389 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -74,29 +74,34 @@ jobs: cdk_extra: n/a - connector: source-chargebee cdk_extra: n/a - # Currently not passing CI (unrelated) - # - connector: source-zendesk-support - # cdk_extra: n/a - connector: source-s3 cdk_extra: file-based - connector: destination-pinecone cdk_extra: vector-db-based - connector: destination-motherduck cdk_extra: sql + # ZenDesk currently failing (as of 2024-12-02) + # TODO: Re-enable once fixed + # - connector: source-zendesk-support + # cdk_extra: n/a # TODO: These are manifest connectors and won't work as expected until we # add `--use-local-cdk` support for manifest connectors. - - connector: source-the-guardian-api - cdk_extra: n/a - - connector: source-pokeapi - cdk_extra: n/a + # - connector: source-the-guardian-api + # cdk_extra: n/a + # - connector: source-pokeapi + # cdk_extra: n/a name: "Check: '${{matrix.connector}}' (skip=${{needs.cdk_changes.outputs[matrix.cdk_extra] == 'false'}})" steps: - name: Abort if extra not changed (${{matrix.cdk_extra}}) id: no_changes - if: ${{ matrix.cdk_extra != 'n/a' && needs.cdk_changes.outputs[matrix.cdk_extra] == 'false' }} + if: ${{ needs.cdk_changes.outputs['src'] == 'false' || matrix.cdk_extra != 'n/a' && needs.cdk_changes.outputs[matrix.cdk_extra] == 'false' }} run: | - echo "Aborting job as specified extra not changed: ${{matrix.cdk_extra}} = ${{ needs.cdk_changes.outputs[matrix.cdk_extra] }}" + echo "Aborting job." + echo "Source code changed: ${{ needs.cdk_changes.outputs['src'] }}" + if [ "${{ matrix.cdk_extra }}" != "n/a" ]; then + echo "Extra not changed: ${{ matrix.cdk_extra }} = ${{ needs.cdk_changes.outputs[matrix.cdk_extra] }}" + fi echo "> Skipped '${{matrix.connector}}' (no relevant changes)" >> $GITHUB_STEP_SUMMARY echo "status=cancelled" >> $GITHUB_OUTPUT exit 0 @@ -112,8 +117,7 @@ jobs: if: steps.no_changes.outputs.status != 'cancelled' with: repository: airbytehq/airbyte - # TODO: Revert to `master` after Airbyte CI released: - ref: aj/airbyte-ci/update-python-local-cdk-code + ref: master path: airbyte - name: Test Connector if: steps.no_changes.outputs.status != 'cancelled' diff --git a/.github/workflows/pytest_matrix.yml b/.github/workflows/pytest_matrix.yml index c0a9e7bb7..740bc06c4 100644 --- a/.github/workflows/pytest_matrix.yml +++ b/.github/workflows/pytest_matrix.yml @@ -17,11 +17,6 @@ on: - 'poetry.lock' - 'pyproject.toml' pull_request: - paths: - - 'airbyte_cdk/**' - - 'unit_tests/**' - - 'poetry.lock' - - 'pyproject.toml' jobs: pytest: @@ -52,21 +47,35 @@ jobs: # Common steps: - name: Checkout code uses: actions/checkout@v4 + - id: changes + uses: dorny/paths-filter@v3.0.2 + with: + filters: | + src: + - 'airbyte_cdk/**' + - 'unit_tests/**' + - 'bin/**' + - 'poetry.lock' + - 'pyproject.toml' - name: Set up Poetry uses: Gr1N/setup-poetry@v9 + if: steps.changes.outputs.src == 'true' with: poetry-version: "1.7.1" - name: Set up Python uses: actions/setup-python@v5 + if: steps.changes.outputs.src == 'true' with: python-version: ${{ matrix.python-version }} cache: "poetry" - name: Install dependencies + if: steps.changes.outputs.src == 'true' run: poetry install --all-extras # Job-specific step(s): - name: Run Pytest timeout-minutes: 60 + if: steps.changes.outputs.src == 'true' env: GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} run: > @@ -75,17 +84,17 @@ jobs: -m "not linting and not super_slow and not flaky" - name: Print Coverage Report - if: always() + if: always() && steps.changes.outputs.src == 'true' run: poetry run coverage report - name: Create Coverage Artifacts - if: always() + if: always() && steps.changes.outputs.src == 'true' run: | poetry run coverage html -d htmlcov poetry run coverage xml -o htmlcov/coverage.xml - name: Upload coverage to GitHub Artifacts - if: always() + if: always() && steps.changes.outputs.src == 'true' uses: actions/upload-artifact@v4 with: name: py${{ matrix.python-version }}-${{ matrix.os }}-test-coverage diff --git a/Dockerfile b/Dockerfile index 79d2880d0..3b0f34a19 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,12 +10,19 @@ COPY dist/*.whl ./dist/ RUN poetry config virtualenvs.create false \ && poetry install --only main --no-interaction --no-ansi || true -# Copy source code -COPY airbyte_cdk ./airbyte_cdk - # Build and install the package RUN pip install dist/*.whl +# Recreate the original structure +RUN mkdir -p source_declarative_manifest \ + && echo 'from source_declarative_manifest.run import run\n\nif __name__ == "__main__":\n run()' > main.py \ + && touch source_declarative_manifest/__init__.py \ + && cp /usr/local/lib/python3.10/site-packages/airbyte_cdk/cli/source_declarative_manifest/_run.py source_declarative_manifest/run.py \ + && cp /usr/local/lib/python3.10/site-packages/airbyte_cdk/cli/source_declarative_manifest/spec.json source_declarative_manifest/ + +# Remove unnecessary build files +RUN rm -rf dist/ pyproject.toml poetry.lock README.md + # Set the entrypoint -ENV AIRBYTE_ENTRYPOINT="source-declarative-manifest" -ENTRYPOINT ["source-declarative-manifest"] +ENV AIRBYTE_ENTRYPOINT="python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]