Skip to content

Commit 43e03fb

Browse files
artem1205artem1205
committed
Merge remote-tracking branch 'origin/main' into artem1205/asyncretriever-split-decoders
2 parents 6304780 + 65ed26f commit 43e03fb

File tree

290 files changed

+3642
-960
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

290 files changed

+3642
-960
lines changed

‎.github/release-drafter.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ categories:
1515
- "chore"
1616
- "ci"
1717
- "refactor"
18+
- "testing"
1819
- title: "Documentation 📖"
1920
label: "docs"
2021
change-template: "- $TITLE (#$NUMBER)"
@@ -38,6 +39,9 @@ autolabeler:
3839
- label: "chore"
3940
title:
4041
- '/^chore(\(.*\))?\:/i'
42+
- label: "testing"
43+
title:
44+
- '/^tests(\(.*\))?\:/i'
4145
- label: "ci"
4246
title:
4347
- '/^ci(\(.*\))?\:/i'

‎.github/workflows/connector-tests.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@ jobs:
7272
cdk_extra: n/a
7373
- connector: source-shopify
7474
cdk_extra: n/a
75+
- connector: source-chargebee
76+
cdk_extra: n/a
7577
# Currently not passing CI (unrelated)
7678
# - connector: source-zendesk-support
7779
# cdk_extra: n/a
@@ -81,6 +83,12 @@ jobs:
8183
cdk_extra: vector-db-based
8284
- connector: destination-motherduck
8385
cdk_extra: sql
86+
# TODO: These are manifest connectors and won't work as expected until we
87+
# add `--use-local-cdk` support for manifest connectors.
88+
- connector: source-the-guardian-api
89+
cdk_extra: n/a
90+
- connector: source-pokeapi
91+
cdk_extra: n/a
8492

8593
name: "Check: '${{matrix.connector}}' (skip=${{needs.cdk_changes.outputs[matrix.cdk_extra] == 'false'}})"
8694
steps:

‎.github/workflows/publish_sdm_connector.yml

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
# This flow publishes the Source-Declarative-Manifest (SDM)
2+
# connector to DockerHub as a Docker image.
3+
4+
name: Publish SDM Connector
5+
6+
on:
7+
workflow_dispatch:
8+
inputs:
9+
version:
10+
description:
11+
The version to publish, ie 1.0.0 or 1.0.0-dev1.
12+
If omitted, and if run from a release branch, the version will be
13+
inferred from the git tag.
14+
If omitted, and if run from a non-release branch, then only a SHA-based
15+
Docker tag will be created.
16+
required: false
17+
dry_run:
18+
description: If true, the workflow will not push to DockerHub.
19+
type: boolean
20+
required: false
21+
default: false
22+
23+
jobs:
24+
build:
25+
runs-on: ubuntu-latest
26+
steps:
27+
28+
- name: Detect Release Tag Version
29+
if: startsWith(github.ref, 'refs/tags/v')
30+
run: |
31+
DETECTED_VERSION=${{ github.ref_name }}
32+
echo "Version ref set to '${DETECTED_VERSION}'"
33+
# Remove the 'v' prefix if it exists
34+
DETECTED_VERSION="${DETECTED_VERSION#v}"
35+
echo "Setting version to '$DETECTED_VERSION'"
36+
echo "DETECTED_VERSION=${DETECTED_VERSION}" >> $GITHUB_ENV
37+
38+
- name: Validate and set VERSION from tag ('${{ github.ref_name }}') and input (${{ github.event.inputs.version || 'none' }})
39+
id: set_version
40+
if: github.event_name == 'workflow_dispatch'
41+
run: |
42+
INPUT_VERSION=${{ github.event.inputs.version }}
43+
echo "Version input set to '${INPUT_VERSION}'"
44+
# Exit with success if both detected and input versions are empty
45+
if [ -z "${DETECTED_VERSION:-}" ] && [ -z "${INPUT_VERSION:-}" ]; then
46+
echo "No version detected or input. Will publish to SHA tag instead."
47+
echo 'VERSION=' >> $GITHUB_ENV
48+
exit 0
49+
fi
50+
# Remove the 'v' prefix if it exists
51+
INPUT_VERSION="${INPUT_VERSION#v}"
52+
# Fail if detected version is non-empty and different from the input version
53+
if [ -n "${DETECTED_VERSION:-}" ] && [ -n "${INPUT_VERSION:-}" ] && [ "${DETECTED_VERSION}" != "${INPUT_VERSION}" ]; then
54+
echo "Error: Version input '${INPUT_VERSION}' does not match detected version '${DETECTED_VERSION}'."
55+
exit 1
56+
fi
57+
# Set the version to the input version if non-empty, otherwise the detected version
58+
VERSION="${INPUT_VERSION:-$DETECTED_VERSION}"
59+
# Fail if the version is still empty
60+
if [ -z "$VERSION" ]; then
61+
echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'."
62+
exit 1
63+
fi
64+
echo "Setting version to '$VERSION'"
65+
echo "VERSION=${VERSION}" >> $GITHUB_ENV
66+
echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT
67+
# Check if version is a prerelease version (will not tag 'latest')
68+
if [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
69+
echo "IS_PRERELEASE=false" >> $GITHUB_ENV
70+
echo "IS_PRERELEASE=false" >> $GITHUB_OUTPUT
71+
else
72+
echo "IS_PRERELEASE=true" >> $GITHUB_ENV
73+
echo "IS_PRERELEASE=true" >> $GITHUB_OUTPUT
74+
fi
75+
76+
- uses: actions/checkout@v4
77+
with:
78+
fetch-depth: 0
79+
80+
- uses: hynek/build-and-inspect-python-package@v2
81+
name: Build package with version ref '${{ env.VERSION || '0.0.0dev0' }}'
82+
env:
83+
# Pass in the evaluated version from the previous step
84+
# More info: https://github.com/mtkennerly/poetry-dynamic-versioning#user-content-environment-variables
85+
POETRY_DYNAMIC_VERSIONING_BYPASS: ${{ env.VERSION || '0.0.0dev0'}}
86+
87+
- uses: actions/upload-artifact@v4
88+
with:
89+
name: Packages-${{ github.run_id }}
90+
path: |
91+
/tmp/baipp/dist/*.whl
92+
/tmp/baipp/dist/*.tar.gz
93+
outputs:
94+
VERSION: ${{ steps.set_version.outputs.VERSION }}
95+
IS_PRERELEASE: ${{ steps.set_version.outputs.IS_PRERELEASE }}
96+
97+
publish_sdm:
98+
name: Publish SDM to DockerHub
99+
if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch'
100+
runs-on: ubuntu-latest
101+
needs: [build]
102+
environment:
103+
name: DockerHub
104+
url: https://hub.docker.com/r/airbyte/source-declarative-manifest/tags
105+
env:
106+
VERSION: ${{ needs.build.outputs.VERSION }}
107+
IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }}
108+
109+
steps:
110+
- uses: actions/checkout@v4
111+
with:
112+
fetch-depth: 0
113+
114+
# We need to download the build artifact again because the previous job was on a different runner
115+
- name: Download Build Artifact
116+
uses: actions/download-artifact@v4
117+
with:
118+
name: Packages-${{ github.run_id }}
119+
path: dist
120+
121+
- name: Set up QEMU for multi-platform builds
122+
uses: docker/setup-qemu-action@v3
123+
124+
- name: Set up Docker Buildx
125+
uses: docker/setup-buildx-action@v3
126+
127+
- name: Login to Docker Hub
128+
uses: docker/login-action@v3
129+
with:
130+
username: ${{ secrets.DOCKER_HUB_USERNAME }}
131+
password: ${{ secrets.DOCKER_HUB_PASSWORD }}
132+
133+
- name: "Check for existing tag (version: ${{ env.VERSION || 'none' }} )"
134+
if: env.VERSION != ''
135+
run: |
136+
tag="airbyte/source-declarative-manifest:${{ env.VERSION }}"
137+
if [ -z "$tag" ]; then
138+
echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'."
139+
exit 1
140+
fi
141+
echo "Checking if tag '$tag' exists on DockerHub..."
142+
if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$tag" > /dev/null 2>&1; then
143+
echo "The tag '$tag' already exists on DockerHub. Skipping publish to prevent overwrite."
144+
exit 1
145+
fi
146+
echo "No existing tag '$tag' found. Proceeding with publish."
147+
148+
- name: Build and push (sha tag)
149+
# Only run if the version is not set
150+
if: env.VERSION == '' && github.event.inputs.dry_run == 'false'
151+
uses: docker/build-push-action@v5
152+
with:
153+
context: .
154+
platforms: linux/amd64,linux/arm64
155+
push: true
156+
tags: |
157+
airbyte/source-declarative-manifest:${{ github.sha }}
158+
159+
- name: "Build and push (version tag: ${{ env.VERSION || 'none'}})"
160+
# Only run if the version is set
161+
if: env.VERSION != '' && github.event.inputs.dry_run == 'false'
162+
uses: docker/build-push-action@v5
163+
with:
164+
context: .
165+
platforms: linux/amd64,linux/arm64
166+
push: true
167+
tags: |
168+
airbyte/source-declarative-manifest:${{ env.VERSION }}
169+
170+
171+
- name: Build and push ('latest' tag)
172+
# Only run if version is set and IS_PRERELEASE is false
173+
if: env.VERSION != '' && env.IS_PRERELEASE == 'false' && github.event.inputs.dry_run == 'false'
174+
uses: docker/build-push-action@v5
175+
with:
176+
context: .
177+
platforms: linux/amd64,linux/arm64
178+
push: true
179+
tags: |
180+
airbyte/source-declarative-manifest:latest

‎.github/workflows/pypi_publish.yml

Lines changed: 30 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,54 @@
1-
name: Python Packaging
1+
# This workflow builds the python package.
2+
# On release tags, it also publishes to PyPI and DockerHub.
3+
# If we rename the workflow file name, we have to also update the
4+
# Trusted Publisher settings on PyPI.
5+
name: Packaging and Publishing
26

37
on:
48
push:
5-
69
workflow_dispatch:
10+
inputs:
11+
version:
12+
description: "The version to publish, ie 1.0.0 or 1.0.0-dev1"
13+
required: true
714

815
jobs:
916
build:
10-
name: Build
1117
runs-on: ubuntu-latest
1218
steps:
1319
- uses: actions/checkout@v4
1420
with:
1521
fetch-depth: 0
22+
ref: ${{ github.event_name == 'workflow_dispatch' && format('v{0}', github.event.inputs.version) || github.ref }}
23+
1624
- uses: hynek/build-and-inspect-python-package@v2
1725

26+
- uses: actions/upload-artifact@v4
27+
with:
28+
name: Packages-${{ github.run_id }}
29+
path: |
30+
/tmp/baipp/dist/*.whl
31+
/tmp/baipp/dist/*.tar.gz
32+
1833
publish:
19-
name: Publish to PyPI
34+
name: Publish CDK version to PyPI
2035
runs-on: ubuntu-latest
2136
needs: [build]
2237
permissions:
23-
id-token: write # IMPORTANT: this permission is mandatory for trusted publishing
24-
contents: write # Needed to upload artifacts to the release
38+
id-token: write
39+
contents: write
2540
environment:
26-
name: PyPI
27-
url: "https://pypi.org/p/airbyte-cdk"
28-
if: startsWith(github.ref, 'refs/tags/v')
41+
name: PyPi
42+
url: https://pypi.org/p/airbyte
43+
if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch'
2944
steps:
3045
- uses: actions/download-artifact@v4
3146
with:
32-
name: Packages
47+
name: Packages-${{ github.run_id }}
3348
path: dist
34-
- name: Attach Wheel to GitHub Release
49+
50+
- name: Upload wheel to release
51+
if: startsWith(github.ref, 'refs/tags/v')
3552
uses: svenstaro/upload-release-action@v2
3653
with:
3754
repo_token: ${{ secrets.GITHUB_TOKEN }}
@@ -40,8 +57,5 @@ jobs:
4057
overwrite: true
4158
file_glob: true
4259

43-
- name: Publish to PyPI (${{vars.PYPI_PUBLISH_URL}})
44-
uses: pypa/gh-action-pypi-publish@v1.12.2
45-
with:
46-
# Can be toggled at the repository level between `https://upload.pypi.org/legacy/` and `https://test.pypi.org/legacy/`
47-
repository-url: ${{vars.PYPI_PUBLISH_URL}}
60+
- name: Publish to PyPI
61+
uses: pypa/gh-action-pypi-publish@v1.10.3

‎.github/workflows/pytest_fast.yml

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,7 @@ on:
44
push:
55
branches:
66
- main
7-
paths:
8-
- 'airbyte_cdk/**'
9-
- 'poetry.lock'
10-
- 'pyproject.toml'
117
pull_request:
12-
paths:
13-
- 'airbyte_cdk/**'
14-
- 'poetry.lock'
15-
- 'pyproject.toml'
168

179
jobs:
1810
pytest-fast:

‎.github/workflows/pytest_matrix.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,13 @@ on:
1313
- main
1414
paths:
1515
- 'airbyte_cdk/**'
16+
- 'unit_tests/**'
1617
- 'poetry.lock'
1718
- 'pyproject.toml'
1819
pull_request:
1920
paths:
2021
- 'airbyte_cdk/**'
22+
- 'unit_tests/**'
2123
- 'poetry.lock'
2224
- 'pyproject.toml'
2325

‎.github/workflows/semantic_pr_check.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,5 +37,5 @@ jobs:
3737
Chore
3838
build
3939
Build
40-
test
41-
Test
40+
tests
41+
Tests

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,5 @@ dist
1212
.mypy_cache
1313
.venv
1414
.pytest_cache
15+
.idea
1516
**/__pycache__

‎Dockerfile

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
FROM docker.io/airbyte/python-connector-base:2.0.0@sha256:c44839ba84406116e8ba68722a0f30e8f6e7056c726f447681bb9e9ece8bd916
2+
3+
WORKDIR /airbyte/integration_code
4+
5+
# Copy project files needed for build
6+
COPY pyproject.toml poetry.lock README.md ./
7+
COPY dist/*.whl ./dist/
8+
9+
# Install dependencies - ignore keyring warnings
10+
RUN poetry config virtualenvs.create false \
11+
&& poetry install --only main --no-interaction --no-ansi || true
12+
13+
# Copy source code
14+
COPY airbyte_cdk ./airbyte_cdk
15+
16+
# Build and install the package
17+
RUN pip install dist/*.whl
18+
19+
# Set the entrypoint
20+
ENV AIRBYTE_ENTRYPOINT="source-declarative-manifest"
21+
ENTRYPOINT ["source-declarative-manifest"]

‎README.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,13 @@ Installing all extras is required to run the full suite of unit tests.
9393

9494
To see all available scripts, run `poetry run poe`.
9595

96+
#### Formatting the code
97+
98+
- Iterate on the CDK code locally
99+
- Run `poetry run ruff format` to format your changes.
100+
101+
To see all available `ruff` options, run `poetry run ruff`.
102+
96103
##### Autogenerated files
97104

98105
Low-code CDK models are generated from `sources/declarative/declarative_component_schema.yaml`. If

‎airbyte_cdk/cli/__init__.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# Copyright (c) 2024 Airbyte, Inc., all rights reserved.

‎airbyte_cdk/cli/source_declarative_manifest/__init__.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
from airbyte_cdk.cli.source_declarative_manifest._run import run
2+
3+
4+
__all__ = [
5+
"run",
6+
]

0 commit comments

Comments
 (0)