Skip to content

Commit 5d3dc2e

Browse files
aaronsteersaaronsteers
authored
Merge branch 'main' into aj/chore/add-ruff-lint-rules-2
2 parents 6460550 + c347760 commit 5d3dc2e

File tree

339 files changed

+7026
-2278
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

339 files changed

+7026
-2278
lines changed

‎.github/dependabot.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,10 @@ updates:
1313
interval: daily
1414
labels:
1515
- chore
16-
open-pull-requests-limit: 8 # default is 5
16+
open-pull-requests-limit: 8 # default is 5
1717

1818
- package-ecosystem: github-actions
19-
open-pull-requests-limit: 5 # default is 5
19+
open-pull-requests-limit: 5 # default is 5
2020
directory: "/"
2121
commit-message:
2222
prefix: "ci(deps): "
@@ -29,5 +29,5 @@ updates:
2929
minor-and-patch:
3030
applies-to: version-updates
3131
update-types:
32-
- patch
33-
- minor
32+
- patch
33+
- minor

‎.github/release-drafter.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ categories:
1515
- "chore"
1616
- "ci"
1717
- "refactor"
18+
- "testing"
1819
- title: "Documentation 📖"
1920
label: "docs"
2021
change-template: "- $TITLE (#$NUMBER)"
@@ -38,6 +39,9 @@ autolabeler:
3839
- label: "chore"
3940
title:
4041
- '/^chore(\(.*\))?\:/i'
42+
- label: "testing"
43+
title:
44+
- '/^tests(\(.*\))?\:/i'
4145
- label: "ci"
4246
title:
4347
- '/^ci(\(.*\))?\:/i'

‎.github/workflows/connector-tests.yml

Lines changed: 70 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -68,50 +68,103 @@ jobs:
6868
fail-fast: false
6969
matrix:
7070
include:
71+
- connector: source-hardcoded-records
72+
cdk_extra: n/a
7173
- connector: source-shopify
7274
cdk_extra: n/a
73-
# Currently not passing CI (unrelated)
74-
# - connector: source-zendesk-support
75+
# Chargebee is being flaky:
76+
# - connector: source-chargebee
7577
# cdk_extra: n/a
76-
- connector: source-s3
77-
cdk_extra: file-based
78-
- connector: destination-pinecone
79-
cdk_extra: vector-db-based
78+
# These two are behind in CDK updates and can't be used as tests until they are updated:
79+
# - connector: source-s3
80+
# cdk_extra: file-based
81+
# - connector: destination-pinecone
82+
# cdk_extra: vector-db-based
8083
- connector: destination-motherduck
8184
cdk_extra: sql
85+
# ZenDesk currently failing (as of 2024-12-02)
86+
# TODO: Re-enable once fixed
87+
# - connector: source-zendesk-support
88+
# cdk_extra: n/a
89+
# TODO: These are manifest connectors and won't work as expected until we
90+
# add `--use-local-cdk` support for manifest connectors.
91+
# - connector: source-the-guardian-api
92+
# cdk_extra: n/a
93+
# - connector: source-pokeapi
94+
# cdk_extra: n/a
8295

83-
name: "Check: '${{matrix.connector}}' (skip=${{needs.cdk_changes.outputs[matrix.cdk_extra] == 'false'}})"
96+
name: "Check: '${{matrix.connector}}' (skip=${{needs.cdk_changes.outputs['src'] == 'false' || needs.cdk_changes.outputs[matrix.cdk_extra] == 'false'}})"
8497
steps:
8598
- name: Abort if extra not changed (${{matrix.cdk_extra}})
8699
id: no_changes
87-
if: ${{ matrix.cdk_extra != 'n/a' && needs.cdk_changes.outputs[matrix.cdk_extra] == 'false' }}
100+
if: ${{ needs.cdk_changes.outputs['src'] == 'false' || matrix.cdk_extra != 'n/a' && needs.cdk_changes.outputs[matrix.cdk_extra] == 'false' }}
88101
run: |
89-
echo "Aborting job as specified extra not changed: ${{matrix.cdk_extra}} = ${{ needs.cdk_changes.outputs[matrix.cdk_extra] }}"
102+
echo "Aborting job."
103+
echo "Source code changed: ${{ needs.cdk_changes.outputs['src'] }}"
104+
if [ "${{ matrix.cdk_extra }}" != "n/a" ]; then
105+
echo "Extra not changed: ${{ matrix.cdk_extra }} = ${{ needs.cdk_changes.outputs[matrix.cdk_extra] }}"
106+
fi
107+
echo "> Skipped '${{matrix.connector}}' (no relevant changes)" >> $GITHUB_STEP_SUMMARY
90108
echo "status=cancelled" >> $GITHUB_OUTPUT
91-
exit 1
109+
exit 0
92110
continue-on-error: true
93111
# Get the monorepo so we can test the connectors
94112
- name: Checkout CDK
95-
if: steps.no_changes.outcome != 'failure'
113+
if: steps.no_changes.outputs.status != 'cancelled'
96114
uses: actions/checkout@v4
97115
with:
98116
path: airbyte-python-cdk
99117
- name: Checkout Airbyte Monorepo
100118
uses: actions/checkout@v4
101-
if: steps.no_changes.outcome != 'failure'
119+
if: steps.no_changes.outputs.status != 'cancelled'
102120
with:
103121
repository: airbytehq/airbyte
104122
ref: master
105123
path: airbyte
106124
- name: Test Connector
107-
if: steps.no_changes.outcome != 'failure'
125+
if: steps.no_changes.outputs.status != 'cancelled'
108126
timeout-minutes: 90
109127
env:
110128
GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
129+
POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0"
130+
# TODO: Revert below to use `tools.airbyte-ci-binary.install` after Airbyte CI released:
111131
run: |
112132
cd airbyte
113-
make tools.airbyte-ci-binary.install
114-
airbyte-ci connectors \
133+
make tools.airbyte-ci-dev.install
134+
airbyte-ci-dev connectors \
115135
--name ${{matrix.connector}} \
116-
test
117-
--global-status-check-context='Connectors Test: ${{matrix.connector}}'"
136+
--use-local-cdk \
137+
test \
138+
--fail-fast \
139+
--skip-step qa_checks \
140+
--skip-step connector_live_tests
141+
142+
- name: Evaluate Test Output
143+
if: always() && steps.no_changes.outputs.status != 'cancelled'
144+
run: |
145+
# save job output json file as ci step output
146+
json_output_file=$(find airbyte/airbyte-ci/connectors/pipelines/pipeline_reports -name 'output.json' -print -quit)
147+
job_output=$(cat ${json_output_file})
148+
success=$(echo ${job_output} | jq -r '.success')
149+
failed_step=$(echo ${job_output} | jq -r '.failed_steps | select(length > 0) | .[0] // "None"')
150+
run_duration=$(echo ${job_output} | jq -r '.run_duration')
151+
echo "## Job Output for ${{matrix.connector}}" >> $GITHUB_STEP_SUMMARY
152+
echo "- Success: ${success}" >> $GITHUB_STEP_SUMMARY
153+
echo "- Test Duration: $(printf "%.0f" ${run_duration})s" >> $GITHUB_STEP_SUMMARY
154+
if [ "${success}" != "true" ]; then
155+
echo "- Failed Step: ${failed_step}" >> $GITHUB_STEP_SUMMARY
156+
fi
157+
echo -e "\n[Download Job Output](${{steps.upload_job_output.outputs.artifact-url}})" >> $GITHUB_STEP_SUMMARY
158+
if [ "${success}" != "true" ]; then
159+
echo "::error::Test failed for connector '${{ matrix.connector }}' on step '${failed_step}'. Check the logs for more details."
160+
exit 1
161+
fi
162+
163+
# Upload the job output to the artifacts
164+
- name: Upload Job Output
165+
id: upload_job_output
166+
if: always() && steps.no_changes.outputs.status != 'cancelled'
167+
uses: actions/upload-artifact@v4
168+
with:
169+
name: ${{matrix.connector}}-job-output
170+
path: airbyte/airbyte-ci/connectors/pipelines/pipeline_reports

‎.github/workflows/publish_sdm_connector.yml

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
# This flow publishes the Source-Declarative-Manifest (SDM)
2+
# connector to DockerHub as a Docker image.
3+
# TODO: Delete this workflow file once the unified publish flow is implemented and proven stable.
4+
5+
name: Publish SDM Connector
6+
7+
on:
8+
workflow_dispatch:
9+
inputs:
10+
version:
11+
description: The version to publish, ie 1.0.0 or 1.0.0-dev1.
12+
If omitted, and if run from a release branch, the version will be
13+
inferred from the git tag.
14+
If omitted, and if run from a non-release branch, then only a SHA-based
15+
Docker tag will be created.
16+
required: false
17+
dry_run:
18+
description: If true, the workflow will not push to DockerHub.
19+
type: boolean
20+
required: false
21+
default: false
22+
23+
jobs:
24+
build:
25+
runs-on: ubuntu-latest
26+
steps:
27+
- name: Detect Release Tag Version
28+
if: startsWith(github.ref, 'refs/tags/v')
29+
run: |
30+
DETECTED_VERSION=${{ github.ref_name }}
31+
echo "Version ref set to '${DETECTED_VERSION}'"
32+
# Remove the 'v' prefix if it exists
33+
DETECTED_VERSION="${DETECTED_VERSION#v}"
34+
echo "Setting version to '$DETECTED_VERSION'"
35+
echo "DETECTED_VERSION=${DETECTED_VERSION}" >> $GITHUB_ENV
36+
37+
- name: Validate and set VERSION from tag ('${{ github.ref_name }}') and input (${{ github.event.inputs.version || 'none' }})
38+
id: set_version
39+
if: github.event_name == 'workflow_dispatch'
40+
run: |
41+
INPUT_VERSION=${{ github.event.inputs.version }}
42+
echo "Version input set to '${INPUT_VERSION}'"
43+
# Exit with success if both detected and input versions are empty
44+
if [ -z "${DETECTED_VERSION:-}" ] && [ -z "${INPUT_VERSION:-}" ]; then
45+
echo "No version detected or input. Will publish to SHA tag instead."
46+
echo 'VERSION=' >> $GITHUB_ENV
47+
exit 0
48+
fi
49+
# Remove the 'v' prefix if it exists
50+
INPUT_VERSION="${INPUT_VERSION#v}"
51+
# Fail if detected version is non-empty and different from the input version
52+
if [ -n "${DETECTED_VERSION:-}" ] && [ -n "${INPUT_VERSION:-}" ] && [ "${DETECTED_VERSION}" != "${INPUT_VERSION}" ]; then
53+
echo "Error: Version input '${INPUT_VERSION}' does not match detected version '${DETECTED_VERSION}'."
54+
exit 1
55+
fi
56+
# Set the version to the input version if non-empty, otherwise the detected version
57+
VERSION="${INPUT_VERSION:-$DETECTED_VERSION}"
58+
# Fail if the version is still empty
59+
if [ -z "$VERSION" ]; then
60+
echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'."
61+
exit 1
62+
fi
63+
echo "Setting version to '$VERSION'"
64+
echo "VERSION=${VERSION}" >> $GITHUB_ENV
65+
echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT
66+
# Check if version is a prerelease version (will not tag 'latest')
67+
if [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
68+
echo "IS_PRERELEASE=false" >> $GITHUB_ENV
69+
echo "IS_PRERELEASE=false" >> $GITHUB_OUTPUT
70+
else
71+
echo "IS_PRERELEASE=true" >> $GITHUB_ENV
72+
echo "IS_PRERELEASE=true" >> $GITHUB_OUTPUT
73+
fi
74+
75+
- uses: actions/checkout@v4
76+
with:
77+
fetch-depth: 0
78+
79+
- uses: hynek/build-and-inspect-python-package@v2
80+
name: Build package with version ref '${{ env.VERSION || '0.0.0dev0' }}'
81+
env:
82+
# Pass in the evaluated version from the previous step
83+
# More info: https://github.com/mtkennerly/poetry-dynamic-versioning#user-content-environment-variables
84+
POETRY_DYNAMIC_VERSIONING_BYPASS: ${{ env.VERSION || '0.0.0dev0'}}
85+
86+
- uses: actions/upload-artifact@v4
87+
with:
88+
name: Packages-${{ github.run_id }}
89+
path: |
90+
/tmp/baipp/dist/*.whl
91+
/tmp/baipp/dist/*.tar.gz
92+
outputs:
93+
VERSION: ${{ steps.set_version.outputs.VERSION }}
94+
IS_PRERELEASE: ${{ steps.set_version.outputs.IS_PRERELEASE }}
95+
96+
publish_sdm:
97+
name: Publish SDM to DockerHub
98+
if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch'
99+
runs-on: ubuntu-latest
100+
needs: [build]
101+
environment:
102+
name: DockerHub
103+
url: https://hub.docker.com/r/airbyte/source-declarative-manifest/tags
104+
env:
105+
VERSION: ${{ needs.build.outputs.VERSION }}
106+
IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }}
107+
108+
steps:
109+
- uses: actions/checkout@v4
110+
with:
111+
fetch-depth: 0
112+
113+
# We need to download the build artifact again because the previous job was on a different runner
114+
- name: Download Build Artifact
115+
uses: actions/download-artifact@v4
116+
with:
117+
name: Packages-${{ github.run_id }}
118+
path: dist
119+
120+
- name: Set up QEMU for multi-platform builds
121+
uses: docker/setup-qemu-action@v3
122+
123+
- name: Set up Docker Buildx
124+
uses: docker/setup-buildx-action@v3
125+
126+
- name: Login to Docker Hub
127+
uses: docker/login-action@v3
128+
with:
129+
username: ${{ secrets.DOCKER_HUB_USERNAME }}
130+
password: ${{ secrets.DOCKER_HUB_PASSWORD }}
131+
132+
- name: "Check for existing tag (version: ${{ env.VERSION || 'none' }} )"
133+
if: env.VERSION != ''
134+
run: |
135+
tag="airbyte/source-declarative-manifest:${{ env.VERSION }}"
136+
if [ -z "$tag" ]; then
137+
echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'."
138+
exit 1
139+
fi
140+
echo "Checking if tag '$tag' exists on DockerHub..."
141+
if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$tag" > /dev/null 2>&1; then
142+
echo "The tag '$tag' already exists on DockerHub. Skipping publish to prevent overwrite."
143+
exit 1
144+
fi
145+
echo "No existing tag '$tag' found. Proceeding with publish."
146+
147+
- name: Build and push (sha tag)
148+
# Only run if the version is not set
149+
if: env.VERSION == '' && github.event.inputs.dry_run == 'false'
150+
uses: docker/build-push-action@v5
151+
with:
152+
context: .
153+
platforms: linux/amd64,linux/arm64
154+
push: true
155+
tags: |
156+
airbyte/source-declarative-manifest:${{ github.sha }}
157+
158+
- name: "Build and push (version tag: ${{ env.VERSION || 'none'}})"
159+
# Only run if the version is set
160+
if: env.VERSION != '' && github.event.inputs.dry_run == 'false'
161+
uses: docker/build-push-action@v5
162+
with:
163+
context: .
164+
platforms: linux/amd64,linux/arm64
165+
push: true
166+
tags: |
167+
airbyte/source-declarative-manifest:${{ env.VERSION }}
168+
169+
- name: Build and push ('latest' tag)
170+
# Only run if version is set and IS_PRERELEASE is false
171+
if: env.VERSION != '' && env.IS_PRERELEASE == 'false' && github.event.inputs.dry_run == 'false'
172+
uses: docker/build-push-action@v5
173+
with:
174+
context: .
175+
platforms: linux/amd64,linux/arm64
176+
push: true
177+
tags: |
178+
airbyte/source-declarative-manifest:latest

0 commit comments

Comments
 (0)