From 87985923c6f4080f1717e51e15398da4533f6cb7 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 3 Dec 2024 16:30:47 -0800 Subject: [PATCH 01/14] ci: create unified cdk, sdm, and builder publish flow --- .github/workflows/publish_sdm_connector.yml | 1 + .github/workflows/pypi_publish.yml | 288 +++++++++++++++++++- 2 files changed, 283 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish_sdm_connector.yml b/.github/workflows/publish_sdm_connector.yml index e994314f8..3fde1d6b3 100644 --- a/.github/workflows/publish_sdm_connector.yml +++ b/.github/workflows/publish_sdm_connector.yml @@ -1,5 +1,6 @@ # This flow publishes the Source-Declarative-Manifest (SDM) # connector to DockerHub as a Docker image. +# TODO: Delete this workflow file once the unified publish flow is implemented and proven stable. name: Publish SDM Connector diff --git a/.github/workflows/pypi_publish.yml b/.github/workflows/pypi_publish.yml index 41329ce62..bc023df79 100644 --- a/.github/workflows/pypi_publish.yml +++ b/.github/workflows/pypi_publish.yml @@ -1,7 +1,9 @@ # This workflow builds the python package. -# On release tags, it also publishes to PyPI and DockerHub. -# If we rename the workflow file name, we have to also update the -# Trusted Publisher settings on PyPI. +# When from from a release tags or a workflow dispatch, it also publishes to PyPI and DockerHub along +# with bumping the Connector Builder pinned version. +# Note: We may want to rename this file at some point. However, if we rename the workflow file name, +# we have to also update the Trusted Publisher settings on PyPI. + name: Packaging and Publishing on: @@ -9,17 +11,78 @@ on: workflow_dispatch: inputs: version: - description: "The version to publish, ie 1.0.0 or 1.0.0-dev1" + description: "Version. The version to publish, ie 1.0.0 or 1.0.0-dev1. In most cases, you can leave this blank. If run from a release tag (recommended), the version number will be inferred from the git tag." + required: false + publish_to_pypi: + description: "Publish to PyPI. If true, the workflow will publish to PyPI." + type: boolean + required: true + default: true + publish_to_dockerhub: + description: "Publish to DockerHub. If true, the workflow will publish the SDM connector to DockerHub." + type: boolean + required: true + default: true + update_connector_builder: + description: "Update Connector Builder. If true, the workflow will create a PR to bump the CDK version used by Connector Builder." + type: boolean required: true + default: true jobs: build: runs-on: ubuntu-latest steps: + - name: Detect Release Tag Version + if: startsWith(github.ref, 'refs/tags/v') + run: | + DETECTED_VERSION=${{ github.ref_name }} + echo "Version ref set to '${DETECTED_VERSION}'" + # Remove the 'v' prefix if it exists + DETECTED_VERSION="${DETECTED_VERSION#v}" + echo "Setting version to '$DETECTED_VERSION'" + echo "DETECTED_VERSION=${DETECTED_VERSION}" >> $GITHUB_ENV + + - name: Validate and set VERSION from git ref ('${{ github.ref_name }}') and input (${{ github.event.inputs.version || 'none' }}) + id: set_version + run: | + INPUT_VERSION=${{ github.event.inputs.version }} + echo "Version input set to '${INPUT_VERSION}'" + # Exit with success if both detected and input versions are empty + if [ -z "${DETECTED_VERSION:-}" ] && [ -z "${INPUT_VERSION:-}" ]; then + echo "No version detected or input. Will publish to SHA tag instead." + echo 'VERSION=' >> $GITHUB_ENV + exit 0 + fi + # Remove the 'v' prefix if it exists + INPUT_VERSION="${INPUT_VERSION#v}" + # Fail if detected version is non-empty and different from the input version + if [ -n "${DETECTED_VERSION:-}" ] && [ -n "${INPUT_VERSION:-}" ] && [ "${DETECTED_VERSION}" != "${INPUT_VERSION}" ]; then + echo "Error: Version input '${INPUT_VERSION}' does not match detected version '${DETECTED_VERSION}'." + exit 1 + fi + # Set the version to the input version if non-empty, otherwise the detected version + VERSION="${INPUT_VERSION:-$DETECTED_VERSION}" + # Fail if the version is still empty + if [ -z "$VERSION" ]; then + echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'." + exit 1 + fi + echo "Setting version to '$VERSION'" + echo "VERSION=${VERSION}" >> $GITHUB_ENV + echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT + # Check if version is a prerelease version (will not tag 'latest') + if [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "IS_PRERELEASE=false" >> $GITHUB_ENV + echo "IS_PRERELEASE=false" >> $GITHUB_OUTPUT + else + echo "IS_PRERELEASE=true" >> $GITHUB_ENV + echo "IS_PRERELEASE=true" >> $GITHUB_OUTPUT + fi + - uses: actions/checkout@v4 with: fetch-depth: 0 - ref: ${{ github.event_name == 'workflow_dispatch' && format('v{0}', github.event.inputs.version) || github.ref }} - uses: hynek/build-and-inspect-python-package@v2 @@ -30,6 +93,10 @@ jobs: /tmp/baipp/dist/*.whl /tmp/baipp/dist/*.tar.gz + outputs: + VERSION: ${{ steps.set_version.outputs.VERSION }} + IS_PRERELEASE: ${{ steps.set_version.outputs.IS_PRERELEASE }} + publish: name: Publish CDK version to PyPI runs-on: ubuntu-latest @@ -40,7 +107,10 @@ jobs: environment: name: PyPi url: https://pypi.org/p/airbyte - if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch' + if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || (github.event_name == 'workflow_dispatch' && github.event.inputs.publish_to_pypi == 'true') + env: + VERSION: ${{ needs.build.outputs.VERSION }} + IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }} steps: - uses: actions/download-artifact@v4 with: @@ -59,3 +129,209 @@ jobs: - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@v1.10.3 + + publish_sdm: + name: Publish SDM to DockerHub + # TODO: When we're ready to publish after each release, prefix the `if` below with: + # (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || + # Until then, this workflow needs to be kicked off manually. + # Last remaining blocker documented here: https://github.com/airbytehq/airbyte-python-cdk/issues/64 + if: (github.event_name == 'workflow_dispatch' && github.event.inputs.publish_to_dockerhub == 'true') + runs-on: ubuntu-latest + needs: [build] + environment: + name: DockerHub + url: https://hub.docker.com/r/airbyte/source-declarative-manifest/tags + env: + VERSION: ${{ needs.build.outputs.VERSION }} + IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # We need to download the build artifact again because the previous job was on a different runner + - name: Download Build Artifact + uses: actions/download-artifact@v4 + with: + name: Packages-${{ github.run_id }} + path: dist + + - name: Set up QEMU for multi-platform builds + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_PASSWORD }} + + - name: "Check for existing tag (version: ${{ env.VERSION || 'none' }} )" + if: env.VERSION != '' + run: | + tag="airbyte/source-declarative-manifest:${{ env.VERSION }}" + if [ -z "$tag" ]; then + echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'." + exit 1 + fi + echo "Checking if tag '$tag' exists on DockerHub..." + if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$tag" > /dev/null 2>&1; then + echo "The tag '$tag' already exists on DockerHub. Skipping publish to prevent overwrite." + exit 1 + fi + echo "No existing tag '$tag' found. Proceeding with publish." + + - name: "Build and push (sha tag: '${{ github.sha }}')" + # Only run if the version is not set + if: env.VERSION == '' + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: | + airbyte/source-declarative-manifest:${{ github.sha }} + + - name: "Build and push (version tag: ${{ env.VERSION || 'none'}})" + # Only run if the version is set + if: env.VERSION != '' + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: | + airbyte/source-declarative-manifest:${{ env.VERSION }} + + - name: Build and push ('latest' tag) + # Only run if version is set and IS_PRERELEASE is false + if: env.VERSION != '' && env.IS_PRERELEASE == 'false' + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: | + airbyte/source-declarative-manifest:latest + + update-connector-builder: + # Create a PR against the Builder, to update the CDK version that it uses. + # In the future, Builder may use the SDM docker image instead of the Python CDK package. + needs: + - build + - publish + if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || (github.event_name == 'workflow_dispatch' && github.event.inputs.update_connector_builder == 'true') + env: + VERSION: ${{ needs.build.outputs.VERSION }} + IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }} + runs-on: ubuntu-latest + steps: + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Checkout Airbyte Platform Internal + uses: actions/checkout@v3 + with: + repository: airbytehq/airbyte-platform-internal + token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + - name: Update Builder's CDK version to ${{ env.VERSION }} + run: | + PREVIOUS_VERSION=$(cat oss/airbyte-connector-builder-resources/CDK_VERSION) + sed -i "s/${PREVIOUS_VERSION}/${VERSION}/g" oss/airbyte-connector-builder-server/Dockerfile + sed -i "s/${PREVIOUS_VERSION}/${VERSION}/g" cloud/airbyte-connector-builder-server-wrapped/Dockerfile + sed -i "s/airbyte-cdk==${PREVIOUS_VERSION}/airbyte-cdk==${VERSION}/g" oss/airbyte-connector-builder-server/requirements.in + echo ${VERSION} > oss/airbyte-connector-builder-resources/CDK_VERSION + cd oss/airbyte-connector-builder-server + pip install pip-tools + pip-compile --upgrade + - name: Create Pull Request + id: create-pull-request + uses: peter-evans/create-pull-request@v6 + with: + token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + commit-message: "chore: update CDK version following release" + title: "chore: update CDK version following release" + body: This is an automatically generated PR triggered by a CDK release + branch: automatic-cdk-release + base: master + delete-branch: true + - name: Post success to Slack channel dev-connectors-extensibility + uses: slackapi/slack-github-action@v1.23.0 + # TODO: Remove the 1 == 0 condition when we're ready to post to Slack + if: 1 == 0 + continue-on-error: true + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + with: + channel-id: C04J1M66D8B + # Channel: #dev-connectors-extensibility-releases + # Link (internal): https://airbytehq-team.slack.com/archives/C04J1M66D8B + payload: | + { + "text": "A new version of Python CDK has been released!", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "A new version of Python CDK has been released with : ${{ github.event.inputs.changelog-message }}\n\n" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "A PR has also been created for the <${{ steps.create-pull-request.outputs.pull-request-url }}|Connector Builder>\n" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "See details on \n" + } + } + ] + } + - name: Post failure to Slack channel dev-connectors-extensibility + # TODO: Remove the 1 == 0 condition when we're ready to post to Slack + if: ${{ failure() }} && 1 == 0 + uses: slackapi/slack-github-action@v1.23.0 + continue-on-error: true + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + with: + channel-id: C04J1M66D8B + # Channel: #dev-connectors-extensibility-releases + # Link (internal): https://airbytehq-team.slack.com/archives/C04J1M66D8B + payload: | + { + "text": ":warning: A new version of Python CDK has been released but Connector Builder hasn't been automatically updated", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "A new version of Python CDK has been released with : ${{ github.event.inputs.changelog-message }}\n\n" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": ":warning: Could not automatically create a PR for Connector Builder>\n" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "See details on \n" + } + } + ] + } From ead30551eecd7af7b454a588c8465caea04d0dac Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Tue, 3 Dec 2024 16:36:25 -0800 Subject: [PATCH 02/14] apply suggestion Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .github/workflows/pypi_publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pypi_publish.yml b/.github/workflows/pypi_publish.yml index bc023df79..2f4990be6 100644 --- a/.github/workflows/pypi_publish.yml +++ b/.github/workflows/pypi_publish.yml @@ -230,11 +230,11 @@ jobs: IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }} runs-on: ubuntu-latest steps: - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.10" - name: Checkout Airbyte Platform Internal - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: airbytehq/airbyte-platform-internal token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} From abc8940e610271cc2931642e1a915b23090ba508 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 3 Dec 2024 16:42:52 -0800 Subject: [PATCH 03/14] ci: add job names --- .github/workflows/pypi_publish.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pypi_publish.yml b/.github/workflows/pypi_publish.yml index 2f4990be6..bac3cb7e6 100644 --- a/.github/workflows/pypi_publish.yml +++ b/.github/workflows/pypi_publish.yml @@ -31,6 +31,7 @@ on: jobs: build: + name: Build Python Package runs-on: ubuntu-latest steps: - name: Detect Release Tag Version @@ -221,6 +222,7 @@ jobs: update-connector-builder: # Create a PR against the Builder, to update the CDK version that it uses. # In the future, Builder may use the SDM docker image instead of the Python CDK package. + name: Bump Connector Builder CDK version needs: - build - publish From 083ac3b32b830427655178578f052bd0e65b0ae9 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 3 Dec 2024 17:27:42 -0800 Subject: [PATCH 04/14] try always condition for builder CDK bump --- .github/workflows/pypi_publish.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pypi_publish.yml b/.github/workflows/pypi_publish.yml index bac3cb7e6..566c2b21b 100644 --- a/.github/workflows/pypi_publish.yml +++ b/.github/workflows/pypi_publish.yml @@ -98,7 +98,7 @@ jobs: VERSION: ${{ steps.set_version.outputs.VERSION }} IS_PRERELEASE: ${{ steps.set_version.outputs.IS_PRERELEASE }} - publish: + publish_cdk: name: Publish CDK version to PyPI runs-on: ubuntu-latest needs: [build] @@ -225,8 +225,8 @@ jobs: name: Bump Connector Builder CDK version needs: - build - - publish - if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || (github.event_name == 'workflow_dispatch' && github.event.inputs.update_connector_builder == 'true') + - publish_cdk + if: always() && (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || (github.event_name == 'workflow_dispatch' && github.event.inputs.update_connector_builder == 'true') env: VERSION: ${{ needs.build.outputs.VERSION }} IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }} From 00e36a9409ee20e59f4bdaa3d2f89c84fc7bef40 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 3 Dec 2024 17:32:06 -0800 Subject: [PATCH 05/14] allow connector builder cdk bump when pypi publish is disabled --- .github/workflows/pypi_publish.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pypi_publish.yml b/.github/workflows/pypi_publish.yml index 566c2b21b..5f9223c89 100644 --- a/.github/workflows/pypi_publish.yml +++ b/.github/workflows/pypi_publish.yml @@ -108,7 +108,7 @@ jobs: environment: name: PyPi url: https://pypi.org/p/airbyte - if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || (github.event_name == 'workflow_dispatch' && github.event.inputs.publish_to_pypi == 'true') + if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || (github.event_name == 'workflow_dispatch' && (github.event.inputs.publish_to_pypi == 'true' || github.event.inputs.update_connector_builder == 'true')) env: VERSION: ${{ needs.build.outputs.VERSION }} IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }} @@ -129,6 +129,7 @@ jobs: file_glob: true - name: Publish to PyPI + if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || github.event.inputs.publish_to_pypi == 'true' uses: pypa/gh-action-pypi-publish@v1.10.3 publish_sdm: From 6ee210f0c3eec55e8a6f48093310eb76e19d7e68 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 3 Dec 2024 17:51:53 -0800 Subject: [PATCH 06/14] inject detected version into poetry dynamic versioning --- .github/workflows/pypi_publish.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pypi_publish.yml b/.github/workflows/pypi_publish.yml index 5f9223c89..48726bb19 100644 --- a/.github/workflows/pypi_publish.yml +++ b/.github/workflows/pypi_publish.yml @@ -86,6 +86,10 @@ jobs: fetch-depth: 0 - uses: hynek/build-and-inspect-python-package@v2 + env: + # Pass in the evaluated version from the previous step + # More info: https://github.com/mtkennerly/poetry-dynamic-versioning#user-content-environment-variables + POETRY_DYNAMIC_VERSIONING_BYPASS: ${{ env.VERSION || '0.0.0dev0'}} - uses: actions/upload-artifact@v4 with: From 798e9cc6fee17a97f3075dd5424813f38548371d Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 3 Dec 2024 18:02:01 -0800 Subject: [PATCH 07/14] add retries to deal with PyPI delayed availability --- .github/workflows/pypi_publish.yml | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pypi_publish.yml b/.github/workflows/pypi_publish.yml index 48726bb19..28d56f719 100644 --- a/.github/workflows/pypi_publish.yml +++ b/.github/workflows/pypi_publish.yml @@ -231,7 +231,7 @@ jobs: needs: - build - publish_cdk - if: always() && (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || (github.event_name == 'workflow_dispatch' && github.event.inputs.update_connector_builder == 'true') + if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || (github.event_name == 'workflow_dispatch' && github.event.inputs.update_connector_builder == 'true') env: VERSION: ${{ needs.build.outputs.VERSION }} IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }} @@ -246,15 +246,21 @@ jobs: repository: airbytehq/airbyte-platform-internal token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - name: Update Builder's CDK version to ${{ env.VERSION }} - run: | - PREVIOUS_VERSION=$(cat oss/airbyte-connector-builder-resources/CDK_VERSION) - sed -i "s/${PREVIOUS_VERSION}/${VERSION}/g" oss/airbyte-connector-builder-server/Dockerfile - sed -i "s/${PREVIOUS_VERSION}/${VERSION}/g" cloud/airbyte-connector-builder-server-wrapped/Dockerfile - sed -i "s/airbyte-cdk==${PREVIOUS_VERSION}/airbyte-cdk==${VERSION}/g" oss/airbyte-connector-builder-server/requirements.in - echo ${VERSION} > oss/airbyte-connector-builder-resources/CDK_VERSION - cd oss/airbyte-connector-builder-server - pip install pip-tools - pip-compile --upgrade + # PyPI servers aren't immediately updated so we may need to retry a few times. + uses: nick-fields/retry@v3 + with: + shell: bash + max_attempts: 5 + retry_wait_seconds: 30 + command: | + PREVIOUS_VERSION=$(cat oss/airbyte-connector-builder-resources/CDK_VERSION) + sed -i "s/${PREVIOUS_VERSION}/${VERSION}/g" oss/airbyte-connector-builder-server/Dockerfile + sed -i "s/${PREVIOUS_VERSION}/${VERSION}/g" cloud/airbyte-connector-builder-server-wrapped/Dockerfile + sed -i "s/airbyte-cdk==${PREVIOUS_VERSION}/airbyte-cdk==${VERSION}/g" oss/airbyte-connector-builder-server/requirements.in + echo ${VERSION} > oss/airbyte-connector-builder-resources/CDK_VERSION + cd oss/airbyte-connector-builder-server + pip install pip-tools + pip-compile --upgrade - name: Create Pull Request id: create-pull-request uses: peter-evans/create-pull-request@v6 From c35575fae83f9ee19703b32e7d426110abb17854 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 3 Dec 2024 18:06:16 -0800 Subject: [PATCH 08/14] add 7 minute timeout --- .github/workflows/pypi_publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pypi_publish.yml b/.github/workflows/pypi_publish.yml index 28d56f719..33832bac6 100644 --- a/.github/workflows/pypi_publish.yml +++ b/.github/workflows/pypi_publish.yml @@ -252,6 +252,7 @@ jobs: shell: bash max_attempts: 5 retry_wait_seconds: 30 + timeout_minutes: 7 command: | PREVIOUS_VERSION=$(cat oss/airbyte-connector-builder-resources/CDK_VERSION) sed -i "s/${PREVIOUS_VERSION}/${VERSION}/g" oss/airbyte-connector-builder-server/Dockerfile From 5f8f2b887b600ad8648570ea89221f831c5490bd Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 3 Dec 2024 18:13:22 -0800 Subject: [PATCH 09/14] allow success publish to slack --- .github/workflows/pypi_publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pypi_publish.yml b/.github/workflows/pypi_publish.yml index 33832bac6..b97c94793 100644 --- a/.github/workflows/pypi_publish.yml +++ b/.github/workflows/pypi_publish.yml @@ -1,4 +1,5 @@ # This workflow builds the python package. + # When from from a release tags or a workflow dispatch, it also publishes to PyPI and DockerHub along # with bumping the Connector Builder pinned version. # Note: We may want to rename this file at some point. However, if we rename the workflow file name, @@ -276,7 +277,6 @@ jobs: - name: Post success to Slack channel dev-connectors-extensibility uses: slackapi/slack-github-action@v1.23.0 # TODO: Remove the 1 == 0 condition when we're ready to post to Slack - if: 1 == 0 continue-on-error: true env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} @@ -313,7 +313,7 @@ jobs: } - name: Post failure to Slack channel dev-connectors-extensibility # TODO: Remove the 1 == 0 condition when we're ready to post to Slack - if: ${{ failure() }} && 1 == 0 + if: failure() uses: slackapi/slack-github-action@v1.23.0 continue-on-error: true env: From 4ed4612c76ea5161bdf0dde7acb7882dafe5783a Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 3 Dec 2024 18:24:49 -0800 Subject: [PATCH 10/14] fix url, changelog, and version text in slack notifications --- .github/workflows/pypi_publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pypi_publish.yml b/.github/workflows/pypi_publish.yml index b97c94793..781ba69f4 100644 --- a/.github/workflows/pypi_publish.yml +++ b/.github/workflows/pypi_publish.yml @@ -292,7 +292,7 @@ jobs: "type": "section", "text": { "type": "mrkdwn", - "text": "A new version of Python CDK has been released with : ${{ github.event.inputs.changelog-message }}\n\n" + "text": "Python CDK `v${{ env.VERSION }}` has been \n\n" } }, { @@ -330,7 +330,7 @@ jobs: "type": "section", "text": { "type": "mrkdwn", - "text": "A new version of Python CDK has been released with : ${{ github.event.inputs.changelog-message }}\n\n" + "text": "Python CDK `v${{ env.VERSION }}` has been \n\n" } }, { From a975e068b4f804f64adf115caefbfd85d4c489e4 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 3 Dec 2024 18:27:43 -0800 Subject: [PATCH 11/14] fix pypi environment url --- .github/workflows/pypi_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi_publish.yml b/.github/workflows/pypi_publish.yml index 781ba69f4..036b1bc60 100644 --- a/.github/workflows/pypi_publish.yml +++ b/.github/workflows/pypi_publish.yml @@ -112,7 +112,7 @@ jobs: contents: write environment: name: PyPi - url: https://pypi.org/p/airbyte + url: https://pypi.org/p/airbyte-cdk/ if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || (github.event_name == 'workflow_dispatch' && (github.event.inputs.publish_to_pypi == 'true' || github.event.inputs.update_connector_builder == 'true')) env: VERSION: ${{ needs.build.outputs.VERSION }} From 0c69e9ef939ba19283b25694882cc8aa7bcf3f25 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 3 Dec 2024 18:36:05 -0800 Subject: [PATCH 12/14] only run Publishing workflow on tag pushes; move generic build and package validation to pytest-fast --- .github/workflows/pypi_publish.yml | 2 ++ .github/workflows/pytest_fast.yml | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/.github/workflows/pypi_publish.yml b/.github/workflows/pypi_publish.yml index 036b1bc60..2104e3b6f 100644 --- a/.github/workflows/pypi_publish.yml +++ b/.github/workflows/pypi_publish.yml @@ -9,6 +9,8 @@ name: Packaging and Publishing on: push: + tags: + - 'v*' workflow_dispatch: inputs: version: diff --git a/.github/workflows/pytest_fast.yml b/.github/workflows/pytest_fast.yml index edbbe3520..d3cb4a8d6 100644 --- a/.github/workflows/pytest_fast.yml +++ b/.github/workflows/pytest_fast.yml @@ -7,6 +7,25 @@ on: pull_request: jobs: + test-build: + name: Build and Inspect Python Package + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - uses: hynek/build-and-inspect-python-package@v2 + env: + # Pass in dummy version '0.0.0dev0' version to appease dynamic versioning + POETRY_DYNAMIC_VERSIONING_BYPASS: 0.0.0dev0 + + - uses: actions/upload-artifact@v4 + with: + name: Packages-${{ github.run_id }} + path: | + /tmp/baipp/dist/*.whl + /tmp/baipp/dist/*.tar.gz + pytest-fast: name: Pytest (Fast) runs-on: ubuntu-latest From 074036fe88181b9cd70414c680ff4157a1214b22 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 3 Dec 2024 18:39:46 -0800 Subject: [PATCH 13/14] apply code rabbit suggestions --- .github/workflows/pypi_publish.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pypi_publish.yml b/.github/workflows/pypi_publish.yml index 2104e3b6f..34de3bbe3 100644 --- a/.github/workflows/pypi_publish.yml +++ b/.github/workflows/pypi_publish.yml @@ -257,13 +257,14 @@ jobs: retry_wait_seconds: 30 timeout_minutes: 7 command: | + set -euo pipefail PREVIOUS_VERSION=$(cat oss/airbyte-connector-builder-resources/CDK_VERSION) - sed -i "s/${PREVIOUS_VERSION}/${VERSION}/g" oss/airbyte-connector-builder-server/Dockerfile - sed -i "s/${PREVIOUS_VERSION}/${VERSION}/g" cloud/airbyte-connector-builder-server-wrapped/Dockerfile + sed -i "s/${PREVIOUS_VERSION}/${VERSION}/g" "oss/airbyte-connector-builder-server/Dockerfile" + sed -i "s/${PREVIOUS_VERSION}/${VERSION}/g" "cloud/airbyte-connector-builder-server-wrapped/Dockerfile" sed -i "s/airbyte-cdk==${PREVIOUS_VERSION}/airbyte-cdk==${VERSION}/g" oss/airbyte-connector-builder-server/requirements.in echo ${VERSION} > oss/airbyte-connector-builder-resources/CDK_VERSION cd oss/airbyte-connector-builder-server - pip install pip-tools + python -m pip install --no-cache-dir pip-tools pip-compile --upgrade - name: Create Pull Request id: create-pull-request From 8ba4bea3c94ba0c3d5f73ef84b7da0d8e65ce96d Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Tue, 3 Dec 2024 18:41:52 -0800 Subject: [PATCH 14/14] resolve todo comments --- .github/workflows/pypi_publish.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/pypi_publish.yml b/.github/workflows/pypi_publish.yml index 34de3bbe3..6325d4882 100644 --- a/.github/workflows/pypi_publish.yml +++ b/.github/workflows/pypi_publish.yml @@ -279,7 +279,6 @@ jobs: delete-branch: true - name: Post success to Slack channel dev-connectors-extensibility uses: slackapi/slack-github-action@v1.23.0 - # TODO: Remove the 1 == 0 condition when we're ready to post to Slack continue-on-error: true env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} @@ -315,7 +314,6 @@ jobs: ] } - name: Post failure to Slack channel dev-connectors-extensibility - # TODO: Remove the 1 == 0 condition when we're ready to post to Slack if: failure() uses: slackapi/slack-github-action@v1.23.0 continue-on-error: true