diff --git a/.github/workflows/publish_sdm_connector.yml b/.github/workflows/publish_sdm_connector.yml index e994314f8..3fde1d6b3 100644 --- a/.github/workflows/publish_sdm_connector.yml +++ b/.github/workflows/publish_sdm_connector.yml @@ -1,5 +1,6 @@ # This flow publishes the Source-Declarative-Manifest (SDM) # connector to DockerHub as a Docker image. +# TODO: Delete this workflow file once the unified publish flow is implemented and proven stable. name: Publish SDM Connector diff --git a/.github/workflows/pypi_publish.yml b/.github/workflows/pypi_publish.yml index 41329ce62..6325d4882 100644 --- a/.github/workflows/pypi_publish.yml +++ b/.github/workflows/pypi_publish.yml @@ -1,27 +1,98 @@ # This workflow builds the python package. -# On release tags, it also publishes to PyPI and DockerHub. -# If we rename the workflow file name, we have to also update the -# Trusted Publisher settings on PyPI. + +# When from from a release tags or a workflow dispatch, it also publishes to PyPI and DockerHub along +# with bumping the Connector Builder pinned version. +# Note: We may want to rename this file at some point. However, if we rename the workflow file name, +# we have to also update the Trusted Publisher settings on PyPI. + name: Packaging and Publishing on: push: + tags: + - 'v*' workflow_dispatch: inputs: version: - description: "The version to publish, ie 1.0.0 or 1.0.0-dev1" + description: "Version. The version to publish, ie 1.0.0 or 1.0.0-dev1. In most cases, you can leave this blank. If run from a release tag (recommended), the version number will be inferred from the git tag." + required: false + publish_to_pypi: + description: "Publish to PyPI. If true, the workflow will publish to PyPI." + type: boolean + required: true + default: true + publish_to_dockerhub: + description: "Publish to DockerHub. If true, the workflow will publish the SDM connector to DockerHub." + type: boolean required: true + default: true + update_connector_builder: + description: "Update Connector Builder. If true, the workflow will create a PR to bump the CDK version used by Connector Builder." + type: boolean + required: true + default: true jobs: build: + name: Build Python Package runs-on: ubuntu-latest steps: + - name: Detect Release Tag Version + if: startsWith(github.ref, 'refs/tags/v') + run: | + DETECTED_VERSION=${{ github.ref_name }} + echo "Version ref set to '${DETECTED_VERSION}'" + # Remove the 'v' prefix if it exists + DETECTED_VERSION="${DETECTED_VERSION#v}" + echo "Setting version to '$DETECTED_VERSION'" + echo "DETECTED_VERSION=${DETECTED_VERSION}" >> $GITHUB_ENV + + - name: Validate and set VERSION from git ref ('${{ github.ref_name }}') and input (${{ github.event.inputs.version || 'none' }}) + id: set_version + run: | + INPUT_VERSION=${{ github.event.inputs.version }} + echo "Version input set to '${INPUT_VERSION}'" + # Exit with success if both detected and input versions are empty + if [ -z "${DETECTED_VERSION:-}" ] && [ -z "${INPUT_VERSION:-}" ]; then + echo "No version detected or input. Will publish to SHA tag instead." + echo 'VERSION=' >> $GITHUB_ENV + exit 0 + fi + # Remove the 'v' prefix if it exists + INPUT_VERSION="${INPUT_VERSION#v}" + # Fail if detected version is non-empty and different from the input version + if [ -n "${DETECTED_VERSION:-}" ] && [ -n "${INPUT_VERSION:-}" ] && [ "${DETECTED_VERSION}" != "${INPUT_VERSION}" ]; then + echo "Error: Version input '${INPUT_VERSION}' does not match detected version '${DETECTED_VERSION}'." + exit 1 + fi + # Set the version to the input version if non-empty, otherwise the detected version + VERSION="${INPUT_VERSION:-$DETECTED_VERSION}" + # Fail if the version is still empty + if [ -z "$VERSION" ]; then + echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'." + exit 1 + fi + echo "Setting version to '$VERSION'" + echo "VERSION=${VERSION}" >> $GITHUB_ENV + echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT + # Check if version is a prerelease version (will not tag 'latest') + if [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "IS_PRERELEASE=false" >> $GITHUB_ENV + echo "IS_PRERELEASE=false" >> $GITHUB_OUTPUT + else + echo "IS_PRERELEASE=true" >> $GITHUB_ENV + echo "IS_PRERELEASE=true" >> $GITHUB_OUTPUT + fi + - uses: actions/checkout@v4 with: fetch-depth: 0 - ref: ${{ github.event_name == 'workflow_dispatch' && format('v{0}', github.event.inputs.version) || github.ref }} - uses: hynek/build-and-inspect-python-package@v2 + env: + # Pass in the evaluated version from the previous step + # More info: https://github.com/mtkennerly/poetry-dynamic-versioning#user-content-environment-variables + POETRY_DYNAMIC_VERSIONING_BYPASS: ${{ env.VERSION || '0.0.0dev0'}} - uses: actions/upload-artifact@v4 with: @@ -30,7 +101,11 @@ jobs: /tmp/baipp/dist/*.whl /tmp/baipp/dist/*.tar.gz - publish: + outputs: + VERSION: ${{ steps.set_version.outputs.VERSION }} + IS_PRERELEASE: ${{ steps.set_version.outputs.IS_PRERELEASE }} + + publish_cdk: name: Publish CDK version to PyPI runs-on: ubuntu-latest needs: [build] @@ -39,8 +114,11 @@ jobs: contents: write environment: name: PyPi - url: https://pypi.org/p/airbyte - if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch' + url: https://pypi.org/p/airbyte-cdk/ + if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || (github.event_name == 'workflow_dispatch' && (github.event.inputs.publish_to_pypi == 'true' || github.event.inputs.update_connector_builder == 'true')) + env: + VERSION: ${{ needs.build.outputs.VERSION }} + IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }} steps: - uses: actions/download-artifact@v4 with: @@ -58,4 +136,217 @@ jobs: file_glob: true - name: Publish to PyPI + if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || github.event.inputs.publish_to_pypi == 'true' uses: pypa/gh-action-pypi-publish@v1.10.3 + + publish_sdm: + name: Publish SDM to DockerHub + # TODO: When we're ready to publish after each release, prefix the `if` below with: + # (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || + # Until then, this workflow needs to be kicked off manually. + # Last remaining blocker documented here: https://github.com/airbytehq/airbyte-python-cdk/issues/64 + if: (github.event_name == 'workflow_dispatch' && github.event.inputs.publish_to_dockerhub == 'true') + runs-on: ubuntu-latest + needs: [build] + environment: + name: DockerHub + url: https://hub.docker.com/r/airbyte/source-declarative-manifest/tags + env: + VERSION: ${{ needs.build.outputs.VERSION }} + IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + # We need to download the build artifact again because the previous job was on a different runner + - name: Download Build Artifact + uses: actions/download-artifact@v4 + with: + name: Packages-${{ github.run_id }} + path: dist + + - name: Set up QEMU for multi-platform builds + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_PASSWORD }} + + - name: "Check for existing tag (version: ${{ env.VERSION || 'none' }} )" + if: env.VERSION != '' + run: | + tag="airbyte/source-declarative-manifest:${{ env.VERSION }}" + if [ -z "$tag" ]; then + echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'." + exit 1 + fi + echo "Checking if tag '$tag' exists on DockerHub..." + if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$tag" > /dev/null 2>&1; then + echo "The tag '$tag' already exists on DockerHub. Skipping publish to prevent overwrite." + exit 1 + fi + echo "No existing tag '$tag' found. Proceeding with publish." + + - name: "Build and push (sha tag: '${{ github.sha }}')" + # Only run if the version is not set + if: env.VERSION == '' + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: | + airbyte/source-declarative-manifest:${{ github.sha }} + + - name: "Build and push (version tag: ${{ env.VERSION || 'none'}})" + # Only run if the version is set + if: env.VERSION != '' + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: | + airbyte/source-declarative-manifest:${{ env.VERSION }} + + - name: Build and push ('latest' tag) + # Only run if version is set and IS_PRERELEASE is false + if: env.VERSION != '' && env.IS_PRERELEASE == 'false' + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: | + airbyte/source-declarative-manifest:latest + + update-connector-builder: + # Create a PR against the Builder, to update the CDK version that it uses. + # In the future, Builder may use the SDM docker image instead of the Python CDK package. + name: Bump Connector Builder CDK version + needs: + - build + - publish_cdk + if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || (github.event_name == 'workflow_dispatch' && github.event.inputs.update_connector_builder == 'true') + env: + VERSION: ${{ needs.build.outputs.VERSION }} + IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }} + runs-on: ubuntu-latest + steps: + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Checkout Airbyte Platform Internal + uses: actions/checkout@v4 + with: + repository: airbytehq/airbyte-platform-internal + token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + - name: Update Builder's CDK version to ${{ env.VERSION }} + # PyPI servers aren't immediately updated so we may need to retry a few times. + uses: nick-fields/retry@v3 + with: + shell: bash + max_attempts: 5 + retry_wait_seconds: 30 + timeout_minutes: 7 + command: | + set -euo pipefail + PREVIOUS_VERSION=$(cat oss/airbyte-connector-builder-resources/CDK_VERSION) + sed -i "s/${PREVIOUS_VERSION}/${VERSION}/g" "oss/airbyte-connector-builder-server/Dockerfile" + sed -i "s/${PREVIOUS_VERSION}/${VERSION}/g" "cloud/airbyte-connector-builder-server-wrapped/Dockerfile" + sed -i "s/airbyte-cdk==${PREVIOUS_VERSION}/airbyte-cdk==${VERSION}/g" oss/airbyte-connector-builder-server/requirements.in + echo ${VERSION} > oss/airbyte-connector-builder-resources/CDK_VERSION + cd oss/airbyte-connector-builder-server + python -m pip install --no-cache-dir pip-tools + pip-compile --upgrade + - name: Create Pull Request + id: create-pull-request + uses: peter-evans/create-pull-request@v6 + with: + token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + commit-message: "chore: update CDK version following release" + title: "chore: update CDK version following release" + body: This is an automatically generated PR triggered by a CDK release + branch: automatic-cdk-release + base: master + delete-branch: true + - name: Post success to Slack channel dev-connectors-extensibility + uses: slackapi/slack-github-action@v1.23.0 + continue-on-error: true + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + with: + channel-id: C04J1M66D8B + # Channel: #dev-connectors-extensibility-releases + # Link (internal): https://airbytehq-team.slack.com/archives/C04J1M66D8B + payload: | + { + "text": "A new version of Python CDK has been released!", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Python CDK `v${{ env.VERSION }}` has been \n\n" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "A PR has also been created for the <${{ steps.create-pull-request.outputs.pull-request-url }}|Connector Builder>\n" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "See details on \n" + } + } + ] + } + - name: Post failure to Slack channel dev-connectors-extensibility + if: failure() + uses: slackapi/slack-github-action@v1.23.0 + continue-on-error: true + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + with: + channel-id: C04J1M66D8B + # Channel: #dev-connectors-extensibility-releases + # Link (internal): https://airbytehq-team.slack.com/archives/C04J1M66D8B + payload: | + { + "text": ":warning: A new version of Python CDK has been released but Connector Builder hasn't been automatically updated", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Python CDK `v${{ env.VERSION }}` has been \n\n" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": ":warning: Could not automatically create a PR for Connector Builder>\n" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "See details on \n" + } + } + ] + } diff --git a/.github/workflows/pytest_fast.yml b/.github/workflows/pytest_fast.yml index edbbe3520..d3cb4a8d6 100644 --- a/.github/workflows/pytest_fast.yml +++ b/.github/workflows/pytest_fast.yml @@ -7,6 +7,25 @@ on: pull_request: jobs: + test-build: + name: Build and Inspect Python Package + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - uses: hynek/build-and-inspect-python-package@v2 + env: + # Pass in dummy version '0.0.0dev0' version to appease dynamic versioning + POETRY_DYNAMIC_VERSIONING_BYPASS: 0.0.0dev0 + + - uses: actions/upload-artifact@v4 + with: + name: Packages-${{ github.run_id }} + path: | + /tmp/baipp/dist/*.whl + /tmp/baipp/dist/*.tar.gz + pytest-fast: name: Pytest (Fast) runs-on: ubuntu-latest