From a557849e054327468f4bc15db2c8ba1c01c7c9b3 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Fri, 22 Nov 2024 10:00:22 -0800 Subject: [PATCH 1/7] move sdm image publish to dedicated step --- .github/workflows/publish_sdm_connector.yml | 159 ++++++++++++++++++++ .github/workflows/pypi_publish.yml | 78 ---------- 2 files changed, 159 insertions(+), 78 deletions(-) create mode 100644 .github/workflows/publish_sdm_connector.yml diff --git a/.github/workflows/publish_sdm_connector.yml b/.github/workflows/publish_sdm_connector.yml new file mode 100644 index 000000000..7b173034e --- /dev/null +++ b/.github/workflows/publish_sdm_connector.yml @@ -0,0 +1,159 @@ +# This flow publishes the Source-Declarative-Manifest (SDM) +# connector to DockerHub as a Docker image. + +name: Publish SDM Connector + +on: + workflow_dispatch: + inputs: + version: + description: + The version to publish, ie 1.0.0 or 1.0.0-dev1. + If omitted, and if run from a release branch, the version will be + inferred from the git tag. + If omitted, and if run from a non-release branch, then only a SHA-based + Docker tag will be created. + required: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: hynek/build-and-inspect-python-package@v2 + + - uses: actions/upload-artifact@v4 + with: + name: Packages-${{ github.run_id }} + path: | + /tmp/baipp/dist/*.whl + /tmp/baipp/dist/*.tar.gz + + publish_sdm: + name: Publish SDM to DockerHub + if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + needs: [build] + environment: + name: DockerHub + url: https://hub.docker.com/r/airbyte/source-declarative-manifest/tags + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Detect Version (release tag) + if: startsWith(github.ref, 'refs/tags/v') + run: | + DETECTED_VERSION=${{ github.ref_name }} + echo "Version ref set to '${DETECTED_VERSION}'" + # Remove the 'v' prefix if it exists + DETECTED_VERSION=${VERSION#v} + echo "Setting version to '$DETECTED_VERSION'" + echo "DETECTED_VERSION=${VERSION}" >> $GITHUB_ENV + + - name: Set Version (workflow_dispatch) + if: github.event_name == 'workflow_dispatch' + run: | + INPUT_VERSION=${{ github.event.inputs.version }} + echo "Version input set to '${INPUT_VERSION}'" + # Exit with success if both detected and input versions are empty + if [ -z "$DETECTED_VERSION" ] && [ -z "$INPUT_VERSION" ]; then + echo "No version detected or input. Will publish to SHA tag instead." + echo 'VERSION=""' >> $GITHUB_ENV + exit 0 + fi + # Remove the 'v' prefix if it exists + INPUT_VERSION=${INPUT_VERSION#v} + # Fail if detected version is non-empty and different from the input version + if [ -n "$DETECTED_VERSION" ] [ -n "$INPUT_VERSION" ] && [ "$DETECTED_VERSION" != "$VERSION" ]; then + echo "Error: Version input '${INPUT_VERSION}' does not match detected version '${DETECTED_VERSION}'." + exit 1 + fi + # Set the version to the input version if non-empty, otherwise the detected version + VERSION=${INPUT_VERSION:-$DETECTED_VERSION} + # Fail if the version is still empty + if [ -z "$VERSION" ]; then + echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'." + exit 1 + fi + echo "Setting version to '$VERSION'" + echo "VERSION=${VERSION}" >> $GITHUB_ENV + # Check if version is a prerelease version (will not tag 'latest') + if [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "IS_PRERELEASE=false" >> $GITHUB_ENV + else + echo "IS_PRERELEASE=true" >> $GITHUB_ENV + fi + + # We need to download the build artifact again because the previous job was on a different runner + - name: Download Build Artifact + uses: actions/download-artifact@v4 + with: + name: Packages-${{ github.run_id }} + path: dist + + - name: Set up QEMU for multi-platform builds + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_PASSWORD }} + + - name: Check for existing tag + if: env.VERSION != '' + run: | + tag="airbyte/source-declarative-manifest:${{ env.VERSION }}" + if [ -z "$tag" ]; then + echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'." + exit 1 + fi + echo "Checking if tag '$tag' exists on DockerHub..." + if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$tag" > /dev/null 2>&1; then + echo "The tag '$tag' already exists on DockerHub. Skipping publish to prevent overwrite." + exit 1 + fi + echo "No existing tag '$tag' found. Proceeding with publish." + + - name: Build and push (sha tag) + # Only run if the version is not set + if: env.VERSION == '' + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: | + airbyte/source-declarative-manifest:${{ github.sha }} + + - name: Build and push (version tag) + # Only run if the version is set + if: env.VERSION != '' + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: | + airbyte/source-declarative-manifest:${{ env.VERSION }} + + + - name: Build and push ('latest' tag) + # Only run if version is set and IS_PRERELEASE is false + if: env.VERSION != '' && env.IS_PRERELEASE == 'false' + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: | + airbyte/source-declarative-manifest:latest diff --git a/.github/workflows/pypi_publish.yml b/.github/workflows/pypi_publish.yml index 09b519375..41329ce62 100644 --- a/.github/workflows/pypi_publish.yml +++ b/.github/workflows/pypi_publish.yml @@ -59,81 +59,3 @@ jobs: - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@v1.10.3 - - # publish_sdm: - # name: Publish SDM to DockerHub - # if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch' - # runs-on: ubuntu-latest - # needs: [publish] - # environment: - # name: DockerHub - # url: https://hub.docker.com/r/airbyte/source-declarative-manifest/tags - - # steps: - # - uses: actions/checkout@v4 - # with: - # fetch-depth: 0 - - # - name: Set Version (workflow_dispatch) - # if: github.event_name == 'workflow_dispatch' - # run: | - # VERSION=${{ github.event.inputs.version }} - # echo "Version input set to '${VERSION}'" - # # Remove the 'v' prefix if it exists - # VERSION=${VERSION#v} - # echo "Setting version to '$VERSION'" - # echo "VERSION=${VERSION}" >> $GITHUB_ENV - - # - name: Set Version (tag) - # if: startsWith(github.ref, 'refs/tags/v') - # run: | - # VERSION=${{ github.ref_name }} - # echo "Version ref set to '${VERSION}'" - # # Remove the 'v' prefix if it exists - # VERSION=${VERSION#v} - # echo "Setting version to '$VERSION'" - # echo "VERSION=${VERSION}" >> $GITHUB_ENV - - # # We need to download the build artifact again because the previous job was on a different runner - # - name: Download Build Artifact - # uses: actions/download-artifact@v4 - # with: - # name: Packages-${{ github.run_id }} - # path: dist - - # - name: Set up QEMU for multi-platform builds - # uses: docker/setup-qemu-action@v3 - - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v3 - - # - name: Login to Docker Hub - # uses: docker/login-action@v3 - # with: - # username: ${{ secrets.DOCKER_HUB_USERNAME }} - # password: ${{ secrets.DOCKER_HUB_PASSWORD }} - - # - name: Check for existing tag - # run: | - # tag="airbyte/source-declarative-manifest:${{ env.VERSION }}" - # if [ -z "$tag" ]; then - # echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'." - # exit 1 - # fi - # echo "Checking if tag '$tag' exists on DockerHub..." - # if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$tag" > /dev/null 2>&1; then - # echo "The tag '$tag' already exists on DockerHub. Skipping publish to prevent overwrite." - # exit 1 - # fi - # echo "No existing tag '$tag' found. Proceeding with publish." - - # - name: Build and push - # uses: docker/build-push-action@v5 - # with: - # context: . - # platforms: linux/amd64,linux/arm64 - # push: true - # tags: | - # airbyte/source-declarative-manifest:latest - # airbyte/source-declarative-manifest:${{ env.VERSION }} - # airbyte/source-declarative-manifest:${{ github.sha }} From 953e66d29356f9516a5bb8a253e6ab4fbe75f00c Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Fri, 22 Nov 2024 10:14:18 -0800 Subject: [PATCH 2/7] add 'dry_run' input arg --- .github/workflows/publish_sdm_connector.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish_sdm_connector.yml b/.github/workflows/publish_sdm_connector.yml index 7b173034e..a3a3dcdbc 100644 --- a/.github/workflows/publish_sdm_connector.yml +++ b/.github/workflows/publish_sdm_connector.yml @@ -14,6 +14,11 @@ on: If omitted, and if run from a non-release branch, then only a SHA-based Docker tag will be created. required: false + dry_run: + description: If true, the workflow will not push to DockerHub. + type: boolean + required: false + default: false jobs: build: @@ -126,7 +131,7 @@ jobs: - name: Build and push (sha tag) # Only run if the version is not set - if: env.VERSION == '' + if: env.VERSION == '' && github.event.inputs.dry_run == 'false' uses: docker/build-push-action@v5 with: context: . @@ -137,7 +142,7 @@ jobs: - name: Build and push (version tag) # Only run if the version is set - if: env.VERSION != '' + if: env.VERSION != '' && github.event.inputs.dry_run == 'false' uses: docker/build-push-action@v5 with: context: . @@ -149,7 +154,7 @@ jobs: - name: Build and push ('latest' tag) # Only run if version is set and IS_PRERELEASE is false - if: env.VERSION != '' && env.IS_PRERELEASE == 'false' + if: env.VERSION != '' && env.IS_PRERELEASE == 'false' && github.event.inputs.dry_run == 'false' uses: docker/build-push-action@v5 with: context: . From a9da960e977995cf73d34113e6c2447228eaaa4b Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Fri, 22 Nov 2024 10:25:40 -0800 Subject: [PATCH 3/7] apply suggestion Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .github/workflows/publish_sdm_connector.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish_sdm_connector.yml b/.github/workflows/publish_sdm_connector.yml index a3a3dcdbc..796ddf9c9 100644 --- a/.github/workflows/publish_sdm_connector.yml +++ b/.github/workflows/publish_sdm_connector.yml @@ -57,9 +57,9 @@ jobs: DETECTED_VERSION=${{ github.ref_name }} echo "Version ref set to '${DETECTED_VERSION}'" # Remove the 'v' prefix if it exists - DETECTED_VERSION=${VERSION#v} + DETECTED_VERSION="${DETECTED_VERSION#v}" echo "Setting version to '$DETECTED_VERSION'" - echo "DETECTED_VERSION=${VERSION}" >> $GITHUB_ENV + echo "DETECTED_VERSION=${DETECTED_VERSION}" >> $GITHUB_ENV - name: Set Version (workflow_dispatch) if: github.event_name == 'workflow_dispatch' @@ -67,15 +67,15 @@ jobs: INPUT_VERSION=${{ github.event.inputs.version }} echo "Version input set to '${INPUT_VERSION}'" # Exit with success if both detected and input versions are empty - if [ -z "$DETECTED_VERSION" ] && [ -z "$INPUT_VERSION" ]; then + if [ -z "${DETECTED_VERSION:-}" ] && [ -z "${INPUT_VERSION:-}" ]; then echo "No version detected or input. Will publish to SHA tag instead." echo 'VERSION=""' >> $GITHUB_ENV exit 0 fi # Remove the 'v' prefix if it exists - INPUT_VERSION=${INPUT_VERSION#v} + INPUT_VERSION="${INPUT_VERSION#v}" # Fail if detected version is non-empty and different from the input version - if [ -n "$DETECTED_VERSION" ] [ -n "$INPUT_VERSION" ] && [ "$DETECTED_VERSION" != "$VERSION" ]; then + if [ -n "${DETECTED_VERSION:-}" ] && [ -n "${INPUT_VERSION:-}" ] && [ "${DETECTED_VERSION}" != "${INPUT_VERSION}" ]; then echo "Error: Version input '${INPUT_VERSION}' does not match detected version '${DETECTED_VERSION}'." exit 1 fi @@ -94,7 +94,6 @@ jobs: else echo "IS_PRERELEASE=true" >> $GITHUB_ENV fi - # We need to download the build artifact again because the previous job was on a different runner - name: Download Build Artifact uses: actions/download-artifact@v4 From 7763245775856645611931a176038e40f284a00e Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Fri, 22 Nov 2024 10:28:51 -0800 Subject: [PATCH 4/7] apply suggestion Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .github/workflows/publish_sdm_connector.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_sdm_connector.yml b/.github/workflows/publish_sdm_connector.yml index 796ddf9c9..609e626d5 100644 --- a/.github/workflows/publish_sdm_connector.yml +++ b/.github/workflows/publish_sdm_connector.yml @@ -80,7 +80,7 @@ jobs: exit 1 fi # Set the version to the input version if non-empty, otherwise the detected version - VERSION=${INPUT_VERSION:-$DETECTED_VERSION} + VERSION="${INPUT_VERSION:-$DETECTED_VERSION}" # Fail if the version is still empty if [ -z "$VERSION" ]; then echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'." From 73f2c1c436226cc5e9a9418834c01b0ea2721cf3 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Fri, 22 Nov 2024 11:25:31 -0800 Subject: [PATCH 5/7] calculate version early and pass it to poetry dynamic versioning --- .github/workflows/publish_sdm_connector.yml | 72 ++++++++++++--------- 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/.github/workflows/publish_sdm_connector.yml b/.github/workflows/publish_sdm_connector.yml index 609e626d5..0e0210a04 100644 --- a/.github/workflows/publish_sdm_connector.yml +++ b/.github/workflows/publish_sdm_connector.yml @@ -24,34 +24,8 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - uses: hynek/build-and-inspect-python-package@v2 - - - uses: actions/upload-artifact@v4 - with: - name: Packages-${{ github.run_id }} - path: | - /tmp/baipp/dist/*.whl - /tmp/baipp/dist/*.tar.gz - publish_sdm: - name: Publish SDM to DockerHub - if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch' - runs-on: ubuntu-latest - needs: [build] - environment: - name: DockerHub - url: https://hub.docker.com/r/airbyte/source-declarative-manifest/tags - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Detect Version (release tag) + - name: Detect Release Tag Version if: startsWith(github.ref, 'refs/tags/v') run: | DETECTED_VERSION=${{ github.ref_name }} @@ -61,7 +35,7 @@ jobs: echo "Setting version to '$DETECTED_VERSION'" echo "DETECTED_VERSION=${DETECTED_VERSION}" >> $GITHUB_ENV - - name: Set Version (workflow_dispatch) + - name: Validate and set VERSION from tag ('${{ github.ref_name }}') and input (${{ github.event.inputs.version || 'none' }}) if: github.event_name == 'workflow_dispatch' run: | INPUT_VERSION=${{ github.event.inputs.version }} @@ -88,12 +62,50 @@ jobs: fi echo "Setting version to '$VERSION'" echo "VERSION=${VERSION}" >> $GITHUB_ENV + echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT # Check if version is a prerelease version (will not tag 'latest') if [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then echo "IS_PRERELEASE=false" >> $GITHUB_ENV + echo "IS_PRERELEASE=false" >> $GITHUB_OUTPUT else echo "IS_PRERELEASE=true" >> $GITHUB_ENV + echo "IS_PRERELEASE=true" >> $GITHUB_OUTPUT fi + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: hynek/build-and-inspect-python-package@v2 + env: + # Pass in the evaluated version from the previous step + # More info: https://github.com/mtkennerly/poetry-dynamic-versioning#user-content-environment-variables + POETRY_DYNAMIC_VERSIONING_BYPASS: ${{ env.VERSION }} + + - uses: actions/upload-artifact@v4 + with: + name: Packages-${{ github.run_id }} + path: | + /tmp/baipp/dist/*.whl + /tmp/baipp/dist/*.tar.gz + + publish_sdm: + name: Publish SDM to DockerHub + if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + needs: [build] + environment: + name: DockerHub + url: https://hub.docker.com/r/airbyte/source-declarative-manifest/tags + env: + VERSION: ${{ needs.build.outputs.VERSION }} + IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + # We need to download the build artifact again because the previous job was on a different runner - name: Download Build Artifact uses: actions/download-artifact@v4 @@ -113,7 +125,7 @@ jobs: username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_PASSWORD }} - - name: Check for existing tag + - name: "Check for existing tag (version: ${{ env.VERSION || 'none' }} )" if: env.VERSION != '' run: | tag="airbyte/source-declarative-manifest:${{ env.VERSION }}" @@ -139,7 +151,7 @@ jobs: tags: | airbyte/source-declarative-manifest:${{ github.sha }} - - name: Build and push (version tag) + - name: "Build and push (version tag: ${{ env.VERSION || 'none'}})" # Only run if the version is set if: env.VERSION != '' && github.event.inputs.dry_run == 'false' uses: docker/build-push-action@v5 From 832c57cc0b630bbaf6e7e6f8b36975373a4c6513 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Fri, 22 Nov 2024 11:27:53 -0800 Subject: [PATCH 6/7] print version in job step name --- .github/workflows/publish_sdm_connector.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish_sdm_connector.yml b/.github/workflows/publish_sdm_connector.yml index 0e0210a04..eec8c381a 100644 --- a/.github/workflows/publish_sdm_connector.yml +++ b/.github/workflows/publish_sdm_connector.yml @@ -77,6 +77,7 @@ jobs: fetch-depth: 0 - uses: hynek/build-and-inspect-python-package@v2 + name: Build package with version ref '${{ env.VERSION }}' env: # Pass in the evaluated version from the previous step # More info: https://github.com/mtkennerly/poetry-dynamic-versioning#user-content-environment-variables From 975b39645db1d03be80cd4603c23e3fd676eb268 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Fri, 22 Nov 2024 11:30:16 -0800 Subject: [PATCH 7/7] declare outputs --- .github/workflows/publish_sdm_connector.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/publish_sdm_connector.yml b/.github/workflows/publish_sdm_connector.yml index eec8c381a..c1b8f81c8 100644 --- a/.github/workflows/publish_sdm_connector.yml +++ b/.github/workflows/publish_sdm_connector.yml @@ -36,6 +36,7 @@ jobs: echo "DETECTED_VERSION=${DETECTED_VERSION}" >> $GITHUB_ENV - name: Validate and set VERSION from tag ('${{ github.ref_name }}') and input (${{ github.event.inputs.version || 'none' }}) + id: set_version if: github.event_name == 'workflow_dispatch' run: | INPUT_VERSION=${{ github.event.inputs.version }} @@ -89,6 +90,9 @@ jobs: path: | /tmp/baipp/dist/*.whl /tmp/baipp/dist/*.tar.gz + outputs: + VERSION: ${{ steps.set_version.outputs.VERSION }} + IS_PRERELEASE: ${{ steps.set_version.outputs.IS_PRERELEASE }} publish_sdm: name: Publish SDM to DockerHub