From db092f08350be91bde1287cdf228c272fb3d5f62 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sat, 28 Feb 2026 17:08:54 +0100 Subject: [PATCH] Speed up 2.11 prod image builds with parallel per-python builds Porting speed up in release image building from main. Split the release_dockerhub_image workflow to build each Python version as a separate job via a new reusable release_single_dockerhub_image workflow. Images are first pushed by digest, then merged into multi-platform manifests. This also adds an airflow_version_check.py script to auto-detect skip-latest from PyPI, extracts docker utility checks into reusable functions, and adds AMD/ARM runner constants for selective checks. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/release_dockerhub_image.yml | 237 ++++------- .../release_single_dockerhub_image.yml | 237 +++++++++++ .../doc/09_release_management_tasks.rst | 18 + .../doc/images/output_prod-image_verify.svg | 62 +-- .../doc/images/output_prod-image_verify.txt | 2 +- .../doc/images/output_release-management.svg | 74 ++-- .../doc/images/output_release-management.txt | 2 +- ...t_release-management_merge-prod-images.svg | 162 ++++++++ ...t_release-management_merge-prod-images.txt | 1 + ...release-management_release-prod-images.svg | 54 ++- ...release-management_release-prod-images.txt | 2 +- ...utput_setup_check-all-params-in-groups.svg | 28 +- ...utput_setup_check-all-params-in-groups.txt | 2 +- ...output_setup_regenerate-command-images.svg | 30 +- ...output_setup_regenerate-command-images.txt | 2 +- .../commands/production_image_commands.py | 36 +- .../production_image_commands_config.py | 1 + .../commands/release_management_commands.py | 390 ++++++++++++------ .../release_management_commands_config.py | 21 +- .../src/airflow_breeze/global_constants.py | 19 + .../utils/docker_command_utils.py | 54 +++ .../airflow_breeze/utils/selective_checks.py | 10 + scripts/ci/airflow_version_check.py | 116 ++++++ 23 files changed, 1141 insertions(+), 419 deletions(-) create mode 100644 .github/workflows/release_single_dockerhub_image.yml create mode 100644 dev/breeze/doc/images/output_release-management_merge-prod-images.svg create mode 100644 dev/breeze/doc/images/output_release-management_merge-prod-images.txt create mode 100755 scripts/ci/airflow_version_check.py diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml index 20901d88cb5d7..0586d1711272a 100644 --- a/.github/workflows/release_dockerhub_image.yml +++ b/.github/workflows/release_dockerhub_image.yml @@ -21,12 +21,16 @@ on: # yamllint disable-line rule:truthy workflow_dispatch: inputs: airflowVersion: - description: 'Airflow version' + description: 'Airflow version (e.g. 3.0.1, 3.0.1rc1, 3.0.1b1)' required: true - skipLatest: - description: 'Skip Latest: Set to true if not latest.' + amdOnly: + type: boolean + description: 'Limit to amd64 images' + default: false + limitPythonVersions: + type: string + description: 'Force python versions (e.g. "3.10 3.11")' default: '' - required: false permissions: contents: read packages: read @@ -40,62 +44,50 @@ jobs: build-info: timeout-minutes: 10 name: "Build Info" - runs-on: ["ubuntu-22.04"] + runs-on: ["ubuntu-24.04"] outputs: - pythonVersions: ${{ steps.selective-checks.outputs.python-versions }} - allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }} - defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }} - chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }} - skipLatest: ${{ github.event.inputs.skipLatest == '' && ' ' || '--skip-latest' }} - limitPlatform: ${{ github.repository == 'apache/airflow' && ' ' || '--limit-platform linux/amd64' }} + pythonVersions: ${{ steps.determine-python-versions.outputs.python-versions }} + platformMatrix: ${{ steps.determine-matrix.outputs.platformMatrix }} + airflowVersion: ${{ steps.check-airflow-version.outputs.airflowVersion }} + skipLatest: ${{ steps.check-airflow-version.outputs.skipLatest }} + amd-runners: ${{ steps.selective-checks.outputs.amd-runners }} + arm-runners: ${{ steps.selective-checks.outputs.arm-runners }} env: GITHUB_CONTEXT: ${{ toJson(github) }} VERBOSE: true - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - with: - persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh - - name: "Install Breeze" - uses: ./.github/actions/breeze - with: - use-uv: "false" - - name: Selective checks - id: selective-checks - env: - VERBOSE: "false" - run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} - - release-images: - timeout-minutes: 120 - name: "Release images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}" - runs-on: ["ubuntu-22.04"] - needs: [build-info] - strategy: - fail-fast: false - matrix: - python-version: ${{ fromJSON(needs.build-info.outputs.pythonVersions) }} + AIRFLOW_VERSION: ${{ github.event.inputs.airflowVersion }} + AMD_ONLY: ${{ github.event.inputs.amdOnly }} + LIMIT_PYTHON_VERSIONS: ${{ github.event.inputs.limitPythonVersions }} + UV_VERSION: "0.10.7" # Keep this comment to allow automatic replacement of uv version if: contains(fromJSON('[ "ashb", + "bugraoz93", "eladkal", "ephraimbuddy", "jedcunningham", + "jscheffl", "kaxil", "pierrejeambrun", "potiuk", - "utkarsharma2" + "utkarsharma2", + "vincbeck", ]'), github.event.sender.login) steps: + - name: "Input parameters summary" + shell: bash + run: | + echo "Input parameters summary" + echo "=========================" + echo "Airflow version: '${AIRFLOW_VERSION}'" + echo "AMD only: '${AMD_ONLY}'" + echo "Limit python versions: '${LIMIT_PYTHON_VERSIONS}'" - name: "Cleanup repo" shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + run: > + docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: "Cleanup docker" @@ -104,116 +96,57 @@ jobs: uses: ./.github/actions/breeze with: use-uv: "false" - - name: Free space - run: breeze ci free-space --answer yes - - name: "Cleanup dist and context file" - run: rm -fv ./dist/* ./docker-context-files/* - - name: "Login to hub.docker.com" - run: > - echo ${{ secrets.DOCKERHUB_TOKEN }} | - docker login --password-stdin --username ${{ secrets.DOCKERHUB_USER }} - - name: Login to ghcr.io + - name: Selective checks + id: selective-checks env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ACTOR: ${{ github.actor }} - run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u ${ACTOR} --password-stdin - - name: "Install buildx plugin" - # yamllint disable rule:line-length - run: | - sudo apt-get update - sudo apt-get install ca-certificates curl - sudo install -m 0755 -d /etc/apt/keyrings - sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc - sudo chmod a+r /etc/apt/keyrings/docker.asc - - # Add the repository to Apt sources: - echo \ - "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ - $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ - sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - sudo apt-get update - sudo apt install docker-buildx-plugin - - name: "Install regctl" - # yamllint disable rule:line-length - run: | - mkdir -p ~/bin - curl -L https://github.com/regclient/regclient/releases/latest/download/regctl-linux-amd64 >${HOME}/bin/regctl - chmod 755 ${HOME}/bin/regctl - echo "${HOME}/bin" >>${GITHUB_PATH} - - name: "Install emulation support" - run: docker run --privileged --rm tonistiigi/binfmt --install all - - name: "Create airflow_cache builder" - run: docker buildx create --name airflow_cache - - name: "Prepare chicken-eggs provider packages" + VERBOSE: "false" + run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} + - name: "Check airflow version" + id: check-airflow-version shell: bash - env: - CHICKEN_EGG_PROVIDERS: ${{ needs.build-info.outputs.chicken-egg-providers }} - run: > - breeze release-management prepare-provider-packages - --package-format wheel - ${CHICKEN_EGG_PROVIDERS} - if: needs.build-info.outputs.chicken-egg-providers != '' - - name: "Copy dist packages to docker-context files" + run: uv run scripts/ci/airflow_version_check.py "${AIRFLOW_VERSION}" >> "${GITHUB_OUTPUT}" + - name: "Determine build matrix" shell: bash - run: cp -v --no-preserve=mode,ownership ./dist/*.whl ./docker-context-files - if: needs.build-info.outputs.chicken-egg-providers != '' - - name: > - Release regular images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }} - env: - COMMIT_SHA: ${{ github.sha }} - REPOSITORY: ${{ github.repository }} - PYTHON_VERSION: ${{ matrix.python-version }} - AIRFLOW_VERSION: ${{ github.event.inputs.airflowVersion }} - SKIP_LATEST: ${{ needs.build-info.outputs.skipLatest }} - LIMIT_PLATFORM: ${{ needs.build-info.outputs.limitPlatform }} - CHICKEN_EGG_PROVIDERS: ${{ needs.build-info.outputs.chicken-egg-providers }} - run: > - breeze release-management release-prod-images - --dockerhub-repo "${REPOSITORY}" - --airflow-version "${AIRFLOW_VERSION}" - ${SKIP_LATEST} - ${LIMIT_PLATFORM} - --limit-python ${PYTHON_VERSION} - --chicken-egg-providers "${CHICKEN_EGG_PROVIDERS}" - - name: > - Release slim images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }} - env: - COMMIT_SHA: ${{ github.sha }} - REPOSITORY: ${{ github.repository }} - PYTHON_VERSION: ${{ matrix.python-version }} - AIRFLOW_VERSION: ${{ github.event.inputs.airflowVersion }} - SKIP_LATEST: ${{ needs.build-info.outputs.skipLatest }} - LIMIT_PLATFORM: ${{ needs.build-info.outputs.limitPlatform }} - run: > - breeze release-management release-prod-images - --dockerhub-repo "${REPOSITORY}" - --airflow-version "${AIRFLOW_VERSION}" - ${SKIP_LATEST} - ${LIMIT_PLATFORM} - --limit-python ${PYTHON_VERSION} --slim-images - - name: > - Verify regular AMD64 image: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - AIRFLOW_VERSION: ${{ github.event.inputs.airflowVersion }} - REPOSITORY: ${{ github.repository }} - run: > - breeze prod-image verify - --pull - --image-name - ${REPOSITORY}:${AIRFLOW_VERSION}-python${PYTHON_VERSION} - - name: > - Verify slim AMD64 image: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }} + id: determine-matrix + run: | + if [[ "${AMD_ONLY}" = "true" ]]; then + echo 'platformMatrix=["linux/amd64"]' >> "${GITHUB_OUTPUT}" + else + echo 'platformMatrix=["linux/amd64", "linux/arm64"]' >> "${GITHUB_OUTPUT}" + fi + - name: "Determine python versions" + shell: bash + id: determine-python-versions env: - PYTHON_VERSION: ${{ matrix.python-version }} - AIRFLOW_VERSION: ${{ github.event.inputs.airflowVersion }} - REPOSITORY: ${{ github.repository }} - run: > - breeze prod-image verify - --pull - --slim-image - --image-name - ${REPOSITORY}:slim-${AIRFLOW_VERSION}-python${PYTHON_VERSION} - - name: "Docker logout" - run: docker logout - if: always() + ALL_PYTHON_VERSIONS: ${{ steps.selective-checks.outputs.all-python-versions }} + # yamllint disable rule:line-length + run: | + # override python versions if specified + if [[ "${LIMIT_PYTHON_VERSIONS}" != "" ]]; then + PYTHON_VERSIONS=$(python3 -c "import json; print(json.dumps('${LIMIT_PYTHON_VERSIONS}'.split(' ')))") + else + PYTHON_VERSIONS=${ALL_PYTHON_VERSIONS} + fi + echo "python-versions=${PYTHON_VERSIONS}" >> "${GITHUB_OUTPUT}" + + + release-images: + name: "Release images" + needs: [build-info] + strategy: + fail-fast: false + matrix: + python: ${{ fromJSON(needs.build-info.outputs.pythonVersions) }} + uses: ./.github/workflows/release_single_dockerhub_image.yml + secrets: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + permissions: + contents: read + with: + pythonVersion: ${{ matrix.python }} + airflowVersion: ${{ needs.build-info.outputs.airflowVersion }} + platformMatrix: ${{ needs.build-info.outputs.platformMatrix }} + skipLatest: ${{ needs.build-info.outputs.skipLatest }} + armRunners: ${{ needs.build-info.outputs.arm-runners }} + amdRunners: ${{ needs.build-info.outputs.amd-runners }} diff --git a/.github/workflows/release_single_dockerhub_image.yml b/.github/workflows/release_single_dockerhub_image.yml new file mode 100644 index 0000000000000..9a7ab22ed0545 --- /dev/null +++ b/.github/workflows/release_single_dockerhub_image.yml @@ -0,0 +1,237 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: "Release single PROD image" +on: # yamllint disable-line rule:truthy + workflow_call: + inputs: + airflowVersion: + description: 'Airflow version (e.g. 3.0.1, 3.0.1rc1, 3.0.1b1)' + type: string + required: true + platformMatrix: + description: 'Platform matrix formatted as json (e.g. ["linux/amd64", "linux/arm64"])' + type: string + required: true + pythonVersion: + description: 'Python version (e.g. 3.10, 3.11)' + type: string + required: true + skipLatest: + description: "Skip tagging latest release (true/false)" + type: string + required: true + amdRunners: + description: "Amd64 runners (e.g. [\"ubuntu-22.04\", \"ubuntu-24.04\"])" + type: string + required: true + armRunners: + description: "Arm64 runners (e.g. [\"ubuntu-22.04\", \"ubuntu-24.04\"])" + type: string + required: true + secrets: + DOCKERHUB_USER: + required: true + DOCKERHUB_TOKEN: + required: true +permissions: + contents: read +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERBOSE: true +jobs: + build-images: + timeout-minutes: 50 + # yamllint disable rule:line-length + name: "Build: ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}, ${{ matrix.platform }}" + runs-on: ${{ (matrix.platform == 'linux/amd64') && fromJSON(inputs.amdRunners) || fromJSON(inputs.armRunners) }} + strategy: + fail-fast: false + max-parallel: 20 + matrix: + platform: ${{ fromJSON(inputs.platformMatrix) }} + env: + AIRFLOW_VERSION: ${{ inputs.airflowVersion }} + PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.pythonVersion }} + PLATFORM: ${{ matrix.platform }} + SKIP_LATEST: ${{ inputs.skipLatest == 'true' && '--skip-latest' || '' }} + COMMIT_SHA: ${{ github.sha }} + REPOSITORY: ${{ github.repository }} + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - name: "Install Breeze" + uses: ./.github/actions/breeze + with: + use-uv: "false" + - name: Free space + run: breeze ci free-space --answer yes + - name: "Cleanup dist and context file" + run: rm -fv ./dist/* ./docker-context-files/* + - name: "Login to hub.docker.com" + run: > + echo ${{ secrets.DOCKERHUB_TOKEN }} | + docker login --password-stdin --username ${{ secrets.DOCKERHUB_USER }} + - name: "Get env vars for metadata" + shell: bash + run: | + echo "ARTIFACT_NAME=metadata-${PYTHON_MAJOR_MINOR_VERSION}-${PLATFORM/\//_}" >> "${GITHUB_ENV}" + echo "MANIFEST_FILE_NAME=metadata-${AIRFLOW_VERSION}-${PLATFORM/\//_}-${PYTHON_MAJOR_MINOR_VERSION}.json" >> "${GITHUB_ENV}" + echo "MANIFEST_SLIM_FILE_NAME=metadata-${AIRFLOW_VERSION}-slim-${PLATFORM/\//_}-${PYTHON_MAJOR_MINOR_VERSION}.json" >> "${GITHUB_ENV}" + - name: Login to ghcr.io + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ACTOR: ${{ github.actor }} + run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u ${ACTOR} --password-stdin + - name: "Install buildx plugin" + # yamllint disable rule:line-length + run: | + sudo apt-get update + sudo apt-get install ca-certificates curl + sudo install -m 0755 -d /etc/apt/keyrings + sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + sudo chmod a+r /etc/apt/keyrings/docker.asc + + # Add the repository to Apt sources: + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + sudo apt install docker-buildx-plugin + - name: "Create airflow_cache builder" + run: docker buildx create --name airflow_cache --driver docker-container + - name: > + Build regular images: ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}, ${{ matrix.platform }} + run: > + breeze release-management release-prod-images --dockerhub-repo "${REPOSITORY}" + --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} + --python ${PYTHON_MAJOR_MINOR_VERSION} + --metadata-folder dist + - name: > + Verify regular image: ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}, ${{ matrix.platform }} + run: > + breeze prod-image verify --pull --manifest-file dist/${MANIFEST_FILE_NAME} + - name: > + Release slim images: ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}, ${{ matrix.platform }} + run: > + breeze release-management release-prod-images --dockerhub-repo "${REPOSITORY}" + --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} + --python ${PYTHON_MAJOR_MINOR_VERSION} --slim-images + --metadata-folder dist + - name: > + Verify slim image: ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}, ${{ matrix.platform }} + run: > + breeze prod-image verify --pull --slim-image --manifest-file dist/${MANIFEST_SLIM_FILE_NAME} + - name: "List upload-able artifacts" + shell: bash + run: find ./dist -name '*.json' + - name: "Upload metadata artifact ${{ env.ARTIFACT_NAME }}" + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ${{ env.ARTIFACT_NAME }} + path: ./dist/metadata-* + retention-days: 7 + if-no-files-found: error + - name: "Docker logout" + run: docker logout + if: always() + + merge-images: + timeout-minutes: 5 + name: "Merge: ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}" + runs-on: ["ubuntu-22.04"] + needs: [build-images] + env: + AIRFLOW_VERSION: ${{ inputs.airflowVersion }} + PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.pythonVersion }} + SKIP_LATEST: ${{ inputs.skipLatest == 'true' && '--skip-latest' || '' }} + COMMIT_SHA: ${{ github.sha }} + REPOSITORY: ${{ github.repository }} + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - name: "Install Breeze" + uses: ./.github/actions/breeze + - name: Free space + run: breeze ci free-space --answer yes + - name: "Cleanup dist and context file" + run: rm -fv ./dist/* ./docker-context-files/* + - name: "Login to hub.docker.com" + run: > + echo ${{ secrets.DOCKERHUB_TOKEN }} | + docker login --password-stdin --username ${{ secrets.DOCKERHUB_USER }} + - name: Login to ghcr.io + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ACTOR: ${{ github.actor }} + run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u ${ACTOR} --password-stdin + - name: "Download metadata artifacts" + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + path: ./dist + pattern: metadata-${{ inputs.pythonVersion }}-* + - name: "List downloaded artifacts" + shell: bash + run: find ./dist -name '*.json' + - name: "Install buildx plugin" + # yamllint disable rule:line-length + run: | + sudo apt-get update + sudo apt-get install ca-certificates curl + sudo install -m 0755 -d /etc/apt/keyrings + sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + sudo chmod a+r /etc/apt/keyrings/docker.asc + + # Add the repository to Apt sources: + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + sudo apt install docker-buildx-plugin + - name: "Install regctl" + # yamllint disable rule:line-length + run: | + mkdir -p ~/bin + curl -L https://github.com/regclient/regclient/releases/latest/download/regctl-linux-amd64 >${HOME}/bin/regctl + chmod 755 ${HOME}/bin/regctl + echo "${HOME}/bin" >>${GITHUB_PATH} + - name: "Merge regular images ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}" + run: > + breeze release-management merge-prod-images --dockerhub-repo "${REPOSITORY}" + --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} + --python ${PYTHON_MAJOR_MINOR_VERSION} --metadata-folder dist + - name: "Merge slim images ${{ inputs.airflowVersion }}, ${{ inputs.pythonVersion }}" + run: > + breeze release-management merge-prod-images --dockerhub-repo "${REPOSITORY}" + --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} + --python ${PYTHON_MAJOR_MINOR_VERSION} --metadata-folder dist --slim-images + - name: "Docker logout" + run: docker logout + if: always() diff --git a/dev/breeze/doc/09_release_management_tasks.rst b/dev/breeze/doc/09_release_management_tasks.rst index 9feb9a5b195e6..9915ff35235d6 100644 --- a/dev/breeze/doc/09_release_management_tasks.rst +++ b/dev/breeze/doc/09_release_management_tasks.rst @@ -189,6 +189,24 @@ These are all of the available flags for the ``release-prod-images`` command: :width: 100% :alt: Breeze release management release prod images +Merging production images +""""""""""""""""""""""""" + +When images are built separately per platform (e.g. amd64 and arm64), they need to be merged into +multi-platform manifests. The ``merge-prod-images`` command reads digest metadata files produced by +``release-prod-images --metadata-folder`` and creates the merged multi-platform images. + +.. code-block:: bash + + breeze release-management merge-prod-images --airflow-version 2.4.0 --metadata-folder dist + +These are all of the available flags for the ``merge-prod-images`` command: + +.. image:: ./images/output_release-management_merge-prod-images.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_merge-prod-images.svg + :width: 100% + :alt: Breeze release management merge prod images + Adding git tags for providers """"""""""""""""""""""""""""" diff --git a/dev/breeze/doc/images/output_prod-image_verify.svg b/dev/breeze/doc/images/output_prod-image_verify.svg index 67976e1d87f46..9099b0bc5538e 100644 --- a/dev/breeze/doc/images/output_prod-image_verify.svg +++ b/dev/breeze/doc/images/output_prod-image_verify.svg @@ -1,4 +1,4 @@ - +