Skip to content

[v2-11-test] Speed up 2.11 prod image builds with parallel per-python builds #62636

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
potiuk merged 1 commit into from
Feb 28, 2026
Merged

[v2-11-test] Speed up 2.11 prod image builds with parallel per-python builds #62636

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
237 changes: 85 additions & 152 deletions .github/workflows/release_dockerhub_image.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,16 @@ on: # yamllint disable-line rule:truthy
workflow_dispatch:
inputs:
airflowVersion:
description: 'Airflow version'
description: 'Airflow version (e.g. 3.0.1, 3.0.1rc1, 3.0.1b1)'
required: true
skipLatest:
description: 'Skip Latest: Set to true if not latest.'
amdOnly:
type: boolean
description: 'Limit to amd64 images'
default: false
limitPythonVersions:
type: string
description: 'Force python versions (e.g. "3.10 3.11")'
default: ''
required: false
permissions:
contents: read
packages: read
Expand All @@ -40,62 +44,50 @@ jobs:
build-info:
timeout-minutes: 10
name: "Build Info"
runs-on: ["ubuntu-22.04"]
runs-on: ["ubuntu-24.04"]
outputs:
pythonVersions: ${{ steps.selective-checks.outputs.python-versions }}
allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }}
defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }}
chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }}
skipLatest: ${{ github.event.inputs.skipLatest == '' && ' ' || '--skip-latest' }}
limitPlatform: ${{ github.repository == 'apache/airflow' && ' ' || '--limit-platform linux/amd64' }}
pythonVersions: ${{ steps.determine-python-versions.outputs.python-versions }}
platformMatrix: ${{ steps.determine-matrix.outputs.platformMatrix }}
airflowVersion: ${{ steps.check-airflow-version.outputs.airflowVersion }}
skipLatest: ${{ steps.check-airflow-version.outputs.skipLatest }}
amd-runners: ${{ steps.selective-checks.outputs.amd-runners }}
arm-runners: ${{ steps.selective-checks.outputs.arm-runners }}
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
VERBOSE: true
steps:
- name: "Cleanup repo"
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
with:
persist-credentials: false
- name: "Cleanup docker"
run: ./scripts/ci/cleanup_docker.sh
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
use-uv: "false"
- name: Selective checks
id: selective-checks
env:
VERBOSE: "false"
run: breeze ci selective-check 2>> ${GITHUB_OUTPUT}

release-images:
timeout-minutes: 120
name: "Release images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}"
runs-on: ["ubuntu-22.04"]
needs: [build-info]
strategy:
fail-fast: false
matrix:
python-version: ${{ fromJSON(needs.build-info.outputs.pythonVersions) }}
AIRFLOW_VERSION: ${{ github.event.inputs.airflowVersion }}
AMD_ONLY: ${{ github.event.inputs.amdOnly }}
LIMIT_PYTHON_VERSIONS: ${{ github.event.inputs.limitPythonVersions }}
UV_VERSION: "0.10.7" # Keep this comment to allow automatic replacement of uv version
if: contains(fromJSON('[
"ashb",
"bugraoz93",
"eladkal",
"ephraimbuddy",
"jedcunningham",
"jscheffl",
"kaxil",
"pierrejeambrun",
"potiuk",
"utkarsharma2"
"utkarsharma2",
"vincbeck",
]'), github.event.sender.login)
steps:
- name: "Input parameters summary"
shell: bash
run: |
echo "Input parameters summary"
echo "========================="
echo "Airflow version: '${AIRFLOW_VERSION}'"
echo "AMD only: '${AMD_ONLY}'"
echo "Limit python versions: '${LIMIT_PYTHON_VERSIONS}'"
- name: "Cleanup repo"
shell: bash
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
run: >
docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"

- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
persist-credentials: false
- name: "Cleanup docker"
Expand All @@ -104,116 +96,57 @@ jobs:
uses: ./.github/actions/breeze
with:
use-uv: "false"
- name: Free space
run: breeze ci free-space --answer yes
- name: "Cleanup dist and context file"
run: rm -fv ./dist/* ./docker-context-files/*
- name: "Login to hub.docker.com"
run: >
echo ${{ secrets.DOCKERHUB_TOKEN }} |
docker login --password-stdin --username ${{ secrets.DOCKERHUB_USER }}
- name: Login to ghcr.io
- name: Selective checks
id: selective-checks
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ACTOR: ${{ github.actor }}
run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u ${ACTOR} --password-stdin
- name: "Install buildx plugin"
# yamllint disable rule:line-length
run: |
sudo apt-get update
sudo apt-get install ca-certificates curl
sudo install -m 0755 -d /etc/apt/keyrings
sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
sudo chmod a+r /etc/apt/keyrings/docker.asc

# Add the repository to Apt sources:
echo \
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
$(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
sudo apt-get update
sudo apt install docker-buildx-plugin
- name: "Install regctl"
# yamllint disable rule:line-length
run: |
mkdir -p ~/bin
curl -L https://github.com/regclient/regclient/releases/latest/download/regctl-linux-amd64 >${HOME}/bin/regctl
chmod 755 ${HOME}/bin/regctl
echo "${HOME}/bin" >>${GITHUB_PATH}
- name: "Install emulation support"
run: docker run --privileged --rm tonistiigi/binfmt --install all
- name: "Create airflow_cache builder"
run: docker buildx create --name airflow_cache
- name: "Prepare chicken-eggs provider packages"
VERBOSE: "false"
run: breeze ci selective-check 2>> ${GITHUB_OUTPUT}
- name: "Check airflow version"
id: check-airflow-version
shell: bash
env:
CHICKEN_EGG_PROVIDERS: ${{ needs.build-info.outputs.chicken-egg-providers }}
run: >
breeze release-management prepare-provider-packages
--package-format wheel
${CHICKEN_EGG_PROVIDERS}
if: needs.build-info.outputs.chicken-egg-providers != ''
- name: "Copy dist packages to docker-context files"
run: uv run scripts/ci/airflow_version_check.py "${AIRFLOW_VERSION}" >> "${GITHUB_OUTPUT}"
- name: "Determine build matrix"
shell: bash
run: cp -v --no-preserve=mode,ownership ./dist/*.whl ./docker-context-files
if: needs.build-info.outputs.chicken-egg-providers != ''
- name: >
Release regular images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}
env:
COMMIT_SHA: ${{ github.sha }}
REPOSITORY: ${{ github.repository }}
PYTHON_VERSION: ${{ matrix.python-version }}
AIRFLOW_VERSION: ${{ github.event.inputs.airflowVersion }}
SKIP_LATEST: ${{ needs.build-info.outputs.skipLatest }}
LIMIT_PLATFORM: ${{ needs.build-info.outputs.limitPlatform }}
CHICKEN_EGG_PROVIDERS: ${{ needs.build-info.outputs.chicken-egg-providers }}
run: >
breeze release-management release-prod-images
--dockerhub-repo "${REPOSITORY}"
--airflow-version "${AIRFLOW_VERSION}"
${SKIP_LATEST}
${LIMIT_PLATFORM}
--limit-python ${PYTHON_VERSION}
--chicken-egg-providers "${CHICKEN_EGG_PROVIDERS}"
- name: >
Release slim images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}
env:
COMMIT_SHA: ${{ github.sha }}
REPOSITORY: ${{ github.repository }}
PYTHON_VERSION: ${{ matrix.python-version }}
AIRFLOW_VERSION: ${{ github.event.inputs.airflowVersion }}
SKIP_LATEST: ${{ needs.build-info.outputs.skipLatest }}
LIMIT_PLATFORM: ${{ needs.build-info.outputs.limitPlatform }}
run: >
breeze release-management release-prod-images
--dockerhub-repo "${REPOSITORY}"
--airflow-version "${AIRFLOW_VERSION}"
${SKIP_LATEST}
${LIMIT_PLATFORM}
--limit-python ${PYTHON_VERSION} --slim-images
- name: >
Verify regular AMD64 image: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}
env:
PYTHON_VERSION: ${{ matrix.python-version }}
AIRFLOW_VERSION: ${{ github.event.inputs.airflowVersion }}
REPOSITORY: ${{ github.repository }}
run: >
breeze prod-image verify
--pull
--image-name
${REPOSITORY}:${AIRFLOW_VERSION}-python${PYTHON_VERSION}
- name: >
Verify slim AMD64 image: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}
id: determine-matrix
run: |
if [[ "${AMD_ONLY}" = "true" ]]; then
echo 'platformMatrix=["linux/amd64"]' >> "${GITHUB_OUTPUT}"
else
echo 'platformMatrix=["linux/amd64", "linux/arm64"]' >> "${GITHUB_OUTPUT}"
fi
- name: "Determine python versions"
shell: bash
id: determine-python-versions
env:
PYTHON_VERSION: ${{ matrix.python-version }}
AIRFLOW_VERSION: ${{ github.event.inputs.airflowVersion }}
REPOSITORY: ${{ github.repository }}
run: >
breeze prod-image verify
--pull
--slim-image
--image-name
${REPOSITORY}:slim-${AIRFLOW_VERSION}-python${PYTHON_VERSION}
- name: "Docker logout"
run: docker logout
if: always()
ALL_PYTHON_VERSIONS: ${{ steps.selective-checks.outputs.all-python-versions }}
# yamllint disable rule:line-length
run: |
# override python versions if specified
if [[ "${LIMIT_PYTHON_VERSIONS}" != "" ]]; then
PYTHON_VERSIONS=$(python3 -c "import json; print(json.dumps('${LIMIT_PYTHON_VERSIONS}'.split(' ')))")
else
PYTHON_VERSIONS=${ALL_PYTHON_VERSIONS}
fi
echo "python-versions=${PYTHON_VERSIONS}" >> "${GITHUB_OUTPUT}"


release-images:
name: "Release images"
needs: [build-info]
strategy:
fail-fast: false
matrix:
python: ${{ fromJSON(needs.build-info.outputs.pythonVersions) }}
uses: ./.github/workflows/release_single_dockerhub_image.yml
secrets:
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
permissions:
contents: read
with:
pythonVersion: ${{ matrix.python }}
airflowVersion: ${{ needs.build-info.outputs.airflowVersion }}
platformMatrix: ${{ needs.build-info.outputs.platformMatrix }}
skipLatest: ${{ needs.build-info.outputs.skipLatest }}
armRunners: ${{ needs.build-info.outputs.arm-runners }}
amdRunners: ${{ needs.build-info.outputs.amd-runners }}
Loading
Loading