Matrix-build-wheels #783
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Matrix-build-wheels | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| push: | |
| branches: | |
| - main | |
| - 'releases/**' | |
| release: | |
| types: [ published ] | |
| schedule: | |
| - cron: '45 4 * * 0' | |
| jobs: | |
| prepare_build: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| fhempyV: ${{steps.split_fhempyV.outputs.result}} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| filter: tree:0 | |
| - name: Split fhempy Version | |
| id: split_fhempyV | |
| run: | | |
| echo "result=$(grep '^fhempy' requirements.txt | cut -d '=' -f3)" >> $GITHUB_OUTPUT | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Setup Docker buildx | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| driver-opts: | | |
| image=moby/buildkit:latest | |
| - name: Get Python version from Dockerfile | |
| id: cache-key | |
| run: | | |
| image_version=$(grep -m 1 '^FROM' Dockerfile | cut -d ' ' -f2 | cut -d ':' -f2 | cut -d '@' -f1) | |
| echo "IMAGE_VERSION_SHORT=${image_version%.*}" >> $GITHUB_ENV | |
| echo "IMAGE_VERSION=${image_version}" >> $GITHUB_ENV | |
| - name: Init wheels cache | |
| id: cache-wheels | |
| uses: actions/cache@v4 | |
| with: | |
| path: ./wheelhouse | |
| key: wheels-${{ env.IMAGE_VERSION }}-${{ hashFiles('requirements.txt') }} | |
| restore-keys: | | |
| wheels-${{ env.IMAGE_VERSION }} | |
| wheels-${{ env.IMAGE_VERSION_SHORT }} | |
| wheels- | |
| - name: create cachedir | |
| run: mkdir -p ./wheelhouse | |
| - name: Build base wheels and export to cache | |
| id: build_basewheels_cross | |
| uses: docker/build-push-action@v6.9.0 | |
| with: | |
| context: . | |
| file: Dockerfile | |
| platforms: linux/amd64,linux/arm/v7 | |
| push: false | |
| target: export-stage | |
| cache-to: type=local,dest=/tmp/.buildx-cache,mode=max | |
| cache-from: | | |
| type=gha,scope=wheel_builder_stage_amd64 | |
| type=gha,scope=wheel_builder_stage_armv7 | |
| type=gha,scope=wheel_builder_stage_armv6 | |
| load: false | |
| outputs: type=local,dest=./wheelhouse | |
| - name: cache wheel builder image amd64 | |
| uses: docker/build-push-action@v6.9.0 | |
| with: | |
| context: . | |
| file: Dockerfile | |
| platforms: linux/amd64 | |
| push: false | |
| target: w-builder | |
| cache-to: type=gha,mode=min,scope=wheel_builder_stage_amd64 | |
| cache-from: type=local,src=/tmp/.buildx-cache | |
| - name: cache wheel builder image armv7 | |
| uses: docker/build-push-action@v6.9.0 | |
| with: | |
| context: . | |
| file: Dockerfile | |
| platforms: linux/arm/v7 | |
| push: false | |
| target: w-builder | |
| cache-to: type=gha,mode=min,scope=wheel_builder_stage_armv7 | |
| cache-from: type=local,src=/tmp/.buildx-cache | |
| # - name: cache wheel builder image armv6 | |
| # uses: docker/build-push-action@v5.1.0 | |
| # with: | |
| # context: . | |
| # file: Dockerfile | |
| # platforms: linux/arm/v6 | |
| # push: false | |
| # target: w-builder | |
| # cache-to: type=gha,mode=min,scope=wheel_builder_stage_armv6 | |
| # cache-from: type=local,src=/tmp/.buildx-cache | |
| - name: Build amd64 base image and export to cache | |
| id: build_base_amd64 | |
| uses: docker/build-push-action@v6.9.0 | |
| with: | |
| context: . | |
| file: Dockerfile | |
| platforms: linux/amd64 | |
| push: false | |
| target: base | |
| cache-from: | | |
| type=gha,scope=base_image_stage_amd64 | |
| cache-to: | | |
| type=gha,scope=base_image_stage_amd64,mode=max | |
| load: false | |
| - name: Build armv7 base image and export to cache | |
| id: build_base_armv7 | |
| uses: docker/build-push-action@v6.9.0 | |
| with: | |
| context: . | |
| file: Dockerfile | |
| platforms: linux/arm/v7 | |
| push: false | |
| target: base | |
| cache-from: | | |
| type=gha,scope=base_image_stage_armv7 | |
| cache-to: | | |
| type=gha,scope=base_image_stage_armv7,mode=max | |
| load: false | |
| - name: generate requierementsfile for extra requirements | |
| run: | | |
| #echo "cryptography" >> requirements_extra.txt | |
| echo "pyaml" >> requirements_extra.txt | |
| - name: Build extra wheels | |
| uses: docker/build-push-action@v6.9.0 | |
| with: | |
| context: . | |
| file: Dockerfile | |
| platforms: linux/amd64,linux/arm/v7 | |
| push: false | |
| target: export-stage | |
| load: false | |
| outputs: type=local,dest=./wheelhouse | |
| build-args: | |
| REQUIREMENTS_FILE=requirements_extra.txt | |
| prepare_matrix: | |
| #name: Find fhempy modules | |
| #needs: prepare_build | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{steps.getModules.outputs.matrix}} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| filter: tree:0 | |
| - name: Split fhempy Version | |
| run: | | |
| echo "fhempyVersion=$(grep '^fhempy' requirements.txt | cut -d '=' -f3)" >> $GITHUB_ENV | |
| - uses: actions/checkout@v4 | |
| with: | |
| repository: 'fhempy/fhempy' | |
| path: fhempyRepo | |
| filter: tree:0 | |
| ref: 'v${{ env.fhempyVersion }}' | |
| - name: Get module names and prepare matrix | |
| id: getModules | |
| working-directory: ./fhempyRepo/FHEM/bindings/python/fhempy/lib | |
| run: | | |
| ${GITHUB_WORKSPACE}/scripts/get-deps.sh . | |
| shell: bash | |
| buildImages: | |
| runs-on: ubuntu-latest | |
| needs: [prepare_build, prepare_matrix] | |
| permissions: | |
| contents: write | |
| packages: write | |
| # This is used to complete the identity challenge | |
| # with sigstore/fulcio when running outside of PRs. | |
| id-token: write | |
| env: | |
| # Use docker.io for Docker Hub if empty | |
| REGISTRY: ghcr.io | |
| # github.repository as <account>/<repo> | |
| IMAGE_NAME: ${{ github.repository }} | |
| strategy: | |
| max-parallel: 5 | |
| fail-fast: false | |
| matrix: | |
| ${{fromJson(needs.prepare_matrix.outputs.matrix)}} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| filter: tree:0 | |
| - name: check variables | |
| run: | | |
| echo '${{ toJSON(matrix) }}' | |
| echo '${{ matrix.module }}' | |
| - name: write requirement for mod from matrix input | |
| run: | | |
| touch ./requirements_mod.txt | |
| echo '${{toJSON(matrix.requirements)}}' | jq --raw-output -e '.[]' > ./requirements_mod.txt || true | |
| cat ./requirements_mod.txt | |
| - name: Run Docker on tmpfs | |
| uses: sidey79/docker-on-tmpfs@main | |
| with: | |
| tmpfs_size: 6 | |
| swap_size: 4 | |
| swap_location: '/mnt/swapfile' | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Setup Docker buildx | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| driver-opts: | | |
| image=moby/buildkit:latest | |
| - name: Extract Docker metadata | |
| id: meta-it | |
| uses: docker/metadata-action@v5.6.1 | |
| with: | |
| images: runtime | |
| - name: Get Python version from Dockerfile | |
| id: cache-key | |
| run: | | |
| image_version=$(grep -m 1 '^FROM' Dockerfile | cut -d ' ' -f2 | cut -d ':' -f2 | cut -d '@' -f1) | |
| echo "IMAGE_VERSION_SHORT=${image_version%.*}" >> $GITHUB_ENV | |
| echo "IMAGE_VERSION=${image_version}" >> $GITHUB_ENV | |
| - uses: Wandalen/wretry.action@v3.7.2 | |
| name: restore base cache with retry | |
| id: cache | |
| with: | |
| attempt_delay: 35000 | |
| attempt_limit: 3 | |
| action: actions/cache/restore@v4 | |
| with: | | |
| path: ./wheelhouse | |
| fail-on-cache-miss: true | |
| key: wheels-${{ env.IMAGE_VERSION }}-${{ hashFiles('requirements.txt') }} | |
| - name: Init wheels cache | |
| id: cache-wheels | |
| uses: actions/cache@v4 | |
| with: | |
| save-always: true | |
| path: ./wheelhouse | |
| key: wheels-${{ matrix.module }}-${{ env.IMAGE_VERSION }} | |
| restore-keys: | | |
| wheels-${{ matrix.module }}-${{ env.IMAGE_VERSION }} | |
| wheels-${{ matrix.module }}- | |
| wheels-${{ env.IMAGE_VERSION }}-${{ hashFiles('requirements.txt') }} | |
| wheels-${{ env.IMAGE_VERSION }} | |
| - name: create cachedir if not exists | |
| run: | | |
| mkdir -p ./wheelhouse/linux_amd64 | |
| mkdir -p ./wheelhouse/linux_arm_v7 | |
| - name: Build wheels for mod ${{ matrix.module }} | |
| uses: docker/build-push-action@v6.9.0 | |
| timeout-minutes: 75 | |
| with: | |
| context: . | |
| file: Dockerfile | |
| platforms: linux/amd64,linux/arm/v7 | |
| push: false | |
| load: false | |
| target: export-stage | |
| cache-from: | | |
| type=gha,scope=wheel_builder_stage_amd64 | |
| type=gha,scope=wheel_builder_stage_armv7 | |
| type=gha,scope=wheel_builder_stage_armv6 | |
| build-args: | | |
| REQUIREMENTS_FILE=requirements_mod.txt | |
| outputs: type=local,dest=./wheelhouse | |
| - name: Build for integration test | |
| id: build-it | |
| uses: docker/build-push-action@v6.9.0 | |
| with: | |
| context: . | |
| file: Dockerfile | |
| platforms: linux/amd64 | |
| push: false | |
| load: true | |
| target: runtime | |
| #cache-to: type=gha,scope=runtime-stage-${{ matrix.module }}_${{ hashFiles('./requirements_mod.txt') }}-amd64,ignore-error=true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| cache-from: | | |
| type=gha,scope=base_image_stage_amd64 | |
| # type=gha,scope=runtime-stage-${{ matrix.module }}_${{ hashFiles('./requirements_mod.txt') }}-amd64 | |
| build-args: | | |
| REQUIREMENTS_FILE=requirements_mod.txt | |
| # PKGS=${{ matrix.PKGS }} | |
| - name: Run integration tests | |
| shell: bash | |
| run: | | |
| ./scripts/test-integration.sh ${{ steps.build-it.outputs.imageid }} | |
| - name: Install cosign | |
| if: github.event_name == 'release' | |
| uses: sigstore/cosign-installer@v3.7.0 | |
| # Login against a Docker registry if it is a release | |
| # https://github.com/docker/login-action | |
| - name: Log into registry ${{ env.REGISTRY }} | |
| if: github.event_name == 'release' | |
| uses: docker/login-action@v3.3.0 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract Docker metadata | |
| id: meta | |
| uses: docker/metadata-action@v5.6.1 | |
| with: | |
| images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}_${{ matrix.module }} | |
| flavor: | | |
| latest= ${{ fromJSON('["auto", "false"]')[github.event.release.prerelease == 1] }} | |
| tags: | | |
| type=ref,enable=true,priority=600,prefix=pr-,suffix=,event=pr | |
| type=semver,pattern={{version}} | |
| type=semver,pattern={{major}},enable=${{ github.event.release.prerelease == 0 }} | |
| type=ref,event=branch | |
| type=ref,event=pr | |
| labels: | | |
| org.opencontainers.image.title=${{ env.IMAGE_NAME }}_${{ matrix.module }} | |
| - name: Build and push Docker image for all plattforms | |
| uses: docker/build-push-action@v6.9.0 | |
| id: docker_build_runtime_cross | |
| with: | |
| context: . | |
| platforms: linux/arm/v7,linux/amd64 | |
| push: ${{ github.event_name == 'release' }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| target: runtime | |
| cache-to: type=local,dest=/tmp/.buildx-cache,mode=max | |
| cache-from: | | |
| type=gha,scope=base_image_stage_amd64 | |
| type=gha,scope=base_image_stage_armv6 | |
| type=gha,scope=base_image_stage_armv7 | |
| # type=gha,scope=runtime-stage-${{ matrix.module }}_${{ hashFiles('./requirements_mod.txt') }}-amd64 | |
| # type=gha,scope=runtime-stage-${{ matrix.module }}_${{ hashFiles('./requirements_mod.txt') }}-armv7 | |
| build-args: | | |
| PKGS=${{ matrix.PKGS }} | |
| - name: show image details | |
| run: | | |
| echo "${{ steps.docker_build_runtime_cross.outputs.imageid }}" | |
| echo "${{ steps.docker_build_runtime_cross.outputs.digest }}" | |
| echo '${{ steps.docker_build_runtime_cross.outputs.metadata }}' | |
| # Sign the resulting Docker image digest except on PRs. | |
| # This will only write to the public Rekor transparency log when the Docker | |
| # repository is public to avoid leaking data. If you would like to publish | |
| # transparency data even for private images, pass --force to cosign below. | |
| # https://github.com/sigstore/cosign | |
| - name: Sign the published Docker image | |
| if: ${{ github.event_name == 'release' }} | |
| env: | |
| TAGS: ${{ steps.meta.outputs.tags }} | |
| DIGEST: ${{ steps.docker_build_runtime_cross.outputs.digest }} | |
| # This step uses the identity token to provision an ephemeral certificate | |
| # against the sigstore community Fulcio instance. | |
| run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} | |
| - name: Create Job Summary | |
| run: | | |
| echo "### Image report for : ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY | |
| echo "| Module | Image name |" >> $GITHUB_STEP_SUMMARY | |
| echo "|-------|----------------------------------------------------------------------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| ${{ matrix.module }} | ${{ fromJSON(steps.docker_build_runtime_cross.outputs.metadata)['image.name'] }} |" >> $GITHUB_STEP_SUMMARY | |
| echo "| ${{ matrix.module }} | ${{ fromJSON(steps.docker_build_runtime_cross.outputs.metadata)['image.name'] }} |" > ${{ matrix.module }}_result.md | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: artifact-${{ matrix.module }} | |
| path: ${{ matrix.module }}_result.md | |
| if-no-files-found: warn | |
| #- name: cache armv7 image | |
| # uses: docker/build-push-action@v5.1.0 | |
| # if: always() | |
| # with: | |
| # context: . | |
| # platforms: linux/arm/v7 | |
| # push: false | |
| # load: false | |
| # target: runtime | |
| # cache-from: type=local,src=/tmp/.buildx-cache | |
| # cache-to: type=gha,scope=runtime-stage-${{ matrix.module }}_${{ hashFiles('./requirements_mod.txt') }}-armv7 | |
| # build-args: | | |
| # PKGS=${{ matrix.PKGS }} | |
| #- name: cache amd64 image | |
| # if: always() | |
| # uses: docker/build-push-action@v5.1.0 | |
| # with: | |
| # context: . | |
| # platforms: linux/amd64 | |
| # push: false | |
| # load: false | |
| # target: runtime | |
| # cache-from: type=local,src=/tmp/.buildx-cache | |
| # cache-to: type=gha,scope=runtime-stage-${{ matrix.module }}_${{ hashFiles('./requirements_mod.txt') }}-amd64 | |
| # build-args: | | |
| # PKGS=${{ matrix.PKGS }} | |
| check-matrix: | |
| runs-on: ubuntu-latest | |
| needs: [buildImages,prepare_build] | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| pattern: artifact-* | |
| merge-multiple: true | |
| path: build-artifact | |
| - name: Find image String | |
| id: extract_image | |
| run: | | |
| echo "image_name=$(grep -m 1 '^FROM' Dockerfile | cut -d ' ' -f2 | cut -d ':' -f1)" >> $GITHUB_OUTPUT | |
| echo "image_version=$(grep -m 1 '^FROM' Dockerfile | cut -d ' ' -f2 | cut -d ':' -f2 | cut -d '@' -f1)" >> $GITHUB_OUTPUT | |
| - name: Update docs (readme.md) | |
| run: | | |
| cat build-artifact/*_result.md > modules.md | |
| IMAGE_LIST=$(cat modules.md) | |
| echo "$IMAGE_LIST" | |
| ESCAPED_DATA="$(echo "${IMAGE_LIST}" | sed 's/\n/\\n/g')" | |
| echo "--------" | |
| echo "$ESCAPED_DATA" | |
| sed -e "s/{FHEMPY_VERSION}/${{ needs.prepare_build.outputs.fhempyV }}/" -e "s/{PYTHON_VERSION}/${{ steps.extract_image.outputs.image_version}}/" -e "s/{DEBIAN_RELEASE}/buster/" -e s,{IMAGE_TAG},${{ github.ref_name }}, < README.tmpl > README.md | |
| sed -i -e "/{MODULES_IMAGE_LIST}/r modules.md" -e "/{MODULES_IMAGE_LIST}/d" README.md | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: readme | |
| path: README.md | |
| if-no-files-found: warn | |
| - name: git commit README.md | |
| id: commit | |
| if: ${{ github.event_name == 'release' }} | |
| run: | | |
| git config --global user.email "action@github.com" | |
| git config --local user.name "GitHub Action" | |
| git commit -m "README.md: Update Versions" -a && echo "::set-output name=status::true" || true | |
| - name: Push back to protected branch (main) | |
| uses: CasperWA/push-protected@v2 | |
| if: ${{ github.event_name == 'release' }} | |
| with: | |
| token: ${{ secrets.BOT_PUSH_TOKEN }} | |
| branch: main | |
| - name: Set step summary | |
| run: | | |
| cat README.md >> $GITHUB_STEP_SUMMARY |