From 4a3e1133c629ec43792329fd21b52cd4d39c0d80 Mon Sep 17 00:00:00 2001 From: Jonas Dedden Date: Fri, 1 Mar 2024 08:56:13 +0100 Subject: [PATCH 1/7] Enable Linux ARM builds; skip tests for Linux ARM; Use external image for now --- .github/workflows/build-wheels.yml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 30c26e9d..7e1b8db0 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -26,9 +26,11 @@ jobs: os: [ ubuntu-latest, windows-latest, macOS-latest ] env: CIBW_SKIP: "*-musllinux*" - CIBW_MANYLINUX_X86_64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux2014_x86_64 + CIBW_MANYLINUX_X86_64_IMAGE: ghcr.io/jonded94/resiliparse-manylinux_2_28_x86_64 # replace with ${{ github.repository_owner }} + CIBW_MANYLINUX_AARCH64_IMAGE: ghcr.io/jonded94/resiliparse-manylinux_2_28_aarch64 CIBW_ARCHS_MACOS: "x86_64 arm64" - CIBW_TEST_SKIP: "*-macosx_arm64" # Apple Silicon wheels cannot be tested + CIBW_ARCHS_LINUX: "x86_64 aarch64" + CIBW_TEST_SKIP: "*-macosx_arm64 *-manylinux_aarch64" # Apple Silicon wheels cannot be tested CIBW_REPAIR_WHEEL_COMMAND_MACOS: >- DYLD_LIBRARY_PATH=$LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} CIBW_BEFORE_BUILD_WINDOWS: "python -m pip install delvewheel" @@ -75,6 +77,12 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v3 + with: + platforms: all + - name: Install cibuildwheel run: python -m pip install cibuildwheel @@ -152,7 +160,7 @@ jobs: build-coverage: runs-on: ubuntu-latest container: - image: ghcr.io/chatnoir-eu/resiliparse-manylinux2014_x86_64 + image: ghcr.io/jonded94/resiliparse-manylinux_2_28_x86_64 needs: [ build-wheels, build-asan, build-sdist ] env: TRACE: "1" @@ -211,8 +219,8 @@ jobs: grep -vE "fastwarc|resiliparse" docs/requirements.txt | xargs python -m pip install PYTHON_ABI="cp${PYTHON_VERSION/./}" - find wheelhouse -name "FastWARC-*-${PYTHON_ABI}-*-manylinux*.whl" | xargs -I% python -m pip install "%[all]" - find wheelhouse -name "Resiliparse-*-${PYTHON_ABI}-*-manylinux*.whl" | xargs -I% python -m pip install "%[all]" + find wheelhouse -name "FastWARC-*-${PYTHON_ABI}-*-manylinux_x86_64.whl" | xargs -I% python -m pip install "%[all]" + find wheelhouse -name "Resiliparse-*-${PYTHON_ABI}-*-manylinux_x86_64.whl" | xargs -I% python -m pip install "%[all]" cd docs make html From 08dd6c4f06a5e120a134d63b8af7cc35b17c4329 Mon Sep 17 00:00:00 2001 From: Jonas Dedden Date: Fri, 1 Mar 2024 11:19:58 +0100 Subject: [PATCH 2/7] Test if directly using cibuildwheel recipe fixes QEMU problems --- .github/workflows/build-wheels.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 7e1b8db0..2fa5a611 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -83,16 +83,24 @@ jobs: with: platforms: all - - name: Install cibuildwheel - run: python -m pip install cibuildwheel +# - name: Install cibuildwheel +# run: python -m pip install cibuildwheel - name: Build FastWARC - run: python -m cibuildwheel --output-dir wheelhouse fastwarc + uses: pypa/cibuildwheel@v2.16.5 + with: + package-dir: fastwarc + output-dir: wheelhouse + # run: python -m cibuildwheel --output-dir wheelhouse fastwarc env: CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/fastwarc - name: Build Resiliparse - run: python -m cibuildwheel --output-dir wheelhouse resiliparse + uses: pypa/cibuildwheel@v2.16.5 + with: + package-dir: resiliparse + output-dir: wheelhouse + # run: python -m cibuildwheel --output-dir wheelhouse resiliparse env: CIBW_BEFORE_TEST: >- python -c "import glob, platform; open('fastwarc.txt', 'w').write(glob.glob('wheelhouse/FastWARC-*cp' + ''.join(map(str, platform.python_version_tuple()[:2])) + '-*_' + platform.machine().lower() + '.whl')[0])" && From 5be6d5395fb884646cc24218d2573265ef8f4811 Mon Sep 17 00:00:00 2001 From: Jonas Dedden Date: Fri, 8 Mar 2024 21:49:37 +0100 Subject: [PATCH 3/7] Move to new chatnoir-eu images --- .github/workflows/build-wheels.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 2fa5a611..355d25ad 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -26,8 +26,8 @@ jobs: os: [ ubuntu-latest, windows-latest, macOS-latest ] env: CIBW_SKIP: "*-musllinux*" - CIBW_MANYLINUX_X86_64_IMAGE: ghcr.io/jonded94/resiliparse-manylinux_2_28_x86_64 # replace with ${{ github.repository_owner }} - CIBW_MANYLINUX_AARCH64_IMAGE: ghcr.io/jonded94/resiliparse-manylinux_2_28_aarch64 + CIBW_MANYLINUX_X86_64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest + CIBW_MANYLINUX_AARCH64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_aarch64:latest CIBW_ARCHS_MACOS: "x86_64 arm64" CIBW_ARCHS_LINUX: "x86_64 aarch64" CIBW_TEST_SKIP: "*-macosx_arm64 *-manylinux_aarch64" # Apple Silicon wheels cannot be tested @@ -142,7 +142,7 @@ jobs: build-asan: runs-on: ubuntu-latest container: - image: ghcr.io/chatnoir-eu/resiliparse-manylinux2014_x86_64 + image: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest env: DEBUG: "1" ASAN: "1" @@ -168,7 +168,7 @@ jobs: build-coverage: runs-on: ubuntu-latest container: - image: ghcr.io/jonded94/resiliparse-manylinux_2_28_x86_64 + image: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest needs: [ build-wheels, build-asan, build-sdist ] env: TRACE: "1" From f062886b8daa4d15c61481c081767a732e285796 Mon Sep 17 00:00:00 2001 From: Jonas Dedden Date: Fri, 8 Mar 2024 22:04:18 +0100 Subject: [PATCH 4/7] Move aarch64 build to separate job since no progress on bug pypa/cibuildwheel#1771 --- .github/workflows/build-wheels.yml | 60 ++++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 355d25ad..b08ec500 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -27,10 +27,9 @@ jobs: env: CIBW_SKIP: "*-musllinux*" CIBW_MANYLINUX_X86_64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest - CIBW_MANYLINUX_AARCH64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_aarch64:latest CIBW_ARCHS_MACOS: "x86_64 arm64" - CIBW_ARCHS_LINUX: "x86_64 aarch64" - CIBW_TEST_SKIP: "*-macosx_arm64 *-manylinux_aarch64" # Apple Silicon wheels cannot be tested + CIBW_ARCHS_LINUX: "x86_64" + CIBW_TEST_SKIP: "*-macosx_arm64" # Apple Silicon wheels cannot be tested CIBW_REPAIR_WHEEL_COMMAND_MACOS: >- DYLD_LIBRARY_PATH=$LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} CIBW_BEFORE_BUILD_WINDOWS: "python -m pip install delvewheel" @@ -77,21 +76,60 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} + - name: Build FastWARC + uses: pypa/cibuildwheel@v2.16.5 + with: + package-dir: fastwarc + output-dir: wheelhouse + env: + CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/fastwarc + + - name: Build Resiliparse + uses: pypa/cibuildwheel@v2.16.5 + with: + package-dir: resiliparse + output-dir: wheelhouse + env: + CIBW_BEFORE_TEST: >- + python -c "import glob, platform; open('fastwarc.txt', 'w').write(glob.glob('wheelhouse/FastWARC-*cp' + ''.join(map(str, platform.python_version_tuple()[:2])) + '-*_' + platform.machine().lower() + '.whl')[0])" && + python -m pip install -r fastwarc.txt + CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/resiliparse + + - name: Upload Wheels + uses: actions/upload-artifact@v3 + with: + name: wheels + path: ./wheelhouse/*.whl + + build-wheels-qemu: + runs-on: ubuntu-latest + env: + CIBW_SKIP: "*-musllinux*" + CIBW_MANYLINUX_AARCH64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_aarch64:latest + CIBW_ARCHS_LINUX: "aarch64" + CIBW_TEST_SKIP: "*-manylinux_aarch64" + # https://github.com/pypa/cibuildwheel/issues/1771#issuecomment-1973003145 + CIBW_CONTAINER_ENGINE: "docker; create_args: --platform linux/arm64/v8" + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Set up QEMU - if: runner.os == 'Linux' uses: docker/setup-qemu-action@v3 with: platforms: all -# - name: Install cibuildwheel -# run: python -m pip install cibuildwheel - - name: Build FastWARC uses: pypa/cibuildwheel@v2.16.5 with: package-dir: fastwarc output-dir: wheelhouse - # run: python -m cibuildwheel --output-dir wheelhouse fastwarc env: CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/fastwarc @@ -100,7 +138,6 @@ jobs: with: package-dir: resiliparse output-dir: wheelhouse - # run: python -m cibuildwheel --output-dir wheelhouse resiliparse env: CIBW_BEFORE_TEST: >- python -c "import glob, platform; open('fastwarc.txt', 'w').write(glob.glob('wheelhouse/FastWARC-*cp' + ''.join(map(str, platform.python_version_tuple()[:2])) + '-*_' + platform.machine().lower() + '.whl')[0])" && @@ -142,7 +179,8 @@ jobs: build-asan: runs-on: ubuntu-latest container: - image: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest + # still old image here since wrong gcc asan toolset was installed in new Dockerfile + image: ghcr.io/chatnoir-eu/resiliparse-manylinux2014_x86_64:latest env: DEBUG: "1" ASAN: "1" @@ -244,7 +282,7 @@ jobs: publish-wheels: runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/v') - needs: [ build-wheels, build-sdist, build-documentation ] + needs: [ build-wheels, build-wheels-qemu, build-sdist, build-documentation ] steps: - name: Download Wheels uses: actions/download-artifact@v3 From ff1fc9f28ac0dfd57237f8f324b5bfc9e1d328bc Mon Sep 17 00:00:00 2001 From: Jonas Dedden Date: Sat, 9 Mar 2024 16:09:44 +0100 Subject: [PATCH 5/7] Update buil-asan job to new image --- .github/workflows/build-wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index b08ec500..fe68d790 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -180,7 +180,7 @@ jobs: runs-on: ubuntu-latest container: # still old image here since wrong gcc asan toolset was installed in new Dockerfile - image: ghcr.io/chatnoir-eu/resiliparse-manylinux2014_x86_64:latest + image: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest env: DEBUG: "1" ASAN: "1" From d006d356bb210e99b49b03e926d987f80006d90f Mon Sep 17 00:00:00 2001 From: Jonas Dedden Date: Sat, 9 Mar 2024 16:14:18 +0100 Subject: [PATCH 6/7] Remove old comment --- .github/workflows/build-wheels.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index fe68d790..3a5628c3 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -179,7 +179,6 @@ jobs: build-asan: runs-on: ubuntu-latest container: - # still old image here since wrong gcc asan toolset was installed in new Dockerfile image: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest env: DEBUG: "1" From 4c10b5c687b86f261b8bc70878429695abb7d43a Mon Sep 17 00:00:00 2001 From: Jonas Dedden Date: Sat, 9 Mar 2024 16:22:59 +0100 Subject: [PATCH 7/7] Fix whitespace --- .github/workflows/build-wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 3a5628c3..af48f0b1 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -26,7 +26,7 @@ jobs: os: [ ubuntu-latest, windows-latest, macOS-latest ] env: CIBW_SKIP: "*-musllinux*" - CIBW_MANYLINUX_X86_64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest + CIBW_MANYLINUX_X86_64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest CIBW_ARCHS_MACOS: "x86_64 arm64" CIBW_ARCHS_LINUX: "x86_64" CIBW_TEST_SKIP: "*-macosx_arm64" # Apple Silicon wheels cannot be tested