Stub files for fastwarc (#44) #744
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Wheels | |
| on: | |
| push: | |
| tags: | |
| - '*' | |
| branches: | |
| - develop | |
| - ci/* | |
| pull_request: | |
| types: | |
| - opened | |
| - synchronize | |
| branches: | |
| - develop | |
| env: | |
| PYTHON_VERSION: "3.12" | |
| jobs: | |
| build-wheels: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ ubuntu-latest, windows-latest, macOS-12 ] | |
| env: | |
| CIBW_SKIP: "*-musllinux*" | |
| CIBW_ARCHS_MACOS: "x86_64 arm64" | |
| CIBW_ARCHS_LINUX: "x86_64" | |
| # CIBW_ARCHS_LINUX: "x86_64 aarch64" # https://github.com/pypa/cibuildwheel/issues/1771#issuecomment-1973003145 | |
| CIBW_MANYLINUX_X86_64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest | |
| CIBW_MANYLINUX_AARCH64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_aarch64:latest | |
| CIBW_TEST_SKIP: "*-macosx_arm64 *-manylinux_aarch64" # ARM64 wheels cannot be tested | |
| CIBW_REPAIR_WHEEL_COMMAND_MACOS: >- | |
| DYLD_LIBRARY_PATH=$LIBRARY_PATH delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} | |
| CIBW_BEFORE_BUILD_WINDOWS: "python -m pip install delvewheel" | |
| CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: >- | |
| delvewheel repair --add-path C:\vcpkg\installed\x64-windows\bin -w {dest_dir} {wheel} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Cache Vcpkg | |
| uses: actions/cache@v4 | |
| id: cache-vcpkg | |
| if: runner.os == 'macOS' || runner.os == 'Windows' | |
| with: | |
| path: | | |
| /usr/local/share/vcpkg/installed | |
| C:\vcpkg\installed | |
| key: ${{ runner.os }}-vcpkg-15 # INCREMENT ME!! | |
| - name: Install Vcpkg Dependencies | |
| if: (runner.os == 'macOS' || runner.os == 'Windows') && steps.cache-vcpkg.outputs.cache-hit != 'true' | |
| shell: bash | |
| run: | | |
| set -e | |
| VCPKG_CMD="vcpkg --overlay-ports .vcpkg/ports --overlay-triplets .vcpkg/triplets install" | |
| PKG_LIST="lexbor lz4 re2 uchardet zlib" | |
| if [ -d /usr/local/share/vcpkg/ports ]; then | |
| # MacOS | |
| $VCPKG_CMD --triplet=x64-osx $PKG_LIST | |
| $VCPKG_CMD --triplet=arm64-osx $PKG_LIST | |
| elif [ -d /c/vcpkg ]; then | |
| # Windows | |
| $VCPKG_CMD --triplet=x64-windows $PKG_LIST | |
| else | |
| echo "Unsupported platform." >&2 | |
| exit 1 | |
| fi | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| # - name: Set up QEMU | |
| # if: runner.os == 'Linux' | |
| # uses: docker/setup-qemu-action@v3 | |
| # with: | |
| # platforms: aarch64 | |
| - name: Build FastWARC | |
| uses: pypa/cibuildwheel@v2.16.5 | |
| with: | |
| package-dir: fastwarc | |
| output-dir: wheelhouse | |
| env: | |
| CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/fastwarc | |
| - name: Build Resiliparse | |
| uses: pypa/cibuildwheel@v2.16.5 | |
| with: | |
| package-dir: resiliparse | |
| output-dir: wheelhouse | |
| env: | |
| CIBW_BEFORE_TEST: >- | |
| python -c "import glob, platform; open('fastwarc.txt', 'w').write(glob.glob('wheelhouse/FastWARC-*cp' + ''.join(map(str, platform.python_version_tuple()[:2])) + '-*_' + platform.machine().lower() + '.whl')[0])" && | |
| python -m pip install -r fastwarc.txt | |
| CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/resiliparse | |
| - name: Upload Wheels | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: wheels | |
| path: ./wheelhouse/*.whl | |
| # To be merged with the above when https://github.com/pypa/cibuildwheel/issues/1771#issuecomment-1973003145 is fixed | |
| build-wheels-aarch64: | |
| runs-on: ubuntu-latest | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| env: | |
| CIBW_SKIP: "*cp38* *cp39* *-musllinux*" | |
| CIBW_MANYLINUX_AARCH64_IMAGE: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_aarch64:latest | |
| CIBW_ARCHS_LINUX: "aarch64" | |
| CIBW_TEST_SKIP: "*" | |
| CIBW_CONTAINER_ENGINE: "docker; create_args: --platform linux/arm64/v8" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| with: | |
| platforms: all | |
| - name: Build FastWARC | |
| uses: pypa/cibuildwheel@v2.16.5 | |
| with: | |
| package-dir: fastwarc | |
| output-dir: wheelhouse | |
| env: | |
| CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/fastwarc | |
| - name: Build Resiliparse | |
| uses: pypa/cibuildwheel@v2.16.5 | |
| with: | |
| package-dir: resiliparse | |
| output-dir: wheelhouse | |
| env: | |
| CIBW_BEFORE_TEST: >- | |
| python -c "import glob, platform; open('fastwarc.txt', 'w').write(glob.glob('wheelhouse/FastWARC-*cp' + ''.join(map(str, platform.python_version_tuple()[:2])) + '-*_' + platform.machine().lower() + '.whl')[0])" && | |
| python -m pip install -r fastwarc.txt | |
| CIBW_TEST_COMMAND: python -m pytest --capture=sys --verbose {project}/tests/resiliparse | |
| - name: Upload Wheels | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: wheels | |
| path: ./wheelhouse/*.whl | |
| build-sdist: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install Build Module | |
| run: python -m pip install build | |
| - name: Build FastWARC Source Dist | |
| run: python -m build --sdist --outdir dist fastwarc | |
| - name: Build Resiliparse Source Dist | |
| run: python -m build --sdist --outdir dist resiliparse | |
| - name: Upload Source Dists | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: sdist | |
| path: ./dist/*.tar.gz | |
| build-asan: | |
| runs-on: ubuntu-latest | |
| container: | |
| image: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest | |
| env: | |
| DEBUG: "1" | |
| ASAN: "1" | |
| ASAN_OPTIONS: leak_check_at_exit=0 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 # TODO: Upgrade to @v4 causes GLIBC failures | |
| with: | |
| fetch-depth: 0 | |
| - name: Build Extensions | |
| run: | | |
| set -e | |
| python${PYTHON_VERSION} -m pip install -e "fastwarc[all,test]" | |
| python${PYTHON_VERSION} -m pip install -e "resiliparse[all,test]" | |
| - name: Run Tests | |
| run: | | |
| export LD_PRELOAD="$(ldconfig -p | grep libasan | head -n1 | awk '{print $4}')" | |
| python${PYTHON_VERSION} -m pytest --capture=sys --verbose tests/ | |
| build-coverage: | |
| runs-on: ubuntu-latest | |
| needs: [ build-asan, build-sdist ] | |
| container: | |
| image: ghcr.io/chatnoir-eu/resiliparse-manylinux_2_28_x86_64:latest | |
| env: | |
| TRACE: "1" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Build Extensions | |
| run: | | |
| set -e | |
| python${PYTHON_VERSION} -m pip install cython codecov-cli | |
| python${PYTHON_VERSION} -m pip install -e "fastwarc[all,test]" | |
| python${PYTHON_VERSION} -m pip install -e "resiliparse[all,test]" | |
| - name: Run Tests | |
| run: python${PYTHON_VERSION} -m pytest --cov --cov-report xml tests/ | |
| # We cannot use the Codecov GitHub action due to incompatible Glibc versions | |
| - name: Upload to Codecov | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| run: | | |
| set -e | |
| python${PYTHON_VERSION} -m codecov_cli.main upload-process \ | |
| --file coverage.xml \ | |
| --sha ${{ github.sha }} \ | |
| --branch ${{ github.ref_name }} \ | |
| --slug ${{ github.repository }} \ | |
| --git-service github \ | |
| --fail-on-error | |
| - name: Upload Artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: coverage | |
| path: coverage.xml | |
| build-documentation: | |
| runs-on: ubuntu-latest | |
| needs: build-wheels | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Download Artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: wheels | |
| path: wheelhouse | |
| - name: Build Documentation | |
| run: | | |
| set -e | |
| python -m pip install twine | |
| grep -vE "fastwarc|resiliparse" docs/requirements.txt | xargs python -m pip install | |
| PYTHON_ABI="cp${PYTHON_VERSION/./}" | |
| find wheelhouse -name "FastWARC-*-${PYTHON_ABI}-*-manylinux_x86_64.whl" | xargs -I% python -m pip install "%[all]" | |
| find wheelhouse -name "Resiliparse-*-${PYTHON_ABI}-*-manylinux_x86_64.whl" | xargs -I% python -m pip install "%[all]" | |
| cd docs | |
| make html | |
| - name: Trigger Readthedocs Build | |
| if: github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/tags/v') | |
| uses: dfm/rtds-action@v1 | |
| with: | |
| webhook_url: ${{ secrets.RTDS_WEBHOOK_URL }} | |
| webhook_token: ${{ secrets.RTDS_WEBHOOK_TOKEN }} | |
| commit_ref: ${{ github.ref }} | |
| publish-wheels: | |
| runs-on: ubuntu-latest | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| needs: [ build-wheels, build-wheels-aarch64, build-sdist, build-documentation ] | |
| steps: | |
| - name: Download Wheels | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: wheels | |
| - name: Download Source Dist | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: sdist | |
| - name: Publish to PyPi | |
| env: | |
| TWINE_USERNAME: __token__ | |
| TWINE_PASSWORD1: ${{ secrets.PYPI_FASTWARC_API_TOKEN }} | |
| TWINE_PASSWORD2: ${{ secrets.PYPI_RESILIPARSE_API_TOKEN }} | |
| run: | | |
| set -e | |
| python3 -m pip install twine | |
| TWINE_PASSWORD="$TWINE_PASSWORD1" python3 -m twine upload FastWARC-*.whl fastwarc-*.tar.gz | |
| TWINE_PASSWORD="$TWINE_PASSWORD2" python3 -m twine upload Resiliparse-*.whl resiliparse-*.tar.gz | |
| - name: Wait | |
| run: sleep 30 | |
| - name: Trigger Readthedocs Build | |
| uses: dfm/rtds-action@v1 | |
| with: | |
| webhook_url: ${{ secrets.RTDS_WEBHOOK_URL }} | |
| webhook_token: ${{ secrets.RTDS_WEBHOOK_TOKEN }} | |
| commit_ref: ${{ github.ref }} |