From ac87b5f19541440c7642ebefa6ea9725daec2a84 Mon Sep 17 00:00:00 2001 From: Jan Date: Fri, 9 Apr 2021 08:16:48 +0200 Subject: [PATCH] BLD: separate cibuildwheel per os platform and tune py 3.9 --- .../workflows/ci-test-xtgeo-cibuildwheel.yml | 160 ++++++++++---- .github/workflows/ci-test-xtgeo.yml | 27 ++- .github/workflows/codecov.yml | 38 ++-- .github/workflows/deploy-xtgeo-pypi.yml | 195 +++++++++++++----- 4 files changed, 295 insertions(+), 125 deletions(-) diff --git a/.github/workflows/ci-test-xtgeo-cibuildwheel.yml b/.github/workflows/ci-test-xtgeo-cibuildwheel.yml index 6aa927444..d0c47ccf5 100644 --- a/.github/workflows/ci-test-xtgeo-cibuildwheel.yml +++ b/.github/workflows/ci-test-xtgeo-cibuildwheel.yml @@ -6,53 +6,31 @@ on: pull_request: branches: [master] -env: - CIBW_BEFORE_ALL_LINUX: "sh scripts/swig_manylinux.sh" - CIBW_BEFORE_ALL_WINDOWS: "choco install -y --no-progress --allow-empty-checksums -r swig --version=4.0.1" - CIBW_BEFORE_ALL_MACOS: brew install swig - - CIBW_BEFORE_TEST: | - pushd {project} - pip install -r requirements/requirements.txt - pip install -r requirements/requirements_test.txt - - # 3.9 have trouble with install pytables on Mac and Win, due to missing wheel - # so full testing on those platforms are postponed. - CIBW_TEST_COMMAND_MACOS: | - pushd {project} - rm -rf ../xtgeo-testdata - git clone --depth 1 https://github.com/equinor/xtgeo-testdata ../xtgeo-testdata - pytest tests/test_surface --disable-warnings - CIBW_TEST_COMMAND_WINDOWS: | - pushd {project} - rmdir /s ../xtgeo-testdata - git clone --depth 1 https://github.com/equinor/xtgeo-testdata ../xtgeo-testdata - pytest tests/test_surface --disable-warnings - CIBW_TEST_COMMAND_LINUX: | - pushd {project} - rm -rf ../xtgeo-testdata - git clone --depth 1 https://github.com/equinor/xtgeo-testdata ../xtgeo-testdata - pytest tests --disable-warnings - - CIBW_BUILD: | - cp39-manylinux_x86_64 - cp39-macosx_x86_64 - cp39-win_amd64 - # Verify that wheel build jobs succeed jobs: - build_multiwheels: - name: CIBW latest on ${{ matrix.os }} - + build_linux_cibuildwheel: + name: CIBW python ${{ matrix.cibw_python }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} + env: + CIBW_BEFORE_ALL: "sh scripts/swig_manylinux.sh" + CIBW_BEFORE_TEST: > + pushd {project} && + pip install -r requirements/requirements.txt && + pip install -r requirements/requirements_test.txt && + git clone --depth 1 https://github.com/equinor/xtgeo-testdata ../xtgeo-testdata + CIBW_TEST_COMMAND: > + pushd {project} && + pytest tests/test_common --disable-warnings -x + CIBW_BUILD: ${{ matrix.cibw_python }}-manylinux_x86_64 + CIBW_MANYLINUX_X86_64_IMAGE: manylinux2010 + strategy: matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [3.8] + os: [ubuntu-latest] + cibw_python: [cp36, cp39] steps: - - uses: actions/checkout@v1 - + - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v2 @@ -65,3 +43,105 @@ jobs: - name: Build wheels and test run: | python -m cibuildwheel --output-dir wheelhouse + + build_macos_cibuildwheel: + name: CIBW python ${{ matrix.cibw_python }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + + env: + CIBW_BEFORE_ALL: "brew install swig" + CIBW_BEFORE_TEST: > + pushd {project} && + pip install -r requirements/requirements.txt && + pip install -r requirements/requirements_test.txt && + git clone --depth 1 https://github.com/equinor/xtgeo-testdata ../xtgeo-testdata + CIBW_TEST_COMMAND: > + pushd {project} && + pytest tests --disable-warnings -x + CIBW_BUILD: ${{ matrix.cibw_python }}-macosx_x86_64 + + strategy: + matrix: + os: [macos-latest] + python-version: [3.8] # PY version for "Install cibuildwheel", default is 2.7! + cibw_python: [cp36, cp39] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + + - name: Install cibuildwheel + run: > + python -m pip install pip -U && + python -m pip install wheel && + python -m pip install cibuildwheel + + - name: Build wheels and test python < 3.9 + if: matrix.cibw_python != 'cp39' + run: > + python -m cibuildwheel --output-dir wheelhouse + + - name: Build wheels and test python 3.9 + if: matrix.cibw_python == 'cp39' + # note tests are a bit restricted due to missing pytables wheel + env: + CIBW_TEST_COMMAND: > + pushd {project} && + pytest tests --disable-warnings --ignore tests/test_well -x + + run: | + python -m cibuildwheel --output-dir wheelhouse + + build_windows_cibuildwheel: + name: CIBW python ${{ matrix.cibw_python }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + + env: + CIBW_BEFORE_ALL: choco install -y --no-progress --allow-empty-checksums -r swig --version=4.0.1" + CIBW_BEFORE_TEST: > + pushd {project} && + pip install -r requirements/requirements.txt && + pip install -r requirements/requirements_test.txt && + git clone --depth 1 https://github.com/equinor/xtgeo-testdata ../xtgeo-testdata + + # a test with forks is skipped as this calls python in a subprocess, where + # the cibuildwheel on windows cannot find the xtgeo module + CIBW_TEST_COMMAND: > + pushd {project} && dir && + pytest tests -x --ignore-glob="*forks.py" + + CIBW_BUILD: ${{ matrix.cibw_python }}-win_amd64 + + strategy: + matrix: + os: [windows-latest] + cibw_python: [cp36, cp39] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + + - name: Install cibuildwheel + run: > + python -m pip install pip -U && + python -m pip install wheel && + python -m pip install cibuildwheel + + - name: Build wheels and test python < 3.9 + if: matrix.cibw_python != 'cp39' + run: | + python -m cibuildwheel --output-dir wheelhouse + + - name: Build wheels and test python 3.9 + if: matrix.cibw_python == 'cp39' + # note tests are a bit restricted due to missing pytables wheel + env: + CIBW_TEST_COMMAND: > + pushd {project} && + pytest tests --disable-warnings --ignore tests/test_well + --ignore-glob="*forks.py" -x + + run: | + python -m cibuildwheel --output-dir wheelhouse diff --git a/.github/workflows/ci-test-xtgeo.yml b/.github/workflows/ci-test-xtgeo.yml index 60c55fbcf..8646235d0 100644 --- a/.github/workflows/ci-test-xtgeo.yml +++ b/.github/workflows/ci-test-xtgeo.yml @@ -23,29 +23,38 @@ jobs: python-version: 3.6 - os: windows-latest python-version: 3.8 + steps: - uses: actions/checkout@v1 + - name: Set up Python uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + - name: Install swig (osx) if: matrix.os == 'macos-latest' run: brew install swig + - name: Install swig (win) if: matrix.os == 'windows-latest' run: "choco install -y --no-progress --allow-empty-checksums -r swig --version=4.0.1" + - name: Install xtgeo - run: | - pip install pip -U + run: > + pip install pip -U && pip install . - - name: Test with pytest - run: | - git clone --depth 1 https://github.com/equinor/xtgeo-testdata ../xtgeo-testdata - pip install -r requirements/requirements.txt - pip install -r requirements/requirements_test.txt + + - name: Small version test + run: python -c "import xtgeo; print(xtgeo.__version__)" + + - name: Full test + run: > + pip install -r requirements/requirements_test.txt && + git clone --depth 1 https://github.com/equinor/xtgeo-testdata ../xtgeo-testdata && pytest --disable-warnings -x + - name: Build docs - run: | - pip install -r requirements/requirements_docs.txt + run: > + pip install -r requirements/requirements_docs.txt && sphinx-build -W docs tmp/docs diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index a62037139..89df89a01 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -13,28 +13,28 @@ jobs: os: [ubuntu-latest] env: OS: ${{ matrix.os }} - PYTHON: '3.7' + PYTHON: "3.7" steps: - - uses: actions/checkout@master + - uses: actions/checkout@master - - name: Setup Python - uses: actions/setup-python@master - with: - python-version: 3.7 + - name: Setup Python + uses: actions/setup-python@master + with: + python-version: 3.7 - - name: Install xtgeo - run: | - pip install pip -U - pip install . + - name: Install xtgeo + run: | + pip install pip -U + pip install . - - name: Generate coverage report - run: | - git clone --depth 1 https://github.com/equinor/xtgeo-testdata ../xtgeo-testdata - pip install -r requirements/requirements_test.txt - pip install pytest-cov - pytest tests --disable-warnings --cov=xtgeo --cov-report=xml:xtgeocoverage.xml; + - name: Generate coverage report + run: | + git clone --depth 1 https://github.com/equinor/xtgeo-testdata ../xtgeo-testdata + pip install -r requirements/requirements_test.txt + pip install pytest-cov + pytest tests --disable-warnings --cov=xtgeo --cov-report=xml:xtgeocoverage.xml; - - name: Upload coverage to Codecov - run: | - bash <(curl -s https://codecov.io/bash) -Z -c -f xtgeocoverage.xml + - name: Upload coverage to Codecov + run: | + bash <(curl -s https://codecov.io/bash) -Z -c -f xtgeocoverage.xml diff --git a/.github/workflows/deploy-xtgeo-pypi.yml b/.github/workflows/deploy-xtgeo-pypi.yml index 8e6c02df1..a872f4874 100644 --- a/.github/workflows/deploy-xtgeo-pypi.yml +++ b/.github/workflows/deploy-xtgeo-pypi.yml @@ -4,83 +4,164 @@ on: release: types: [created] -env: - CIBW_BEFORE_ALL_LINUX: "sh scripts/swig_manylinux.sh" - CIBW_BEFORE_ALL_WINDOWS: "choco install -y --no-progress --allow-empty-checksums -r swig --version=4.0.1" - CIBW_BEFORE_ALL_MACOS: "brew install swig" - CIBW_BEFORE_TEST: | - pushd {project} - pip install -r requirements/requirements.txt - pip install -r requirements/requirements_test.txt - - # 3.9 have trouble with install pytables on Mac and Win, due to missing wheel - # so full testing on those platforms are postponed. - CIBW_TEST_COMMAND_MACOS: | - pushd {project} - rm -rf ../xtgeo-testdata - git clone --depth 1 https://github.com/equinor/xtgeo-testdata ../xtgeo-testdata - pytest tests/test_surface --disable-warnings - CIBW_TEST_COMMAND_WINDOWS: | - pushd {project} - rm -rf ../xtgeo-testdata - git clone --depth 1 https://github.com/equinor/xtgeo-testdata ../xtgeo-testdata - pytest tests/test_surface --disable-warnings - CIBW_TEST_COMMAND_LINUX: | - pushd {project} - rm -rf ../xtgeo-testdata - git clone --depth 1 https://github.com/equinor/xtgeo-testdata ../xtgeo-testdata - pytest tests --disable-warnings - - CIBW_BUILD: | - cp36-manylinux_x86_64 - cp37-manylinux_x86_64 - cp38-manylinux_x86_64 - cp39-manylinux_x86_64 - cp36-macosx_x86_64 - cp37-macosx_x86_64 - cp38-macosx_x86_64 - cp39-macosx_x86_64 - cp36-win_amd64 - cp37-win_amd64 - cp38-win_amd64 - cp38-win_amd64 - cp39-win_amd64 - +# Verify that wheel build jobs succeed jobs: - build_wheels: - name: Deploy wheels to PYPI for ${{ matrix.os }} + deploy_linux_cibuildwheel: + name: CIBW deploy ${{ matrix.cibw_python }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + + env: + CIBW_BEFORE_ALL: "sh scripts/swig_manylinux.sh" + CIBW_BEFORE_TEST: > + pushd {project} && + pip install -r requirements/requirements.txt && + pip install -r requirements/requirements_test.txt && + git clone --depth 1 https://github.com/equinor/xtgeo-testdata ../xtgeo-testdata + CIBW_TEST_COMMAND: > + pushd {project} && + pytest tests --disable-warnings + CIBW_BUILD: ${{ matrix.cibw_python }}-manylinux_x86_64 + CIBW_MANYLINUX_X86_64_IMAGE: manylinux2010 + + strategy: + matrix: + os: [ubuntu-latest] + cibw_python: [cp36, cp37, cp38, cp39] + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + + - name: Install cibuildwheel + run: > + python -m pip install pip -U && + python -m pip install wheel && + python -m pip install cibuildwheel + + - name: Build wheels and test + run: | + python -m cibuildwheel --output-dir wheelhouse + + - name: Publish to PYPI Linux + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: > + pip install twine && + twine upload wheelhouse/* + deploy_macos_cibuildwheel: + name: CIBW deploy ${{ matrix.cibw_python }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} + env: + CIBW_BEFORE_ALL: "brew install swig" + CIBW_BEFORE_TEST: > + pushd {project} && + pip install -r requirements/requirements.txt && + pip install -r requirements/requirements_test.txt && + git clone --depth 1 https://github.com/equinor/xtgeo-testdata ../xtgeo-testdata + CIBW_TEST_COMMAND: > + pushd {project} && + pytest tests --disable-warnings && + CIBW_BUILD: ${{ matrix.cibw_python }}-macosx_x86_64 + strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-latest] - python-version: [3.8] + os: [macos-latest] + python-version: [3.8] # PY version for "Install cibuildwheel", default is 2.7! + cibw_python: [cp36, cp37, cp38, cp39] steps: - - uses: actions/checkout@v1 - - - name: Set up Python ${{ matrix.python-version }} + - uses: actions/checkout@v2 + - name: Set up Python uses: actions/setup-python@v2 - name: Install cibuildwheel - run: | + run: > + python -m pip install pip -U && + python -m pip install wheel && python -m pip install cibuildwheel - - name: Build wheels and test + - name: Build wheels and test python < 3.9 + if: matrix.cibw_python != 'cp39' run: | python -m cibuildwheel --output-dir wheelhouse - - name: Check wheels and paths (ubuntu only) + - name: Build wheels and test python 3.9 + if: matrix.cibw_python == 'cp39' + # note tests are a bit restricted due to missing pytables wheel + env: + CIBW_TEST_COMMAND: > + pushd {project} && + pytest tests --disable-warnings --ignore tests/test_well + run: | - ls -l wheelhouse - ls -latr - if: matrix.os == 'ubuntu-latest' + python -m cibuildwheel --output-dir wheelhouse - - name: Publish to PYPI + - name: Publish to PYPI MacOS env: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: > + pip install twine && + twine upload wheelhouse/* + + deploy_windows_cibuildwheel: + name: CIBW deploy ${{ matrix.cibw_python }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + + env: + CIBW_BEFORE_ALL: choco install -y --no-progress --allow-empty-checksums -r swig --version=4.0.1" + CIBW_BEFORE_TEST: > + pushd {project} && + pip install -r requirements/requirements.txt && + pip install -r requirements/requirements_test.txt && + git clone --depth 1 https://github.com/equinor/xtgeo-testdata ../xtgeo-testdata + + # a test with forks is skipped as this calls python in a subprocess, where + # the cibuildwheel on windows cannot find the xtgeo module + CIBW_TEST_COMMAND: > + pushd {project} && + pytest tests --disable-warnings --ignore-glob="*forks.py" + CIBW_BUILD: ${{ matrix.cibw_python }}-win_amd64 + + strategy: + matrix: + os: [windows-latest] + cibw_python: [cp36, cp37, cp38, cp39] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + + - name: Install cibuildwheel + run: > + python -m pip install pip -U && + python -m pip install wheel && + python -m pip install cibuildwheel + + - name: Build wheels and test python < 3.9 + if: matrix.cibw_python != 'cp39' + run: | + python -m cibuildwheel --output-dir wheelhouse + + - name: Build wheels and test python 3.9 + if: matrix.cibw_python == 'cp39' + # note tests are a bit restricted due to missing pytables wheel + env: + CIBW_TEST_COMMAND: > + pushd {project} && + pytest tests --disable-warnings --ignore tests/test_well + --ignore-glob="*forks.py" -x run: | - pip install twine + python -m cibuildwheel --output-dir wheelhouse + + - name: Publish to PYPI Windows + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: > + pip install twine && twine upload wheelhouse/*