BLD: separate cibuildwheel per os platform and tune py 3.9

equinor · Apr 20, 2021 · ac87b5f · ac87b5f
1 parent 7902696
commit ac87b5f
Show file tree

Hide file tree

Showing 4 changed files with 295 additions and 125 deletions.
diff --git a/.github/workflows/ci-test-xtgeo-cibuildwheel.yml b/.github/workflows/ci-test-xtgeo-cibuildwheel.yml
@@ -6,53 +6,31 @@ on:
   pull_request:
     branches: [master]
 
-env:
-  CIBW_BEFORE_ALL_LINUX: "sh scripts/swig_manylinux.sh"
-  CIBW_BEFORE_ALL_WINDOWS: "choco install -y --no-progress --allow-empty-checksums -r swig --version=4.0.1"
-  CIBW_BEFORE_ALL_MACOS: brew install swig
-
-  CIBW_BEFORE_TEST: |
-    pushd {project}
-    pip install -r requirements/requirements.txt
-    pip install -r requirements/requirements_test.txt
-
-  # 3.9 have trouble with install pytables on Mac and Win, due to missing wheel
-  # so full testing on those platforms are postponed.
-  CIBW_TEST_COMMAND_MACOS: |
-    pushd {project}
-    rm -rf ../xtgeo-testdata
-    git clone --depth 1 https://github.com/equinor/xtgeo-testdata ../xtgeo-testdata
-    pytest tests/test_surface --disable-warnings
-  CIBW_TEST_COMMAND_WINDOWS: |
-    pushd {project}
-    rmdir /s ../xtgeo-testdata
-    git clone --depth 1 https://github.com/equinor/xtgeo-testdata ../xtgeo-testdata
-    pytest tests/test_surface --disable-warnings
-  CIBW_TEST_COMMAND_LINUX: |
-    pushd {project}
-    rm -rf ../xtgeo-testdata
-    git clone --depth 1 https://github.com/equinor/xtgeo-testdata ../xtgeo-testdata
-    pytest tests --disable-warnings
-
-  CIBW_BUILD: |
-    cp39-manylinux_x86_64
-    cp39-macosx_x86_64
-    cp39-win_amd64
-
 # Verify that wheel build jobs succeed
 jobs:
-  build_multiwheels:
-    name: CIBW latest on ${{ matrix.os }}
-
+  build_linux_cibuildwheel:
+    name: CIBW python ${{ matrix.cibw_python }} on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
 
+    env:
+      CIBW_BEFORE_ALL: "sh scripts/swig_manylinux.sh"
+      CIBW_BEFORE_TEST: >
+        pushd {project} &&
+        pip install -r requirements/requirements.txt &&
+        pip install -r requirements/requirements_test.txt &&
+        git clone --depth 1 https://github.com/equinor/xtgeo-testdata ../xtgeo-testdata
+      CIBW_TEST_COMMAND: >
+        pushd {project} &&
+        pytest tests/test_common --disable-warnings -x
+      CIBW_BUILD: ${{ matrix.cibw_python }}-manylinux_x86_64
+      CIBW_MANYLINUX_X86_64_IMAGE: manylinux2010
+
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.8]
+        os: [ubuntu-latest]
+        cibw_python: [cp36, cp39]
     steps:
-      - uses: actions/checkout@v1
-
+      - uses: actions/checkout@v2
       - name: Set up Python
         uses: actions/setup-python@v2
 
@@ -65,3 +43,105 @@ jobs:
       - name: Build wheels and test
         run: |
           python -m cibuildwheel --output-dir wheelhouse
+
+  build_macos_cibuildwheel:
+    name: CIBW python ${{ matrix.cibw_python }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+
+    env:
+      CIBW_BEFORE_ALL: "brew install swig"
+      CIBW_BEFORE_TEST: >
+        pushd {project} &&
+        pip install -r requirements/requirements.txt &&
+        pip install -r requirements/requirements_test.txt &&
+        git clone --depth 1 https://github.com/equinor/xtgeo-testdata ../xtgeo-testdata
+      CIBW_TEST_COMMAND: >
+        pushd {project} &&
+        pytest tests --disable-warnings -x
+      CIBW_BUILD: ${{ matrix.cibw_python }}-macosx_x86_64
+
+    strategy:
+      matrix:
+        os: [macos-latest]
+        python-version: [3.8] # PY version for "Install cibuildwheel", default is 2.7!
+        cibw_python: [cp36, cp39]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+
+      - name: Install cibuildwheel
+        run: >
+          python -m pip install pip -U &&
+          python -m pip install wheel &&
+          python -m pip install cibuildwheel
+
+      - name: Build wheels and test python < 3.9
+        if: matrix.cibw_python != 'cp39'
+        run: >
+          python -m cibuildwheel --output-dir wheelhouse
+
+      - name: Build wheels and test python 3.9
+        if: matrix.cibw_python == 'cp39'
+        # note tests are a bit restricted due to missing pytables wheel
+        env:
+          CIBW_TEST_COMMAND: >
+            pushd {project} &&
+            pytest tests --disable-warnings --ignore tests/test_well -x
+
+        run: |
+          python -m cibuildwheel --output-dir wheelhouse
+
+  build_windows_cibuildwheel:
+    name: CIBW python ${{ matrix.cibw_python }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+
+    env:
+      CIBW_BEFORE_ALL: choco install -y --no-progress --allow-empty-checksums -r swig --version=4.0.1"
+      CIBW_BEFORE_TEST: >
+        pushd {project} &&
+        pip install -r requirements/requirements.txt &&
+        pip install -r requirements/requirements_test.txt &&
+        git clone --depth 1 https://github.com/equinor/xtgeo-testdata ../xtgeo-testdata
+
+      # a test with forks is skipped as this calls python in a subprocess, where
+      # the cibuildwheel on windows cannot find the xtgeo module
+      CIBW_TEST_COMMAND: >
+        pushd {project} && dir &&
+        pytest tests -x --ignore-glob="*forks.py"
+
+      CIBW_BUILD: ${{ matrix.cibw_python }}-win_amd64
+
+    strategy:
+      matrix:
+        os: [windows-latest]
+        cibw_python: [cp36, cp39]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+
+      - name: Install cibuildwheel
+        run: >
+          python -m pip install pip -U &&
+          python -m pip install wheel &&
+          python -m pip install cibuildwheel
+
+      - name: Build wheels and test python < 3.9
+        if: matrix.cibw_python != 'cp39'
+        run: |
+          python -m cibuildwheel --output-dir wheelhouse
+
+      - name: Build wheels and test python 3.9
+        if: matrix.cibw_python == 'cp39'
+        # note tests are a bit restricted due to missing pytables wheel
+        env:
+          CIBW_TEST_COMMAND: >
+            pushd {project} &&
+            pytest tests --disable-warnings --ignore tests/test_well
+            --ignore-glob="*forks.py" -x
+
+        run: |
+          python -m cibuildwheel --output-dir wheelhouse
diff --git a/.github/workflows/ci-test-xtgeo.yml b/.github/workflows/ci-test-xtgeo.yml
@@ -23,29 +23,38 @@ jobs:
             python-version: 3.6
           - os: windows-latest
             python-version: 3.8
+
     steps:
       - uses: actions/checkout@v1
+
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+
       - name: Install swig (osx)
         if: matrix.os == 'macos-latest'
         run: brew install swig
+
       - name: Install swig (win)
         if: matrix.os == 'windows-latest'
         run: "choco install -y --no-progress --allow-empty-checksums -r swig --version=4.0.1"
+
       - name: Install xtgeo
-        run: |
-          pip install pip -U
+        run: >
+          pip install pip -U &&
           pip install .
-      - name: Test with pytest
-        run: |
-          git clone --depth 1 https://github.com/equinor/xtgeo-testdata ../xtgeo-testdata
-          pip install -r requirements/requirements.txt
-          pip install -r requirements/requirements_test.txt
+
+      - name: Small version test
+        run: python -c "import xtgeo; print(xtgeo.__version__)"
+
+      - name: Full test
+        run: >
+          pip install -r requirements/requirements_test.txt &&
+          git clone --depth 1 https://github.com/equinor/xtgeo-testdata ../xtgeo-testdata &&
           pytest --disable-warnings -x
+
       - name: Build docs
-        run: |
-          pip install -r requirements/requirements_docs.txt
+        run: >
+          pip install -r requirements/requirements_docs.txt &&
           sphinx-build -W docs tmp/docs
diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml
@@ -13,28 +13,28 @@ jobs:
         os: [ubuntu-latest]
     env:
       OS: ${{ matrix.os }}
-      PYTHON: '3.7'
+      PYTHON: "3.7"
 
     steps:
-    - uses: actions/checkout@master
+      - uses: actions/checkout@master
 
-    - name: Setup Python
-      uses: actions/setup-python@master
-      with:
-        python-version: 3.7
+      - name: Setup Python
+        uses: actions/setup-python@master
+        with:
+          python-version: 3.7
 
-    - name: Install xtgeo
-      run: |
-        pip install pip -U
-        pip install .
+      - name: Install xtgeo
+        run: |
+          pip install pip -U
+          pip install .
 
-    - name: Generate coverage report
-      run: |
-        git clone --depth 1 https://github.com/equinor/xtgeo-testdata ../xtgeo-testdata
-        pip install -r requirements/requirements_test.txt
-        pip install pytest-cov
-        pytest tests --disable-warnings --cov=xtgeo --cov-report=xml:xtgeocoverage.xml;
+      - name: Generate coverage report
+        run: |
+          git clone --depth 1 https://github.com/equinor/xtgeo-testdata ../xtgeo-testdata
+          pip install -r requirements/requirements_test.txt
+          pip install pytest-cov
+          pytest tests --disable-warnings --cov=xtgeo --cov-report=xml:xtgeocoverage.xml;
 
-    - name: Upload coverage to Codecov
-      run: |
-        bash <(curl -s https://codecov.io/bash) -Z -c -f xtgeocoverage.xml
+      - name: Upload coverage to Codecov
+        run: |
+          bash <(curl -s https://codecov.io/bash) -Z -c -f xtgeocoverage.xml