Merge pull request #89 from ecmwf/feature/binary-wheel

WIP: Feature/binary wheel

.github/workflows/build-wheel-linux.yml

@@ -0,0 +1,150 @@
+# (C) Copyright 2024- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+name: Build Linux
+
+on:
+  # Trigger the workflow manually
+  workflow_dispatch: ~
+
+  # Allow to be called from another workflow
+  workflow_call: ~
+
+  # repository_dispatch:
+  #   types: [eccodes-updated]
+
+  push:
+    paths:
+    - 'scripts/common.sh'
+    - 'scripts/select-python-linux.sh'
+    - 'scripts/wheel-linux.sh'
+    - 'scripts/build-linux.sh'
+    - 'scripts/test-linux.sh'
+    - 'scripts/copy-licences.py'
+    - '.github/workflows/build-wheel-linux.yml'
+
+# to allow the action to run on the manylinux docker image based on CentOS 7
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+
+jobs:
+
+  build:
+
+    # if: false  # for temporarily disabling for debugging
+
+    runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6]
+    container: dockcross/manylinux2014-x64:latest
+
+    name: Build manylinux2014
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - run: ./scripts/build-linux.sh
+
+      ################################################################
+      - run: ./scripts/wheel-linux.sh 3.8
+      - uses: actions/upload-artifact@v2
+        name: Upload wheel 3.8
+        with:
+          name: wheel-manylinux2014-3.8
+          path: wheelhouse/*.whl
+
+      # ################################################################
+      - run: ./scripts/wheel-linux.sh 3.9
+      - uses: actions/upload-artifact@v2
+        name: Upload wheel 3.9
+        with:
+          name: wheel-manylinux2014-3.9
+          path: wheelhouse/*.whl
+
+      # ################################################################
+      - run: ./scripts/wheel-linux.sh 3.10
+      - uses: actions/upload-artifact@v2
+        name: Upload wheel 3.10
+        with:
+          name: wheel-manylinux2014-3.10
+          path: wheelhouse/*.whl
+
+      # ################################################################
+      - run: ./scripts/wheel-linux.sh 3.11
+      - uses: actions/upload-artifact@v2
+        name: Upload wheel 3.11
+        with:
+          name: wheel-manylinux2014-3.11
+          path: wheelhouse/*.whl
+
+      # ################################################################
+      - run: ./scripts/wheel-linux.sh 3.12
+      - uses: actions/upload-artifact@v2
+        name: Upload wheel 3.12
+        with:
+          name: wheel-manylinux2014-3.12
+          path: wheelhouse/*.whl
+
+  test:
+
+    needs: build
+
+    strategy:
+      fail-fast: false
+      matrix: # We don't test 3.6, as it is not supported anymore by github actions
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+
+    runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6]
+
+    name: Test with ${{ matrix.python-version }}
+
+    steps:
+
+      - uses: actions/checkout@v2
+
+      - uses: actions/download-artifact@v2
+        with:
+          name: wheel-manylinux2014-${{ matrix.python-version }}
+
+      - run: ./scripts/test-linux.sh  ${{ matrix.python-version }}
+
+
+  deploy:
+
+    if: ${{ github.ref_type == 'tag' || github.event_name == 'release' }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+
+    needs: [test, build]
+
+    name: Deploy wheel ${{ matrix.python-version }}
+
+    runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6]
+
+
+    steps:
+
+      - run: mkdir artifact-${{ matrix.python-version }}
+
+      - uses: actions/checkout@v2
+
+      - uses: actions/download-artifact@v2
+        with:
+          name: wheel-manylinux2014-${{ matrix.python-version }}
+          path: artifact-${{ matrix.python-version }}
+
+      - run: |
+          source ./scripts/select-python-linux.sh 3.10
+          pip3 install twine
+          ls -l artifact-${{ matrix.python-version }}/*.whl
+          twine upload artifact-${{ matrix.python-version }}/*.whl
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/build-wheel-macos.yml

@@ -0,0 +1,155 @@
+# (C) Copyright 2024- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+name: Build MacOS ARM
+
+on:
+  # Trigger the workflow manually
+  workflow_dispatch: ~
+
+  # allow to be called from another workflow
+  workflow_call: ~
+
+  # repository_dispatch:
+  #   types: [eccodes-updated]
+
+  push:
+    paths:
+    - 'scripts/common.sh'
+    - 'scripts/select-python-macos.sh'
+    - 'scripts/build-macos.sh'
+    - 'scripts/wheel-macos.sh'
+    - 'scripts/test-macos.sh'
+    - 'scripts/copy-licences.py'
+    - '.github/workflows/build-wheel-macos.yml'
+
+# We don't use "actions/setup-python@v4" as it installs a universal python
+# which creates universal wheels. We want to create wheels for the specific
+# architecture we are running on.
+
+jobs:
+
+  build:
+
+    # if: false  # for temporarily disabling for debugging
+
+    strategy:
+      matrix:
+        arch_type: [ARM64, X64]
+    runs-on: [self-hosted, macOS, "${{ matrix.arch_type }}"]
+
+    name: Build
+
+    steps:
+
+    - run: sudo mkdir -p /Users/runner
+    - run: sudo chown administrator:staff /Users/runner
+
+    - uses: actions/checkout@v2
+
+    - run: ./scripts/build-macos.sh "3.10"
+
+    - run: ./scripts/wheel-macos.sh "3.9"
+    - run: ls -l wheelhouse
+    - uses: actions/upload-artifact@v2
+      name: Upload wheel 3.9 ${{ matrix.arch_type }}
+      with:
+        name: wheel-macos-${{ matrix.arch_type }}-3.9
+        path: wheelhouse/*.whl
+    - run: rm -fr wheelhouse
+
+    - run: ./scripts/wheel-macos.sh "3.10"
+    - run: ls -l wheelhouse
+    - uses: actions/upload-artifact@v2
+      name: Upload wheel 3.10 ${{ matrix.arch_type }}
+      with:
+        name: wheel-macos-${{ matrix.arch_type }}-3.10
+        path: wheelhouse/*.whl
+    - run: rm -fr wheelhouse
+
+    - run: ./scripts/wheel-macos.sh "3.11"
+    - run: ls -l wheelhouse
+    - uses: actions/upload-artifact@v2
+      name: Upload wheel 3.11 ${{ matrix.arch_type }}
+      with:
+        name: wheel-macos-${{ matrix.arch_type }}-3.11
+        path: wheelhouse/*.whl
+    - run: rm -fr wheelhouse
+
+    - run: ./scripts/wheel-macos.sh "3.12"
+    - run: ls -l wheelhouse
+    - uses: actions/upload-artifact@v2
+      name: Upload wheel 3.12 ${{ matrix.arch_type }}
+      with:
+        name: wheel-macos-${{ matrix.arch_type }}-3.12
+        path: wheelhouse/*.whl
+    - run: rm -fr wheelhouse
+
+  test:
+    needs: build
+
+    strategy:
+      fail-fast: true
+      max-parallel: 1
+      matrix:
+        arch_type: [ARM64, X64]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+
+    runs-on: [self-hosted, macOS, "${{ matrix.arch_type }}"]
+
+    name: Test with Python ${{ matrix.python-version }} ${{ matrix.arch_type }}
+
+    steps:
+
+    - uses: actions/checkout@v2
+
+    - uses: actions/download-artifact@v2
+      with:
+        name: wheel-macos-${{ matrix.arch_type }}-${{ matrix.python-version }}
+
+    - run: ./scripts/test-macos.sh  ${{ matrix.python-version }}
+
+
+  deploy:
+
+    if: ${{ github.ref_type == 'tag' || github.event_name == 'release' }}
+
+    needs: [test, build]
+
+    name: Deploy wheel ${{ matrix.python-version }} ${{ matrix.arch_type }}
+
+    strategy:
+      fail-fast: true
+      max-parallel: 1
+      matrix:
+        arch_type: [ARM64, X64]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+
+    runs-on: [self-hosted, macOS, "${{ matrix.arch_type }}"]
+
+    steps:
+
+      - run: mkdir artifact-${{ matrix.arch_type }}-${{ matrix.python-version }}
+
+      - uses: actions/download-artifact@v2
+        with:
+          name: wheel-macos-${{ matrix.arch_type }}-${{ matrix.python-version }}
+          path: artifact-${{ matrix.arch_type }}-${{ matrix.python-version }}
+
+      - run: |
+          source ./scripts/select-python-macos.sh ${{ matrix.python-version }}
+          VENV_DIR=./dist_venv_${{ matrix.python-version }}
+          rm -rf ${VENV_DIR}
+          python3 -m venv ${VENV_DIR}
+          source ${VENV_DIR}/bin/activate
+          pip3 install twine
+          ls -l artifact-${{ matrix.arch_type }}-${{ matrix.python-version }}/*.whl
+          twine upload artifact-${{ matrix.arch_type }}-${{ matrix.python-version }}/*.whl
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/cd.yml

@@ -1,11 +1,31 @@
+# (C) Copyright 2024- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
 name: cd
 
 on:
+  # Trigger the workflow manually
+  workflow_dispatch: ~
+
   push:
     tags:
       - '**'
-
+    paths:
+      - '.github/workflows/cd.yml'
 jobs:
+  wheel-linux:
+    uses: ./.github/workflows/build-wheel-linux.yml
+    secrets: inherit
+  wheel-macos:
+    uses: ./.github/workflows/build-wheel-macos.yml
+    secrets: inherit
   pypi:
+    needs: [wheel-linux, wheel-macos]
     uses: ecmwf-actions/reusable-workflows/.github/workflows/cd-pypi.yml@v2
     secrets: inherit
+

.github/workflows/check-and-publish.yml

@@ -74,17 +74,19 @@ jobs:
       - run: pip install -r tests/requirements.txt
       - run: pip freeze
       - run: env | sort
-      - run: python -m eccodes selfcheck
+      - run: ECCODES_PYTHON_USE_FINDLIBS=1 python -m eccodes selfcheck
 
       - run: pytest
         if: matrix.method == 'conda' && matrix.platform == 'windows-latest'
         env:
           ECCODES_DEFINITION_PATH: 'C:/Miniconda/Library/share/eccodes/definitions'
           ECCODES_SAMPLES_PATH: 'C:/Miniconda/Library/share/eccodes/samples'
+          ECCODES_PYTHON_USE_FINDLIBS: '1'
 
       - run: pytest
         if: matrix.method != 'conda' || matrix.platform != 'windows-latest'
-
+        env:
+          ECCODES_PYTHON_USE_FINDLIBS: '1'
       # pytest -v --cov=. --cov-report=xml --cov-branch .
 
       - name: Upload coverage to Codecov

.github/workflows/ci.yml

@@ -8,6 +8,10 @@ on:
       - "develop"
     tags-ignore:
       - "**"
+    paths-ignore:
+      - "scripts/**"
+      - ".github/workflows/*wheel*.yml"
+      - ".github/workflows/cd.yml"
 
   # Trigger the workflow on pull request
   pull_request: ~
@@ -28,6 +32,7 @@ jobs:
     with:
       eccodes-python: ecmwf/eccodes-python@${{ github.event.pull_request.head.sha || github.sha }}
       codecov_upload: true
+      python_qa: true
     secrets: inherit
 
   # Build downstream packages on HPC