Automate build and upload of CPython wheels (#72)

* use PyPA build package to produce wheels

* update the Makefile and install_pytrexio to use PyPA build package

* remove MacOS-11 from runners

* [MacOS] portable expression for FreeBSD sed

* disable usage of NUMPY_INCLUDEDIR env variable

* activate PyPI upload and disable TestPyPI

.github/workflows/actions.yml

@@ -34,6 +34,7 @@ jobs:
 
     runs-on: ubuntu-latest
     name: x86 Ubuntu latest
+    needs: get_commit_message
 
     steps:
       - uses: actions/checkout@v2
@@ -61,24 +62,23 @@ jobs:
 
       - name: install Python API
         run: make python-install
-        # alternatively we can also run pip install trexio to check PyPI installation
 
       - name: check Python API
         run: make python-test
 
-      - name: build Python API distribution
-        run: make python-sdist
+      - name: build and move Python API distribution
+        run: |
+          make python-sdist
+          cp python/dist/trexio-*.tar.gz .
 
       - name: publish Python API distribution as an artifact
-        needs: get_commit_message
         if: >-
           contains(needs.get_commit_message.outputs.message, '[wheel build]') ||
           github.event_name == 'release'
         uses: actions/upload-artifact@v2
         with:
           name: pytrexio-source
           path: ./trexio-*.tar.gz
-        working-directory: python/dist
 
       - name: clean
         run: make clean

.github/workflows/build-wheels.yml

@@ -1,18 +1,16 @@
 
 # Controls when the workflow will run
 on:
-  # run this workflow after the TREXIO CI completed
+  # Run this workflow after the TREXIO CI completed
   workflow_run:
     workflows: [ "TREXIO CI" ]
     branches: [ master ]
     types:
       - completed
 
 # Workflow to build and publish wheels.
-#
-# in the get_commit_message job: Include [wheel build] in your commit message to trigger the build.
-name: PyPI wheels build
-
+# in the get_commit_message job: Include [wheel build] in your commit message to trigger this build.
+name: Build CPython wheels
 jobs:
 
   get_commit_message:
@@ -45,7 +43,7 @@ jobs:
     strategy:
       matrix:
         manylinux_tag: [2010_x86_64, 2014_x86_64, 2_24_x86_64]
-
+      
     steps:
       - name: Checkout the branch
         uses: actions/checkout@v2
@@ -57,26 +55,44 @@ jobs:
 
       - name: Install build dependencies
         run: python -m pip install -U setuptools
-
+
+      - name: Compute the PYTREXIO_VERSION environment variable
+        run: echo "PYTREXIO_VERSION=$(grep __version__ python/pytrexio/_version.py | cut -d\  -f3 | tr -d '"')" >> $GITHUB_ENV
+
+      - name: Print the PYTREXIO_VERSION
+        run: echo ${{ env.PYTREXIO_VERSION }}
+
+      # Conventional download-artifact action does not work for artifact produced in a different workflow,
+      # which is the case here (TREXIO CI produced the Python API distribution tarball)
       - name: Download the Python API distribution tarball
-        uses: actions/download-artifact@v2
+        uses: dawidd6/action-download-artifact@v2
         with:
+          # Specify the name of the workflow file which uploaded the tarball
+          workflow: actions.yml
+          workflow_conclusion: success
           name: pytrexio-source
           path: python
-
-      - name: Build manylinux wheels
-        run: |
-          docker pull ghcr.io/q-posev/hdf5_1_12_on_${{ matrix.manylinux_tag }}:latest
-          docker run --rm --env PLAT=manylinux${{ matrix.manylinux_tag }} --volume `pwd`:/tmp --workdir /tmp ghcr.io/q-posev/hdf5_1_12_on_${{ matrix.manylinux_tag }} /bin/bash build_manylinux_wheels.sh trexio-*.tar.gz
+
+      # at the moment we have to pull the custom container with pre-installed HDF5
+      # the containers are built and stored in GitHub container registry ghcr.io/q-posev
+      - name: Pull the manylinux Docker container with HDF5
+        run: docker pull ghcr.io/q-posev/hdf5_1_12_on_${{ matrix.manylinux_tag }}:latest
+
+      - name: Build wheels for different versions of CPython inside the Docker container
+        run: >
+          docker run --rm 
+          --env PLAT=manylinux${{ matrix.manylinux_tag }} 
+          --volume `pwd`:/tmp 
+          --workdir /tmp 
+          ghcr.io/q-posev/hdf5_1_12_on_${{ matrix.manylinux_tag }} 
+          /bin/bash build_manylinux_wheels.sh trexio-${{ env.PYTREXIO_VERSION }}.tar.gz
         working-directory: python
 
       - name: Upload produced wheels as artifacts
         uses: actions/upload-artifact@v2
         with:
           name: pytrexio-manylinux-${{ matrix.manylinux_tag }}
-          path: ./wheelhouse/*.whl
-        working-directory: python
-
+          path: ./python/wheelhouse/*.whl
 
   build_macos_wheels:
     name: Build MacOS wheels for different versions of CPython
@@ -87,11 +103,14 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [macos-11, macos-10.15]
-        python-version: ['3.7', '3.8', '3.9', '3.10']
+        os: [macos-10.15]
+        python-version: ['3.6', '3.7', '3.8', '3.9', '3.10']
+# TODO: normally, one could include macos-11 and the OS list above but the produced wheels receive an error upon installation:
+# ERROR: trexio-1.1.0-cp39-cp39-macosx_11_0_x86_64.whl is not a supported wheel on this platform.
+# This happens even with the MACOSX_DEPLOYMENT_TARGET trick. Perhaps it can be solved by configuring the build system
+# to produce the wheels for MacOS-11 from the very beginning
         #exclude:
-        #  - os: macos-10.15
-        #    python-version: '3.8'
+        #  - os: macos-11
     env:
         H5_LDFLAGS: '-L/usr/local/Cellar/hdf5/1.12.1/lib'
         H5_CFLAGS: '-I/usr/local/Cellar/hdf5/1.12.1/include'
@@ -106,36 +125,58 @@ jobs:
       - name: Display Python version
         run: python --version
 
-      - name: Install build dependencies
-        run: |
-          brew install hdf5@1.12
-          python -m pip install -U setuptools build delocate numpy
+      - name: Install HDF5
+        run: brew install hdf5@1.12
 
-      # this step is needed to produce wheels with the correct platform tag
-      - name: Set MACOSX_DEPLOYMENT_TARGET environment variable
-        if: ${{ matrix.os == 'macos-11' }}
-        # it is not possible to set ENV variables conditionally, so we improvise below
-        run: echo "MACOSX_DEPLOYMENT_TARGET=11.0" >> $GITHUB_ENV
+      # This step is needed to produce wheels with the correct platform tag for MacOS-11 (Big Sur)
+      #- name: Set MACOSX_DEPLOYMENT_TARGET environment variable
+      #  if: ${{ matrix.os == 'macos-11' }}
+      #  run: echo "MACOSX_DEPLOYMENT_TARGET=11.0" >> $GITHUB_ENV
+
+      - name: Compute the PYTREXIO_VERSION environment variable
+        run: echo "PYTREXIO_VERSION=$(grep __version__ python/pytrexio/_version.py | cut -d\  -f3 | tr -d '"')" >> $GITHUB_ENV
+
+      - name: Print the PYTREXIO_VERSION
+        run: echo ${{ env.PYTREXIO_VERSION }}
 
       - name: Download the Python API distribution tarball
-        uses: actions/download-artifact@v2
+        uses: dawidd6/action-download-artifact@v2
         with:
+          workflow: actions.yml
+          workflow_conclusion: success
           name: pytrexio-source
           path: python
-
+
+      - name: Extract the Python distribution
+        run: gzip -cd trexio-${{ env.PYTREXIO_VERSION }}.tar.gz | tar xvf -
+        working-directory: python
+
+      - name: Install Python dependencies
+        run: pip install --upgrade pip setuptools build delocate
+
       - name: Build wheel for a given version of CPython
         run: |
-          source tools/set_NUMPY_INCLUDEDIR.sh
-          python -m build --wheel --outdir ./
+          mkdir wheelhouse/
+          cd trexio-${{ env.PYTREXIO_VERSION }}/
+          python -m build --wheel --outdir=./
           delocate-wheel trexio-*.whl
+          mv trexio-*.whl ../wheelhouse/
         working-directory: python
-
+
+      # Some issues with Python 3.10 wheels on MacOS-11
+      - name: Install the wheel
+        run: python -m pip install wheelhouse/trexio-*.whl
+        working-directory: python
+
+      - name: Test the wheel
+        run: python test_api.py
+        working-directory: python/test
+
       - name: Upload produced wheels as artifacts
         uses: actions/upload-artifact@v2
         with:
           name: pytrexio-${{ matrix.os }}
-          path: ./*.whl
-        working-directory: python
+          path: ./python/wheelhouse/*.whl
 
 
   publish_wheels:
@@ -155,34 +196,40 @@ jobs:
       - name: Install build dependencies
         run: python -m pip install -U setuptools twine
 
-      - name: Download the build artifacts
+      - name: Download the build artifacts (wheels) of this workflow
         uses: actions/download-artifact@v2
         with:
           path: dist
-     # if name is not specified - all artifacts will be downloaded
-     #    name: pytrexio-manylinux
 
-      # the artifacts have to be in dist/ directory so that
+      - name: Download the Python API distribution tarball
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: actions.yml
+          workflow_conclusion: success
+          name: pytrexio-source
+          path: dist
+
+      # The artifacts have to be in dist/ directory so that
       # pypa/gh-action-pypi-publish action can discover them
       - name: Display and rearrange the downloaded artifacts
         run: |
           ls -R
           mv pytrexio-manylinux-*/trexio-*.whl ./
           mv pytrexio-macos-*/trexio-*.whl ./
-          mv pytrexio-source/trexio-*.tar.gz ./
-          rm -rf -- pytrexio-manylinux/ pytrexio-macos/ pytrexio-source/
+          rm -rf -- pytrexio-manylinux-*/ pytrexio-macos-*/
           ls -sh -w 1
         working-directory: dist
 
       #- name: Publish distribution 📦 to Test PyPI
       #  uses: pypa/gh-action-pypi-publish@master
-      #  with:
+      # with:
       #    password: ${{ secrets.TEST_PYPI_API_TOKEN }}
       #    repository_url: https://test.pypi.org/legacy/
           #verbose: true
-
-      #- name: Publish distribution 📦 to PyPI
-      #  if: startsWith(github.ref, 'refs/tags')
-      #  uses: pypa/gh-action-pypi-publish@master
-      #  with:
-      #    password: ${{ secrets.PYPI_API_TOKEN }}
+
+      # Only upload to PyPI if the commit was tagged !
+      - name: Publish distribution 📦 to PyPI
+        if: startsWith(github.ref, 'refs/tags')
+        uses: pypa/gh-action-pypi-publish@master
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}

Makefile.am

@@ -223,7 +223,7 @@ python-install: $(pytrexio_py) $(setup_py) $(setup_cfg)
 
 python-sdist: $(pytrexio_py) $(setup_py) $(setup_cfg)
 	cd python && \
-		python setup.py sdist
+		python3 -m build --sdist
 
 $(pytrexio_py): $(pytrexio_c)
 	cd tools && ./prepare_python.sh

python/README.md

@@ -22,32 +22,31 @@ In short, you can run the following command:
 
 However, it is good practice to first check for updates of the build-system packages. This can be achieved by running
 
-`python3 -m pip install --upgrade pip setuptools wheel`
+`python -m pip install --upgrade pip setuptools build wheel`
 
 **Note: we highly recommend to use virtual environments to avoid compatibility issues and to improve reproducibility.** 
 For more details, see the corresponding part of the [Python documentation](https://docs.python.org/3/library/venv.html#creating-virtual-environments).
 
 
 ## Additional requirements (for installation from source)
 
-- C compiler 	(gcc/icc)
+- C compiler 	(gcc/icc/clang)
 - HDF5 library	(>= 1.8)
 - pkgconfig     (Python package)
-
+- build         (Python package)
 
 ## Installation from source
 
 1. Download the `trexio-<version>.tar.gz` file with the latest Python API
 2. `gzip -cd trexio-<version>.tar.gz | tar xvf -`
 3. `cd trexio-<version>`
-4. `pip3 install -r requirements.txt` (this installs all required python dependencies)
+4. `pip install -r requirements.txt` (this installs all required python dependencies)
 5. Export custom environment variables needed for the installation following the procedure below and replacing `/path/to/hdf5/` with your paths. 
-Steps (i) and (ii) can be skipped if HDF5 is properly configured for `pkg-config` (i.e. if executing `pkg-config --libs hdf5` returns a list of options).
+The following two steps can be skipped if HDF5 is properly configured for `pkg-config` (i.e. if executing `pkg-config --libs hdf5` returns a list of options).
    1. `export H5_CFLAGS=-I/path/to/hdf5/include`
    2. `export H5_LDFLAGS=-L/path/to/hdf5/lib`
-   3. `source tools/set_NUMPY_INCLUDEDIR.sh`
-6. `pip3 install .` (this installs `trexio` in your environment)
-7. `cd test && python3 test_api.py` (this executes several tests that verify the installation)
+6. `pip install .` (this installs `trexio` in your environment)
+7. `cd test && python test_api.py` (this executes several tests that verify the installation)
 
 You are ready to go!
 

python/build_manylinux_wheels.sh

@@ -71,22 +71,10 @@ function build_wheel_for_py()
    # upgrade pip, otherwise it complains that manylinux wheel is "...not supported wheel on this platform"
    pip install --upgrade pip
    # install dependencies needed to build manylinux wheel
-   pip install --upgrade setuptools wheel auditwheel
-   if [ ${PYVERSION} -eq 36 ] || [ ${PYVERSION} -eq 37 ]; then
-       pip install numpy==1.17.3
-   elif [ ${PYVERSION} -eq 38 ]; then
-       pip install numpy==1.18.3
-   elif [ ${PYVERSION} -eq 39 ]; then
-       pip install numpy==1.19.3
-   else
-       pip install numpy==1.21.4
-   fi
-
-   # set an environment variable needed to locate numpy header files
-   source tools/set_NUMPY_INCLUDEDIR.sh
+   pip install --upgrade setuptools build
 
    # produce conventional (non-manylinux) wheel
-   python3 setup.py bdist_wheel
+   python3 -m build --wheel --outdir dist/
 
    # use auditwheel from PyPA to repair all wheels and make them manylinux-compatible
    auditwheel repair dist/trexio-${TR_VERSION}-${CPYTHON}-*.whl

python/install_pytrexio.sh

@@ -26,33 +26,42 @@ else
 fi
 
 # Install/upgrade packages required for the installation
-python3 -m pip install --upgrade setuptools wheel pip
+python3 -m pip install --upgrade setuptools build pip
 python3 -m pip install -r requirements.txt
 
 # export NUMPY_INCLUDEDIR environment variable needed for the proper setup
-source tools/set_NUMPY_INCLUDEDIR.sh
-
-if [[ -z ${NUMPY_INCLUDEDIR} ]] ; then
-    echo "NUMPY_INCLUDEDIR is not set. Check that numpy is installed (e.g. call pip freeze)."
-    exit 1
-fi
+#source tools/set_NUMPY_INCLUDEDIR.sh
+#if [[ -z ${NUMPY_INCLUDEDIR} ]] ; then
+#    echo "NUMPY_INCLUDEDIR is not set. Check that numpy is installed (e.g. call pip freeze)."
+#    exit 1
+#fi
 
 # Create build directory and compile extension files (*.c)
 # --no-user-cfg disables custom .cfg files of the user machine, so that only setup.cfg is used
-python3 -s setup.py --no-user-cfg build 
+#python3 -s setup.py --no-user-cfg build 
 
 # Local inplace build of the .so module with SWIG-produced pytrexio_wrap.c (from the SWIG documentation)
 #python3 setup.py build_ext --inplace --swig-opts="-modern"
 
 # Create distributions: 
+
+# OLD WAY (DEPRECATED BY PYPA)
 #   1) sdist produces .tar.gz with all files necessary for manual compilation;
 #   2) bdist_whell produces .whl wheel distribution (see https://www.python.org/dev/peps/pep-0425/).
-python3 setup.py sdist bdist_wheel
+#python3 setup.py sdist bdist_wheel
+
+# NEW WAY (USING BUILD PACKAGE OF PYPA)
+python3 -m build --sdist --wheel --outdir dist/
 
 # Install pytrexio in the current environment from the aforementioned wheel
+
+# OLD WAY
 # --force-reinstall is needed here because build-system pre-installs pytrexio in the environment 
 #                   but does not install things in the corresponding site-packages directory
-python3 -m pip install dist/trexio-*.whl --force-reinstall
+#python3 -m pip install dist/trexio-*.whl --force-reinstall
+
+# NEW WAY
+python3 -m pip install dist/trexio-*.whl
 
 # Run the command below in the root directory to install the package in 'editable' (-e) mode without dependencies (--no-deps)
 #python -m pip install -e . --no-deps