Many updates related to packaging:

* Update bundled pybind11 and random123 to latest versions for better compatibility with clang++ on arm64. * Add __init__.py files to data directories to silence warnings. * Use importlib.resources.as_file() instead of pkg_resources for compatibility with python-3.12. * Build wheels with numpy-2.0.x, which is backwards compatible at runtime with numpy-1.x and also compatible with numpy-2.1.x. * When building suitesparse for wheels, remove patch and use the cmake system to enable only cholmod and build everything (rather than use the archaic Makefiles directly). * Lift runtime requirements on suitesparse and numpy since we are now compatible with the latest versions. * Bump versions of vendored OpenBLAS and suitesparse. * For wheels on macos, build our own openblas rather than use libscipy_openblas, which has strange symbol name mangling that does not seem to work with clang++. * In the unit test workflow, add python-3.12 tests and also run tests on macos arm64. * In the wheel test and deploy frameworks, add python-3.12 and macos arm64 to the build matrix.
hpc4cmb · Oct 22, 2024 · bbe6a7b · bbe6a7b
1 parent 4b2c5b8
commit bbe6a7b
Show file tree

Hide file tree

Showing 81 changed files with 3,904 additions and 1,574 deletions.
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -17,56 +17,71 @@ env:
 
 jobs:
   build_wheels:
-    name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}-${{ matrix.manylinux_image }}
+    name: Build wheel for cp${{ matrix.python }}-${{ matrix.builder }}_${{ matrix.arch }}
     runs-on: ${{ matrix.os }}
     strategy:
       # Ensure that a wheel builder finishes even if another fails.  Useful for
       # debugging multiple problems in parallel.
       fail-fast: false
       matrix:
         include:
-          # Linux 64 bit manylinux2014
+          # Linux 64 bit manylinux
           - os: ubuntu-latest
+            arch: x86_64
             python: 39
-            platform_id: manylinux_x86_64
-            manylinux_image: manylinux2014
+            builder: manylinux
           - os: ubuntu-latest
+            arch: x86_64
             python: 310
-            platform_id: manylinux_x86_64
-            manylinux_image: manylinux2014
+            builder: manylinux
           - os: ubuntu-latest
+            arch: x86_64
             python: 311
-            platform_id: manylinux_x86_64
-            manylinux_image: manylinux2014
+            builder: manylinux
+          - os: ubuntu-latest
+            arch: x86_64
+            python: 312
+            builder: manylinux
 
-          # MacOS x86_64.
-          - os: macos-12
-            python: 39
-            platform_id: macosx_x86_64
-          - os: macos-12
+          # MacOS x86_64.  The macos-13 runner is the last
+          # Intel-based runner version.  At some point we'll
+          # need to switch to macos-latest and test cross compiling.
+          - os: macos-13
+            arch: x86_64
             python: 310
-            platform_id: macosx_x86_64
-          - os: macos-12
+            builder: macosx
+            deploy: 13.0
+          - os: macos-13
+            arch: x86_64
             python: 311
-            platform_id: macosx_x86_64
+            builder: macosx
+            deploy: 13.0
+          - os: macos-13
+            arch: x86_64
+            python: 312
+            builder: macosx
+            deploy: 13.0
 
-          # MacOS arm64 - Re-enable after github CI supports M1 runners:
-          #   https://github.com/actions/virtual-environments/issues/2187
-          #
-          # - os: macos-latest
-          #   python: 39
-          #   platform_id: macosx_arm64
-          # - os: macos-latest
-          #   python: 310
-          #   platform_id: macosx_arm64
-          # - os: macos-latest
-          #   python: 311
-          #   platform_id: macosx_arm64
+          # MacOS arm64
+          - os: macos-latest
+            arch: arm64
+            python: 310
+            builder: macosx
+            deploy: 14.0
+          - os: macos-latest
+            arch: arm64
+            python: 311
+            builder: macosx
+            deploy: 14.0
+          - os: macos-latest
+            arch: arm64
+            python: 312
+            builder: macosx
+            deploy: 14.0
     env:
-      CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }}
-      CIBW_ARCHS_MACOS: "x86_64 arm64"
-      CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }}
-      CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }}
+      CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.builder }}_${{ matrix.arch }}
+      CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.builder }}2014
+      CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.builder }}2014
       CIBW_BUILD_VERBOSITY: 3
       CIBW_ENVIRONMENT_LINUX: >
         TOAST_BUILD_CMAKE_VERBOSE_MAKEFILE=ON
@@ -78,10 +93,11 @@ jobs:
         TOAST_BUILD_SUITESPARSE_INCLUDE_DIR_HINTS=/usr/local/include
         TOAST_BUILD_SUITESPARSE_LIBRARY_DIR_HINTS=/usr/local/lib
       CIBW_ENVIRONMENT_MACOS: >
+        MACOSX_DEPLOYMENT_TARGET=${{ matrix.deploy }}
         TOAST_BUILD_CMAKE_C_COMPILER=clang
         TOAST_BUILD_CMAKE_CXX_COMPILER=clang++
-        TOAST_BUILD_CMAKE_C_FLAGS='-O3 -g -fPIC'
-        TOAST_BUILD_CMAKE_CXX_FLAGS='-O3 -g -fPIC -std=c++11 -stdlib=libc++'
+        TOAST_BUILD_CMAKE_C_FLAGS='-O3 -fPIC'
+        TOAST_BUILD_CMAKE_CXX_FLAGS='-O3 -fPIC -std=c++11 -stdlib=libc++'
         TOAST_BUILD_CMAKE_VERBOSE_MAKEFILE=ON
         TOAST_BUILD_DISABLE_OPENMP=1
         TOAST_BUILD_BLAS_LIBRARIES='/usr/local/lib/libopenblas.dylib'
@@ -91,10 +107,9 @@ jobs:
         TOAST_BUILD_SUITESPARSE_INCLUDE_DIR_HINTS=/usr/local/include
         TOAST_BUILD_SUITESPARSE_LIBRARY_DIR_HINTS=/usr/local/lib
       CIBW_BEFORE_BUILD_LINUX: ./packaging/wheels/install_deps_linux.sh
-      CIBW_BEFORE_BUILD_MACOS: ./packaging/wheels/install_deps_osx.sh ${{ matrix.platform_id }}
+      CIBW_BEFORE_BUILD_MACOS: ./packaging/wheels/install_deps_osx.sh
       CIBW_BEFORE_TEST: export OMP_NUM_THREADS=1
       CIBW_TEST_COMMAND: source {project}/packaging/wheels/cibw_run_tests.sh
-      CIBW_TEST_SKIP: "*-macosx_arm64 *-macosx_universal2:arm64"
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -106,12 +121,12 @@ jobs:
 
       - name: Install cibuildwheel
         run: |
-          python -m pip install twine cibuildwheel==2.18.0
+          python -m pip install twine cibuildwheel==2.20.0
 
       - name: Build wheel
         run: |
           python -m cibuildwheel --output-dir wheelhouse
 
       - name: Upload to PyPI
         run: |
-          python -m twine upload wheelhouse/toast*.whl
+          python -m twine upload wheelhouse/toast*cp${{ matrix.python }}-${{ matrix.builder }}*${{ matrix.arch }}*.whl
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -40,20 +40,24 @@ jobs:
             arch: Linux-x86_64
             ompdisable: 0
           - os: ubuntu-latest
-            python: "3.11"
+            python: "3.12"
             arch: Linux-x86_64
             ompdisable: 0
           - os: macos-latest
-            python: "3.9"
+            python: "3.10"
             arch: MacOSX-x86_64
             ompdisable: 1
           - os: macos-latest
-            python: "3.10"
+            python: "3.12"
             arch: MacOSX-x86_64
             ompdisable: 1
           - os: macos-latest
-            python: "3.11"
-            arch: MacOSX-x86_64
+            python: "3.10"
+            arch: MacOSX-arm64
+            ompdisable: 1
+          - os: macos-latest
+            python: "3.12"
+            arch: MacOSX-arm64
             ompdisable: 1
     steps:
       - name: Checkout

diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
@@ -19,56 +19,71 @@ concurrency:
 
 jobs:
   build_wheels:
-    name: Build wheel for cp${{ matrix.python }}-${{ matrix.platform_id }}-${{ matrix.manylinux_image }}
+    name: Build wheel for cp${{ matrix.python }}-${{ matrix.builder }}_${{ matrix.arch }}
     runs-on: ${{ matrix.os }}
     strategy:
       # Ensure that a wheel builder finishes even if another fails.  Useful for
       # debugging multiple problems in parallel.
       fail-fast: false
       matrix:
         include:
-          # Linux 64 bit manylinux2014
+          # Linux 64 bit manylinux
           - os: ubuntu-latest
+            arch: x86_64
             python: 39
-            platform_id: manylinux_x86_64
-            manylinux_image: manylinux2014
+            builder: manylinux
           - os: ubuntu-latest
+            arch: x86_64
             python: 310
-            platform_id: manylinux_x86_64
-            manylinux_image: manylinux2014
+            builder: manylinux
           - os: ubuntu-latest
+            arch: x86_64
             python: 311
-            platform_id: manylinux_x86_64
-            manylinux_image: manylinux2014
+            builder: manylinux
+          - os: ubuntu-latest
+            arch: x86_64
+            python: 312
+            builder: manylinux
 
-          # MacOS x86_64.
-          - os: macos-12
-            python: 39
-            platform_id: macosx_x86_64
-          - os: macos-12
+          # MacOS x86_64.  The macos-13 runner is the last
+          # Intel-based runner version.  At some point we'll
+          # need to switch to macos-latest and test cross compiling.
+          - os: macos-13
+            arch: x86_64
             python: 310
-            platform_id: macosx_x86_64
-          - os: macos-12
+            builder: macosx
+            deploy: 13.0
+          - os: macos-13
+            arch: x86_64
             python: 311
-            platform_id: macosx_x86_64
+            builder: macosx
+            deploy: 13.0
+          - os: macos-13
+            arch: x86_64
+            python: 312
+            builder: macosx
+            deploy: 13.0
 
-          # MacOS arm64 - Re-enable after github CI supports M1 runners:
-          #   https://github.com/actions/virtual-environments/issues/2187
-          #
-          # - os: macos-latest
-          #   python: 39
-          #   platform_id: macosx_arm64
-          # - os: macos-latest
-          #   python: 310
-          #   platform_id: macosx_arm64
-          # - os: macos-latest
-          #   python: 311
-          #   platform_id: macosx_arm64
+          # MacOS arm64
+          - os: macos-latest
+            arch: arm64
+            python: 310
+            builder: macosx
+            deploy: 14.0
+          - os: macos-latest
+            arch: arm64
+            python: 311
+            builder: macosx
+            deploy: 14.0
+          - os: macos-latest
+            arch: arm64
+            python: 312
+            builder: macosx
+            deploy: 14.0
     env:
-      CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.platform_id }}
-      CIBW_ARCHS_MACOS: "x86_64 arm64"
-      CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.manylinux_image }}
-      CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.manylinux_image }}
+      CIBW_BUILD: cp${{ matrix.python }}-${{ matrix.builder }}_${{ matrix.arch }}
+      CIBW_MANYLINUX_X86_64_IMAGE: ${{ matrix.builder }}2014
+      CIBW_MANYLINUX_I686_IMAGE: ${{ matrix.builder }}2014
       CIBW_BUILD_VERBOSITY: 3
       CIBW_ENVIRONMENT_LINUX: >
         TOAST_BUILD_CMAKE_VERBOSE_MAKEFILE=ON
@@ -80,10 +95,11 @@ jobs:
         TOAST_BUILD_SUITESPARSE_INCLUDE_DIR_HINTS=/usr/local/include
         TOAST_BUILD_SUITESPARSE_LIBRARY_DIR_HINTS=/usr/local/lib
       CIBW_ENVIRONMENT_MACOS: >
+        MACOSX_DEPLOYMENT_TARGET=${{ matrix.deploy }}
         TOAST_BUILD_CMAKE_C_COMPILER=clang
         TOAST_BUILD_CMAKE_CXX_COMPILER=clang++
-        TOAST_BUILD_CMAKE_C_FLAGS='-O3 -g -fPIC'
-        TOAST_BUILD_CMAKE_CXX_FLAGS='-O3 -g -fPIC -std=c++11 -stdlib=libc++'
+        TOAST_BUILD_CMAKE_C_FLAGS='-O3 -fPIC'
+        TOAST_BUILD_CMAKE_CXX_FLAGS='-O3 -fPIC -std=c++11 -stdlib=libc++'
         TOAST_BUILD_CMAKE_VERBOSE_MAKEFILE=ON
         TOAST_BUILD_DISABLE_OPENMP=1
         TOAST_BUILD_BLAS_LIBRARIES='/usr/local/lib/libopenblas.dylib'
@@ -93,7 +109,7 @@ jobs:
         TOAST_BUILD_SUITESPARSE_INCLUDE_DIR_HINTS=/usr/local/include
         TOAST_BUILD_SUITESPARSE_LIBRARY_DIR_HINTS=/usr/local/lib
       CIBW_BEFORE_BUILD_LINUX: ./packaging/wheels/install_deps_linux.sh
-      CIBW_BEFORE_BUILD_MACOS: ./packaging/wheels/install_deps_osx.sh ${{ matrix.platform_id }}
+      CIBW_BEFORE_BUILD_MACOS: ./packaging/wheels/install_deps_osx.sh
       CIBW_BEFORE_TEST: export OMP_NUM_THREADS=2
       CIBW_TEST_COMMAND_LINUX: >
         tst=$(dirname $(python -c 'import toast; print(toast.__file__)')) &&
@@ -105,7 +121,6 @@ jobs:
         echo "Toast test install at ${tst}" &&
         otool -L ${tst}/_libtoast.cpython* &&
         source {project}/packaging/wheels/cibw_run_tests.sh
-      CIBW_TEST_SKIP: "*-macosx_arm64 *-macosx_universal2:arm64"
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -117,13 +132,13 @@ jobs:
 
       - name: Install cibuildwheel
         run: |
-          python -m pip install cibuildwheel==2.18.0
+          python -m pip install cibuildwheel==2.20.0
 
       - name: Build wheel
         run: |
           python -m cibuildwheel --output-dir wheelhouse
 
       - uses: actions/upload-artifact@v4
         with:
-          name: wheels_${{ matrix.python }}-${{ matrix.platform_id }}
-          path: ./wheelhouse/toast*.whl
+          name: wheels_cp${{ matrix.python }}-${{ matrix.builder }}_${{ matrix.arch }}
+          path: ./wheelhouse/toast*cp${{ matrix.python }}-${{ matrix.builder }}*${{ matrix.arch }}*.whl
diff --git a/packaging/conda/deps.txt b/packaging/conda/deps.txt
@@ -8,8 +8,8 @@ pytest
 tomlkit
 traitlets
 h5py
-numpy<2
-scipy<1.14
+numpy
+scipy
 astropy
 ephem
 healpy

diff --git a/packaging/conda/extdeps.txt b/packaging/conda/extdeps.txt
@@ -2,5 +2,5 @@ libopenblas=*=*openmp*
 libblas=*=*openblas
 fftw
 libaatm
-suitesparse<7.3
+suitesparse
 libflac
diff --git a/packaging/deps/openblas.sh b/packaging/deps/openblas.sh
@@ -12,7 +12,7 @@
 # - MAKEJ
 # - CLEANUP (yes/no)
 
-openblas_version=0.3.27
+openblas_version=0.3.28
 openblas_dir=OpenBLAS-${openblas_version}
 openblas_pkg=${openblas_dir}.tar.gz
 
@@ -30,11 +30,16 @@ if [ "${STATIC}" = "yes" ]; then
     targ="libs netlib"
 fi
 
+omp="USE_OPENMP=1"
+if [ "x${OMPFLAGS}" = "x" ]; then
+    omp="USE_OPENMP=0"
+fi
+
 start_dir=$(pwd)
 rm -rf ${openblas_dir}
 tar xzf ${openblas_pkg} \
     && pushd ${openblas_dir} >/dev/null 2>&1 \
-    && make USE_OPENMP=1 ${shr} \
+    && make ${omp} ${shr} \
     MAKE_NB_JOBS=${MAKEJ} \
     CC="${CC}" FC="${FC}" DYNAMIC_ARCH=1 TARGET=GENERIC \
     COMMON_OPT="${CFLAGS}" FCOMMON_OPT="${FCFLAGS}" \