adding cibuildwheel

.github/actions/spelling/allow.txt

@@ -2145,3 +2145,26 @@ stardev
 noutput
 nprobinfo
 interm
+capfd
+cibuildwheel
+dtype
+dummybaseobject
+GIL
+maxcv
+myargs
+ndarray
+newfun
+NEWPYTHON
+nfev
+nlconstrlist
+nlcs
+pybind
+pypa
+pystr
+rtol
+scikit
+ucrt
+whl
+xlist
+ARCHS
+CIBW

.github/workflows/build_python.yml

@@ -1,86 +1,39 @@
-name: CMake build
+name: Build python wheels
 
-on:
-  push:
-  pull_request:
-  schedule:
-    - cron: '0 0 * * 1'  # 0h mondays
-  workflow_dispatch:
-    inputs:
-      git-ref:
-        description: Git Ref (Optional)
-        required: false
-
-# Show the git ref in the workflow name if it is invoked manually.
-run-name: ${{ github.event_name == 'workflow_dispatch' && format('Manual run {0} ', inputs.git-ref) || '' }}
-
-
-permissions:
-  contents: read
+on: [push, pull_request]
 
 jobs:
-
-  build-python:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
-    continue-on-error: true
     strategy:
-      fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-2019]
-    # env:
-    #   BUILD_PYTHON_CI: 1  # Or maybe just a cmake option BUILD_PYTHON? Defaulted to false? And maybe we can set shared libs to false for this build?
+        os: [ubuntu-20.04, windows-2019, macos-11]
 
     steps:
-
-      - name: Clone Repository (Latest)
-        uses: actions/checkout@v4
-        if: github.event.inputs.git-ref == ''
-        with:
-          ssh-key: ${{ secrets.SSH_PRIVATE_KEY_ACT }} # This forces checkout to use SSH, not HTTPS
-          submodules: recursive
-      - name: Clone Repository (Custom Ref)
-        uses: actions/checkout@v4
-        if: github.event.inputs.git-ref != ''
+      - uses: actions/checkout@v4
         with:
-          ref: ${{ github.event.inputs.git-ref }}
-          ssh-key: ${{ secrets.SSH_PRIVATE_KEY_ACT }} # This forces checkout to use SSH, not HTTPS
-          submodules: recursive
+          fetch-depth: 0  # Get tags for use with git describe
 
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.11'
-
-      - run: pip install numpy pytest
-      - run: python -c "import sys; print('PYTHON_EXECUTABLE=' + sys.executable)" >> $GITHUB_ENV
-        shell: bash
-
-      - name: Install Ninja / Ubuntu
-        if: ${{ matrix.os == 'ubuntu-latest' }}
-        run: sudo apt update && sudo apt install ninja-build
-      - name: Install Ninja / MacOS
-        if: ${{ matrix.os == 'macos-latest' }}
-        run: brew install ninja
-      - name: Install Ninja / Windows
-        if: ${{ matrix.os == 'windows-2022' }}
-        run: choco install ninja
+      - name: Checkout pybind11 submodule
+        run: git submodule update --init python/pybind11
 
-      - name: Win_amd64 - install rtools
-        if: ${{ matrix.os == 'windows-2022' }}
-        run: |
-          # mingw-w64
-          choco install rtools -y --no-progress --force --version=4.0.0.20220206
-          echo "c:\rtools40\ucrt64\bin;" >> $env:GITHUB_PATH
-
-# TODO: I need to move the mingw folder so that it doesn't pick up the wrong gcc, yes?
+      - uses: fortran-lang/setup-fortran@main
+        id: setup-fortran
+        with:
+          compiler: gcc
+          version: 8
 
-      - name: Build
-        run: |
-          cmake --version
-          cmake -G Ninja -DPython_EXECUTABLE=$PYTHON_EXECUTABLE -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX=. -LAH .
-          cmake --build . --target prima_pybind
-          cmake --build . --target tests
-          ctest --output-on-failure -V -R python-tests
-        shell: bash
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v2.16.2
 
-# This is a work in progress, more to come.
+      - uses: actions/upload-artifact@v3
+        with:
+          path: ./wheelhouse/*.whl
+        env:
+          # On windows we get a complaint from CMake:
+          # "CMake Error at python/pybind11/tools/FindPythonLibsNew.cmake:191 (message):
+          #   Python config failure: Python is 32-bit, chosen compiler is 64-bit"
+          # I do not see a way to install a 32-bit compiler with the setup-fortran action,
+          # so we will just build 64-bit wheels on windows.
+          CIBW_ARCHS_WINDOWS: "AMD64"

.gitignore

@@ -109,6 +109,7 @@ parts/
 sdist/
 var/
 wheels/
+wheelhouse
 share/python-wheels/
 *.egg-info/
 .installed.cfg

CMakeLists.txt

@@ -63,6 +63,14 @@ if (PRIMA_ENABLE_C)
 endif ()
 
 # Get the version number
+file(STRINGS .git-archival.txt GIT_ARCHIVAL_CONTENTS)
+message(STATUS "GIT_ARCHIVAL_CONTENTS: ${GIT_ARCHIVAL_CONTENTS}")
+if(EXISTS _version.txt)
+  file(STRINGS _version.txt VERSION_CONTENTS)
+  message(STATUS "VERSION_CONTENTS: ${VERSION_CONTENTS}")
+else()
+  message(STATUS "No _version.txt found")
+endif()
 find_package(Git)
 set(IS_REPO FALSE)
 if(GIT_EXECUTABLE)

c/CMakeLists.txt

@@ -19,7 +19,9 @@ if (NOT BUILD_SHARED_LIBS)
   # The line below caused issues when compiling prima_pybind on Windows with MinGW. We get errors
   # about multiple definition of unwind_resume due to the inclusion of gcc_s.
   # It's unclear why this was added as initial reason given in the issue (108) seems to work fine for me
-  # target_link_libraries (primac INTERFACE ${CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES})
+  if (NOT WIN32)
+    target_link_libraries (primac INTERFACE ${CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES})
+  endif()
 endif ()
 
 # Export symbols on Windows. See more comments in fortran/CMakeLists.txt.

pyproject.toml

@@ -1,17 +1,33 @@
 [build-system]
-requires = ["scikit-build-core", "numpy"]
+# scikit-build-core claims there's no need to explicitly specify Ninja,
+# as it will "automatically be downloaded if needed", but I don't know
+# how it determines "if needed", all I know is that we need it, particularly
+# for Windows.
+requires = ["scikit-build-core", "numpy", "ninja"]
 build-backend = "scikit_build_core.build"
 
 [project]
 name = "prima"
 dependencies = ["numpy"]
 dynamic = ["version"]
+requires-python = ">= 3.7"  # Driving factor is availavility of scikit-build-core
 
 [tool.scikit-build]
-cmake.targets = ["_prima"]
+cmake.args = ["-G Ninja"]
+cmake.verbose = true
+logging.level = "INFO"
 metadata.version.provider = "scikit_build_core.metadata.setuptools_scm"
 sdist.include = [".git-archival.txt"]
 install.components = ["Prima_Python_C_Extension"]
 
 [tool.setuptools_scm]  # Section required
-version_file = "_version.txt"
+write_to = "_version.txt"
+
+[tool.cibuildwheel]
+build-verbosity = 3
+test-command = "pytest -s {project}/python/tests"
+test-requires = ["pytest", "scipy", "pdfo"]
+
+
+[tool.cibuildwheel.windows]
+archs = ["AMD64"]

python/CMakeLists.txt

@@ -8,18 +8,19 @@ if ((APPLE OR WIN32) AND DEFINED ENV{CI})
   set(Python_FIND_REGISTRY "NEVER")
 endif ()
 
-find_package (Python 3.6 REQUIRED COMPONENTS Interpreter Development)
+set(PYBIND11_NEWPYTHON ON)
+add_subdirectory(pybind11)
+
 
 # numpy is needed since we use pybind11::array and array_t
+find_package (Python 3.6 REQUIRED COMPONENTS Interpreter)
 execute_process(COMMAND ${Python_EXECUTABLE} -c "import numpy" RESULT_VARIABLE _IMPORT_NUMPY)
 if (NOT _IMPORT_NUMPY EQUAL 0)
   message(SEND_ERROR "numpy: NOT FOUND, not installing/testing Python bindings")
   return()
 endif ()
 
 
-set(PYBIND11_NEWPYTHON ON)
-add_subdirectory(pybind11)
 pybind11_add_module(_prima _prima.cpp)
 target_include_directories(_prima PRIVATE ${CMAKE_SOURCE_DIR}/c/include)
 target_link_libraries(_prima PRIVATE primac primaf)
@@ -40,29 +41,21 @@ install (TARGETS _prima DESTINATION prima/ COMPONENT Prima_Python_C_Extension)
 file(GLOB SUPPORTING_PY_FILES "${CMAKE_CURRENT_SOURCE_DIR}/prima/*.py")
 install (FILES ${SUPPORTING_PY_FILES} DESTINATION prima)
 
+# We test the example files in CMake but not the test files. Those are tested by the Python
+# build system (specifically cibuildwheel).
 macro (prima_add_py_test name)
-  add_test (NAME example_${name}_python COMMAND ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/examples/${name}_example.py)
-  # set_tests_properties (example_${name}_python PROPERTIES ENVIRONMENT "PYTHONPATH=${CMAKE_INSTALL_PREFIX}/${PYTHON_SITE_PACKAGES}")
-  if (WIN32)
-    file(TO_NATIVE_PATH "${PROJECT_BINARY_DIR}/bin" _BIN_PATH)
-    set_property(TEST example_${name}_python APPEND PROPERTY ENVIRONMENT "PATH=${_BIN_PATH}\\;$ENV{PATH}")
-  else ()
-    set_property(TEST example_${name}_python APPEND PROPERTY ENVIRONMENT "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/c:$ENV{LD_LIBRARY_PATH};DYLD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/c:$ENV{DYLD_LIBRARY_PATH}")
-  endif ()
+  add_test (NAME ${name}_example_python COMMAND ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/examples/${name}_example.py)
+  set_tests_properties (${name}_example_python PROPERTIES ENVIRONMENT "PYTHONPATH=${CMAKE_INSTALL_PREFIX}")
+  # if (WIN32)
+  #   file(TO_NATIVE_PATH "${PROJECT_BINARY_DIR}/bin" _BIN_PATH)
+  #   set_property(TEST ${name}_example_python APPEND PROPERTY ENVIRONMENT "PATH=${_BIN_PATH}\\;$ENV{PATH}")
+  # else ()
+  #   set_property(TEST ${name}_example_python APPEND PROPERTY ENVIRONMENT "LD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/c:$ENV{LD_LIBRARY_PATH};DYLD_LIBRARY_PATH=${PROJECT_BINARY_DIR}/c:$ENV{DYLD_LIBRARY_PATH}")
+  # endif ()
 endmacro ()
 
 prima_add_py_test (cobyla)
 prima_add_py_test (bobyqa)
 prima_add_py_test (newuoa)
 prima_add_py_test (uobyqa)
 prima_add_py_test (lincoa)
-
-add_test (NAME python-tests
-  # For some reason we have to set capture=no so that the tests
-  # will capture output (for those tests that check that warnings were emitted)
-  COMMAND ${Python_EXECUTABLE} -m pytest --capture=no
-  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests
-)
-#set_tests_properties(python-tests
-    #PROPERTIES ENVIRONMENT "PYTHONPATH=${CMAKE_INSTALL_PREFIX}/${PYTHON_SITE_PACKAGES}")
-

python/examples/newuoa_example.py

@@ -10,10 +10,10 @@ def callback(x, f, nf, tr, *args):
 # Test default arguments
 res = minimize(fun, x0, method='NEWUOA')
 print(res)  # test repr
-assert abs(res.x[0] - 3.0) < 2e-1 and abs(res.x[1] - 2.0) < 2e-1
+assert fun.result_point_and_value_are_optimal(res)
 
 # Test callback and options
 options = {'rhobeg': 0.1}
 res = minimize(fun, x0, method='NEWUOA', callback=callback, options=options)
 print(res.message)
-assert abs(res.x[0] - 3.0) < 2e-1 and abs(res.x[1] - 2.0) < 2e-1
+assert not fun.result_point_and_value_are_optimal(res)

python/examples/uobyqa_example.py

@@ -7,4 +7,4 @@
 
 res = minimize(fun, x0, method='UOBYQA')
 print(res)
-assert abs(res.x[0] - 3.0) < 2e-1 and abs(res.x[1] - 2.0) < 2e-1
+assert fun.result_point_and_value_are_optimal(res)

python/prima/__init__.py

@@ -101,8 +101,8 @@ def minimize(fun, x0, args=(), method=None, bounds=None, constraints=None, callb
         A_ineq.append(linear_constraint.A[i])
         b_ineq.append(linear_constraint.ub[i])
         # Flip the lb to to take format preferred by PRIMA
-        A_ineq.append(-linear_constraint.A[i])
-        b_ineq.append(-linear_constraint.lb[i])
+        A_ineq.append( - linear_constraint.A[i])
+        b_ineq.append( - linear_constraint.lb[i])
     # Convert to numpy arrays, or set to None if empty
     A_eq = np.array(A_eq, dtype=np.float64) if len(A_eq) > 0 else None
     b_eq = np.array(b_eq, dtype=np.float64) if len(b_eq) > 0 else None

python/prima/_nonlinear_constraints.py

@@ -32,7 +32,7 @@ def process_single_nl_constraint(x0, nlc, options):
 
     Since we presume the call to the constraint function is expensive, we want to save the results so that
     COBYLA might make use of them in its initial iteration, and so we save them in the variable nlconstr0.
-    However COBLYA needs either both nlconstr0 and f0, or neither, and so we need to call the objective function
+    However COBYLA needs either both nlconstr0 and f0, or neither, and so we need to call the objective function
     as well so that we can save the result in f0.
 
     In order to avoid having this function return either 1 or 3 values, we take the options as an input and overwrite

python/tests/README.md

@@ -20,7 +20,7 @@ Things to test
 - [x] providing x0 as either a Python list, a numpy array, or a scalar - test_x0_list_array_scalar.py
 - [x] test anonymous lambda as: objective function, constraint function, callback - test_anonymous_lambda.py
 - [x] test calling with an improper method throws an exception and can still exit cleanly (relates to the test regarding anonymous lambda functions) - test_anonymous_lambda.py
-- [ ] cobyla with: number of nonlninear constraints provided, not provided, and with a list of nonlinear constraints only some of which provide the number
+- [ ] cobyla with: number of nonlinear constraints provided, not provided, and with a list of nonlinear constraints only some of which provide the number
 - [ ] test each method (while most will be tested by the above, I'd like to explicitly make sure each method is triggered)
 
 

python/tests/test_anonymous_lambda.py

@@ -16,7 +16,7 @@ def test_anonymous_lambda():
 
 def test_anonymous_lambda_unclean_exit():
     '''
-    Another potential issue with memory manangement is when the minimize function throws an exception instead of terminating
+    Another potential issue with memory management is when the minimize function throws an exception instead of terminating
     normally. This can be triggered by providing an invalid method. Another option might be to raise an exception within any
     of the anonymous lambdas.
     '''

python/tests/test_callback.py

@@ -9,7 +9,7 @@ def test_callback():
     def callback(x, f, nf, tr, cstrv, nlconstr):
         nonlocal callback_called
         callback_called = True
-        print(f"best points so far: {x=} {f=} {cstrv=} {nlconstr=} {nf=} {tr=}")
+        print(f"best points so far: x={x} f={f} cstrv={cstrv} nlconstr={nlconstr} nf={nf} tr={tr}")
     res = minimize(fun, x0, method="COBYLA", constraints=nlc, callback=callback)
     assert callback_called
     assert fun.result_point_and_value_are_optimal(res)

python/tests/test_cobyla.py

@@ -19,31 +19,36 @@
 
 def test_constraint_function_returns_numpy_array():
     nlc = NLC(lambda x: np.array([x[0], x[1]]), lb=[-np.inf]*2, ub=[10]*2)
-    res = minimize(fun, x0:=[0, 0], method='COBYLA', constraints=nlc)
+    x0 = [0, 0]
+    res = minimize(fun, x0, method='COBYLA', constraints=nlc)
     assert fun.result_point_and_value_are_optimal(res)
 
 
 def test_constraint_function_returns_list():
     nlc = NLC(lambda x: [x[0], x[1]], lb=[-np.inf]*2, ub=[10]*2)
-    res = minimize(fun, x0:=[0, 0], method='COBYLA', constraints=nlc)
+    x0 = [0, 0]
+    res = minimize(fun, x0, method='COBYLA', constraints=nlc)
     assert fun.result_point_and_value_are_optimal(res)
 
 
 def test_constraint_function_returns_scalar():
     nlc = NLC(lambda x: float(np.linalg.norm(x) - np.linalg.norm(fun.optimal_x)), lb=[-np.inf], ub=[0])
-    res = minimize(fun, x0:=[0, 0], method='COBYLA', constraints=nlc)
+    x0 = [0, 0]
+    res = minimize(fun, x0, method='COBYLA', constraints=nlc)
     assert fun.result_point_and_value_are_optimal(res)
 
 
 def test_single_nonlinear_constraint():
     nlc = NLC(lambda x: np.array([x[0], x[1]]), lb=[-np.inf]*2, ub=[10]*2)
-    res = minimize(fun, x0:=[0, 0], method='COBYLA', constraints=nlc)
+    x0 = [0, 0]
+    res = minimize(fun, x0, method='COBYLA', constraints=nlc)
     assert fun.result_point_and_value_are_optimal(res)
 
 
 def test_nonlinear_constraints_not_provided():
     with pytest.raises(ValueError) as e_info:
-        minimize(fun, x0:=[0, 0], method='COBYLA')
+        x0 = [0, 0]
+        minimize(fun, x0, method='COBYLA')
         assert e_info.msg == "Nonlinear constraints must be provided for COBYLA"
 
 

python/tests/test_linear_constraints.py

@@ -15,7 +15,7 @@
 # - Test with constraints that do affect the solution
 # - Test with lb constraints
 # - Test with ub constraints
-# - Test with ub and lb consraints that do not imply equality
+# - Test with ub and lb constraints that do not imply equality
 # - Test with lb and ub constraints that do imply equality
 # - Test with infeasible starting point
 # - Test with A as scalar, A as list