From a96abcb4f2b04b2a8b6b19a2aa9c6d440da7cc0c Mon Sep 17 00:00:00 2001 From: Nickolai Belakovski Date: Fri, 12 Jan 2024 12:23:17 -0800 Subject: [PATCH] adding cibuildwheel --- .github/actions/spelling/allow.txt | 21 +++++ .github/workflows/build_python.yml | 92 ++++--------------- .gitignore | 1 + CMakeLists.txt | 8 ++ pyproject.toml | 13 ++- python/CMakeLists.txt | 13 +-- python/prima/_nonlinear_constraints.py | 2 +- python/tests/README.md | 2 +- python/tests/test_anonymous_lambda.py | 2 +- python/tests/test_callback.py | 2 +- python/tests/test_cobyla.py | 15 ++- python/tests/test_linear_constraints.py | 2 +- .../test_process_nonlinear_constraints.py | 18 ++-- 13 files changed, 94 insertions(+), 97 deletions(-) diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index da63feff26..a3c497c35f 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -2145,3 +2145,24 @@ stardev noutput nprobinfo interm +capfd +cibuildwheel +dtype +dummybaseobject +GIL +maxcv +myargs +ndarray +newfun +NEWPYTHON +nfev +nlconstrlist +nlcs +pybind +pypa +pystr +rtol +scikit +ucrt +whl +xlist diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml index 5bc2ef4e90..80c8cab124 100644 --- a/.github/workflows/build_python.yml +++ b/.github/workflows/build_python.yml @@ -1,86 +1,32 @@ -name: CMake build +name: Build python wheels -on: - push: - pull_request: - schedule: - - cron: '0 0 * * 1' # 0h mondays - workflow_dispatch: - inputs: - git-ref: - description: Git Ref (Optional) - required: false - -# Show the git ref in the workflow name if it is invoked manually. -run-name: ${{ github.event_name == 'workflow_dispatch' && format('Manual run {0} ', inputs.git-ref) || '' }} - - -permissions: - contents: read +on: [push, pull_request] jobs: - - build-python: + build_wheels: + name: Build wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} - continue-on-error: true strategy: - fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-2019] - # env: - # BUILD_PYTHON_CI: 1 # Or maybe just a cmake option BUILD_PYTHON? Defaulted to false? And maybe we can set shared libs to false for this build? + os: [ubuntu-20.04, windows-2019, macos-11] steps: - - - name: Clone Repository (Latest) - uses: actions/checkout@v4 - if: github.event.inputs.git-ref == '' - with: - ssh-key: ${{ secrets.SSH_PRIVATE_KEY_ACT }} # This forces checkout to use SSH, not HTTPS - submodules: recursive - - name: Clone Repository (Custom Ref) - uses: actions/checkout@v4 - if: github.event.inputs.git-ref != '' + - uses: actions/checkout@v4 with: - ref: ${{ github.event.inputs.git-ref }} - ssh-key: ${{ secrets.SSH_PRIVATE_KEY_ACT }} # This forces checkout to use SSH, not HTTPS - submodules: recursive + fetch-depth: 0 # Get tags for use with git describe - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - run: pip install numpy pytest - - run: python -c "import sys; print('PYTHON_EXECUTABLE=' + sys.executable)" >> $GITHUB_ENV - shell: bash - - - name: Install Ninja / Ubuntu - if: ${{ matrix.os == 'ubuntu-latest' }} - run: sudo apt update && sudo apt install ninja-build - - name: Install Ninja / MacOS - if: ${{ matrix.os == 'macos-latest' }} - run: brew install ninja - - name: Install Ninja / Windows - if: ${{ matrix.os == 'windows-2022' }} - run: choco install ninja + - name: Checkout pybind11 submodule + run: git submodule update --init python/pybind11 - - name: Win_amd64 - install rtools - if: ${{ matrix.os == 'windows-2022' }} - run: | - # mingw-w64 - choco install rtools -y --no-progress --force --version=4.0.0.20220206 - echo "c:\rtools40\ucrt64\bin;" >> $env:GITHUB_PATH - -# TODO: I need to move the mingw folder so that it doesn't pick up the wrong gcc, yes? + - uses: fortran-lang/setup-fortran@main + id: setup-fortran + with: + compiler: gcc + version: 8 - - name: Build - run: | - cmake --version - cmake -G Ninja -DPython_EXECUTABLE=$PYTHON_EXECUTABLE -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX=. -LAH . - cmake --build . --target prima_pybind - cmake --build . --target tests - ctest --output-on-failure -V -R python-tests - shell: bash + - name: Build wheels + uses: pypa/cibuildwheel@v2.16.2 -# This is a work in progress, more to come. + - uses: actions/upload-artifact@v3 + with: + path: ./wheelhouse/*.whl diff --git a/.gitignore b/.gitignore index cda60f75f7..bffd203ac5 100644 --- a/.gitignore +++ b/.gitignore @@ -109,6 +109,7 @@ parts/ sdist/ var/ wheels/ +wheelhouse share/python-wheels/ *.egg-info/ .installed.cfg diff --git a/CMakeLists.txt b/CMakeLists.txt index 52c96388c1..bfe6f3a965 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,6 +63,14 @@ if (PRIMA_ENABLE_C) endif () # Get the version number +file(STRINGS .git-archival.txt GIT_ARCHIVAL_CONTENTS) +message(STATUS "GIT_ARCHIVAL_CONTENTS: ${GIT_ARCHIVAL_CONTENTS}") +if(EXISTS _version.txt) + file(STRINGS _version.txt VERSION_CONTENTS) + message(STATUS "VERSION_CONTENTS: ${VERSION_CONTENTS}") +else() + message(STATUS "No _version.txt found") +endif() find_package(Git) set(IS_REPO FALSE) if(GIT_EXECUTABLE) diff --git a/pyproject.toml b/pyproject.toml index f3e8acb570..e1cfe6d4c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,12 +6,21 @@ build-backend = "scikit_build_core.build" name = "prima" dependencies = ["numpy"] dynamic = ["version"] +requires-python = ">= 3.7" # Driving factor is availavility of scikit-build-core [tool.scikit-build] -cmake.targets = ["_prima"] +cmake.args = ["-G Ninja"] +cmake.verbose = true +logging.level = "INFO" metadata.version.provider = "scikit_build_core.metadata.setuptools_scm" sdist.include = [".git-archival.txt"] install.components = ["Prima_Python_C_Extension"] +ninja.minimum-version = ">=1.11" [tool.setuptools_scm] # Section required -version_file = "_version.txt" +write_to = "_version.txt" + +[tool.cibuildwheel] +build-verbosity = 3 +test-command = "pytest -s {project}/python/tests" +test-requires = ["pytest", "scipy", "pdfo"] \ No newline at end of file diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index b66b2c0345..1bd82b15d3 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -8,9 +8,12 @@ if ((APPLE OR WIN32) AND DEFINED ENV{CI}) set(Python_FIND_REGISTRY "NEVER") endif () -find_package (Python 3.6 REQUIRED COMPONENTS Interpreter Development) +set(PYBIND11_NEWPYTHON ON) +add_subdirectory(pybind11) + # numpy is needed since we use pybind11::array and array_t +find_package (Python 3.6 REQUIRED COMPONENTS Interpreter) execute_process(COMMAND ${Python_EXECUTABLE} -c "import numpy" RESULT_VARIABLE _IMPORT_NUMPY) if (NOT _IMPORT_NUMPY EQUAL 0) message(SEND_ERROR "numpy: NOT FOUND, not installing/testing Python bindings") @@ -18,8 +21,6 @@ if (NOT _IMPORT_NUMPY EQUAL 0) endif () -set(PYBIND11_NEWPYTHON ON) -add_subdirectory(pybind11) pybind11_add_module(_prima _prima.cpp) target_include_directories(_prima PRIVATE ${CMAKE_SOURCE_DIR}/c/include) target_link_libraries(_prima PRIVATE primac primaf) @@ -42,7 +43,7 @@ install (FILES ${SUPPORTING_PY_FILES} DESTINATION prima) macro (prima_add_py_test name) add_test (NAME example_${name}_python COMMAND ${Python_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/examples/${name}_example.py) - # set_tests_properties (example_${name}_python PROPERTIES ENVIRONMENT "PYTHONPATH=${CMAKE_INSTALL_PREFIX}/${PYTHON_SITE_PACKAGES}") + set_tests_properties (example_${name}_python PROPERTIES ENVIRONMENT "PYTHONPATH=${CMAKE_INSTALL_PREFIX}") if (WIN32) file(TO_NATIVE_PATH "${PROJECT_BINARY_DIR}/bin" _BIN_PATH) set_property(TEST example_${name}_python APPEND PROPERTY ENVIRONMENT "PATH=${_BIN_PATH}\\;$ENV{PATH}") @@ -63,6 +64,6 @@ add_test (NAME python-tests COMMAND ${Python_EXECUTABLE} -m pytest --capture=no WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tests ) -#set_tests_properties(python-tests - #PROPERTIES ENVIRONMENT "PYTHONPATH=${CMAKE_INSTALL_PREFIX}/${PYTHON_SITE_PACKAGES}") +set_tests_properties(python-tests + PROPERTIES ENVIRONMENT "PYTHONPATH=${CMAKE_INSTALL_PREFIX}") diff --git a/python/prima/_nonlinear_constraints.py b/python/prima/_nonlinear_constraints.py index d350e58762..735134c369 100644 --- a/python/prima/_nonlinear_constraints.py +++ b/python/prima/_nonlinear_constraints.py @@ -32,7 +32,7 @@ def process_single_nl_constraint(x0, nlc, options): Since we presume the call to the constraint function is expensive, we want to save the results so that COBYLA might make use of them in its initial iteration, and so we save them in the variable nlconstr0. - However COBLYA needs either both nlconstr0 and f0, or neither, and so we need to call the objective function + However COBYLA needs either both nlconstr0 and f0, or neither, and so we need to call the objective function as well so that we can save the result in f0. In order to avoid having this function return either 1 or 3 values, we take the options as an input and overwrite diff --git a/python/tests/README.md b/python/tests/README.md index 88786b6f00..22d47aec08 100644 --- a/python/tests/README.md +++ b/python/tests/README.md @@ -20,7 +20,7 @@ Things to test - [x] providing x0 as either a Python list, a numpy array, or a scalar - test_x0_list_array_scalar.py - [x] test anonymous lambda as: objective function, constraint function, callback - test_anonymous_lambda.py - [x] test calling with an improper method throws an exception and can still exit cleanly (relates to the test regarding anonymous lambda functions) - test_anonymous_lambda.py -- [ ] cobyla with: number of nonlninear constraints provided, not provided, and with a list of nonlinear constraints only some of which provide the number +- [ ] cobyla with: number of nonlinear constraints provided, not provided, and with a list of nonlinear constraints only some of which provide the number - [ ] test each method (while most will be tested by the above, I'd like to explicitly make sure each method is triggered) diff --git a/python/tests/test_anonymous_lambda.py b/python/tests/test_anonymous_lambda.py index 313fb0eda5..c634fd1641 100644 --- a/python/tests/test_anonymous_lambda.py +++ b/python/tests/test_anonymous_lambda.py @@ -16,7 +16,7 @@ def test_anonymous_lambda(): def test_anonymous_lambda_unclean_exit(): ''' - Another potential issue with memory manangement is when the minimize function throws an exception instead of terminating + Another potential issue with memory management is when the minimize function throws an exception instead of terminating normally. This can be triggered by providing an invalid method. Another option might be to raise an exception within any of the anonymous lambdas. ''' diff --git a/python/tests/test_callback.py b/python/tests/test_callback.py index f0c0cac983..807bf0e0ec 100644 --- a/python/tests/test_callback.py +++ b/python/tests/test_callback.py @@ -9,7 +9,7 @@ def test_callback(): def callback(x, f, nf, tr, cstrv, nlconstr): nonlocal callback_called callback_called = True - print(f"best points so far: {x=} {f=} {cstrv=} {nlconstr=} {nf=} {tr=}") + print(f"best points so far: x={x} f={f} cstrv={cstrv} nlconstr={nlconstr} nf={nf} tr={tr}") res = minimize(fun, x0, method="COBYLA", constraints=nlc, callback=callback) assert callback_called assert fun.result_point_and_value_are_optimal(res) diff --git a/python/tests/test_cobyla.py b/python/tests/test_cobyla.py index 01f0e78940..1168080a4e 100644 --- a/python/tests/test_cobyla.py +++ b/python/tests/test_cobyla.py @@ -19,31 +19,36 @@ def test_constraint_function_returns_numpy_array(): nlc = NLC(lambda x: np.array([x[0], x[1]]), lb=[-np.inf]*2, ub=[10]*2) - res = minimize(fun, x0:=[0, 0], method='COBYLA', constraints=nlc) + x0 = [0, 0] + res = minimize(fun, x0, method='COBYLA', constraints=nlc) assert fun.result_point_and_value_are_optimal(res) def test_constraint_function_returns_list(): nlc = NLC(lambda x: [x[0], x[1]], lb=[-np.inf]*2, ub=[10]*2) - res = minimize(fun, x0:=[0, 0], method='COBYLA', constraints=nlc) + x0 = [0, 0] + res = minimize(fun, x0, method='COBYLA', constraints=nlc) assert fun.result_point_and_value_are_optimal(res) def test_constraint_function_returns_scalar(): nlc = NLC(lambda x: float(np.linalg.norm(x) - np.linalg.norm(fun.optimal_x)), lb=[-np.inf], ub=[0]) - res = minimize(fun, x0:=[0, 0], method='COBYLA', constraints=nlc) + x0 = [0, 0] + res = minimize(fun, x0, method='COBYLA', constraints=nlc) assert fun.result_point_and_value_are_optimal(res) def test_single_nonlinear_constraint(): nlc = NLC(lambda x: np.array([x[0], x[1]]), lb=[-np.inf]*2, ub=[10]*2) - res = minimize(fun, x0:=[0, 0], method='COBYLA', constraints=nlc) + x0 = [0, 0] + res = minimize(fun, x0, method='COBYLA', constraints=nlc) assert fun.result_point_and_value_are_optimal(res) def test_nonlinear_constraints_not_provided(): with pytest.raises(ValueError) as e_info: - minimize(fun, x0:=[0, 0], method='COBYLA') + x0 = [0, 0] + minimize(fun, x0, method='COBYLA') assert e_info.msg == "Nonlinear constraints must be provided for COBYLA" diff --git a/python/tests/test_linear_constraints.py b/python/tests/test_linear_constraints.py index 283fcd64af..beba1b651b 100644 --- a/python/tests/test_linear_constraints.py +++ b/python/tests/test_linear_constraints.py @@ -15,7 +15,7 @@ # - Test with constraints that do affect the solution # - Test with lb constraints # - Test with ub constraints -# - Test with ub and lb consraints that do not imply equality +# - Test with ub and lb constraints that do not imply equality # - Test with lb and ub constraints that do imply equality # - Test with infeasible starting point # - Test with A as scalar, A as list diff --git a/python/tests/test_process_nonlinear_constraints.py b/python/tests/test_process_nonlinear_constraints.py index 5f2795c0ab..914de17a3c 100644 --- a/python/tests/test_process_nonlinear_constraints.py +++ b/python/tests/test_process_nonlinear_constraints.py @@ -11,7 +11,8 @@ def test_multiple_nl_constraints_all_provide_lb_ub_as_list(): nlc1 = NonlinearConstraint(lambda x: x, lb=[-np.inf], ub=[0]) nlc2 = NonlinearConstraint(lambda x: [x, x], lb=[-np.inf]*2, ub=[0]*2) nlcs = [nlc1, nlc2] - nlc = process_multiple_nl_constraints(x0:=0, nlcs, None) # We can provide options as None since this shouldn't trigger the code path that requires it + x0 = 0 + nlc = process_multiple_nl_constraints(x0, nlcs, None) # We can provide options as None since this shouldn't trigger the code path that requires it assert all(nlc.lb == [-np.inf, -np.inf, -np.inf]) assert all(nlc.ub == [0, 0, 0]) assert all(nlc.fun(0) == [0, 0, 0]) @@ -22,7 +23,8 @@ def test_multiple_nl_constraints_some_provide_lb_ub_as_list(): nlc2 = NonlinearConstraint(lambda x: [x, x], lb=-np.inf, ub=0) nlcs = [nlc1, nlc2] options = {} - nlc = process_multiple_nl_constraints(x0:=0, nlcs, options) + x0 = 0 + nlc = process_multiple_nl_constraints(x0, nlcs, options) assert all(nlc.lb == [-np.inf, -np.inf, -np.inf]) assert all(nlc.ub == [0, 0, 0]) assert all(nlc.fun(0) == [0, 0, 0]) @@ -34,7 +36,8 @@ def test_multiple_nl_constraints_none_provide_lb_ub_as_list(): nlc2 = NonlinearConstraint(lambda x: [x, x], lb=-np.inf, ub=0) nlcs = [nlc1, nlc2] options = {} - nlc = process_multiple_nl_constraints(x0:=0, nlcs, options) + x0 = 0 + nlc = process_multiple_nl_constraints(x0, nlcs, options) assert all(nlc.lb == [-np.inf, -np.inf, -np.inf]) assert all(nlc.ub == [0, 0, 0]) assert all(nlc.fun(0) == [0, 0, 0]) @@ -44,7 +47,8 @@ def test_multiple_nl_constraints_none_provide_lb_ub_as_list(): def test_single_nl_constraint_provides_lb_as_list(): num_constraints = 3 nlc = NonlinearConstraint(lambda x: x, lb=[-np.inf]*num_constraints, ub=0) - processed_nlc = process_single_nl_constraint(x0:=0, nlc, None) + x0 = 0 + processed_nlc = process_single_nl_constraint(x0, nlc, None) assert len(processed_nlc.lb) == num_constraints assert len(processed_nlc.ub) == num_constraints @@ -52,7 +56,8 @@ def test_single_nl_constraint_provides_lb_as_list(): def test_single_nl_constraint_provides_lb_as_scalar_with_scalar_constr_function(): nlc = NonlinearConstraint(lambda x: x, lb=-np.inf, ub=0) options = {} - processed_nlc = process_single_nl_constraint(x0:= 0, nlc, options) + x0 = 0 + processed_nlc = process_single_nl_constraint(x0, nlc, options) assert processed_nlc.lb == [-np.inf] assert processed_nlc.ub == [0] assert options['nlconstr0'] == 0 @@ -61,7 +66,8 @@ def test_single_nl_constraint_provides_lb_as_scalar_with_scalar_constr_function( def test_single_nl_constraint_provides_lb_as_scalar_with_vector_constr_function(): nlc = NonlinearConstraint(lambda x: [x, x], lb=-np.inf, ub=0) options = {} - processed_nlc = process_single_nl_constraint(x0:=2.1, nlc, options) + x0 = 2.1 + processed_nlc = process_single_nl_constraint(x0, nlc, options) assert all(processed_nlc.lb == [-np.inf, -np.inf]) assert all(processed_nlc.ub == [0, 0]) assert options['nlconstr0'] == [2.1, 2.1]