From f223ce6c779fa42b8eeeeff4cd661448d20dca71 Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Mon, 5 Sep 2022 11:52:31 +0200 Subject: [PATCH 01/36] [py] Added scikit-build. Build with: ``` pip install . ``` --- .gitignore | 4 ++++ pyproject.toml | 8 ++++++++ scripts/cmake/packaging/Pack.cmake | 4 +++- setup.py | 14 ++++++++++++++ 4 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 pyproject.toml create mode 100644 setup.py diff --git a/.gitignore b/.gitignore index 1c8d2d8653f..a023a253f49 100644 --- a/.gitignore +++ b/.gitignore @@ -38,4 +38,8 @@ CMakeUserPresets.json nohup.out /Documentation/.vale + +# Python build +/_skbuild +*.egg-info/ .ipynb_checkpoints diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000000..5c7d0579bbd --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,8 @@ +[build-system] +requires = [ + "setuptools>=42", + "scikit-build>=0.15.0", + "cmake>=3.22", + "ninja; platform_system!='Windows'", +] +build-backend = "setuptools.build_meta" diff --git a/scripts/cmake/packaging/Pack.cmake b/scripts/cmake/packaging/Pack.cmake index 8c1daaf61bc..33b91a33fc6 100644 --- a/scripts/cmake/packaging/Pack.cmake +++ b/scripts/cmake/packaging/Pack.cmake @@ -76,7 +76,9 @@ if(OGS_USE_PYTHON) install(FILES ${PYTHON_RUNTIME_LIBS} DESTINATION bin) file(COPY ${PYTHON_RUNTIME_LIBS} DESTINATION ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) else() - install(FILES ${Python_LIBRARIES} DESTINATION bin) + file(INSTALL ${Python_LIBRARIES} DESTINATION ${CMAKE_INSTALL_LIBDIR} + FOLLOW_SYMLINK_CHAIN + ) endif() endif() diff --git a/setup.py b/setup.py new file mode 100644 index 00000000000..58e18d9ad73 --- /dev/null +++ b/setup.py @@ -0,0 +1,14 @@ +from skbuild import setup +from setuptools import find_packages + +setup( + name="ogs", + version="6.4.2", + description="OpenGeoSys", + author="OpenGeoSys Community", + license="BSD-3-Clause", + packages=find_packages(where="."), + package_dir={"": "."}, + cmake_args=["-DOGS_BUILD_PROCESSES=SteadyStateDiffusion", "-DOGS_BUILD_UTILS=OFF"], + python_requires=">=3.6", +) From 0abb165583580e2e3639571b41cccd911b2f64d7 Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Mon, 5 Sep 2022 13:58:39 +0200 Subject: [PATCH 02/36] [py] Module works. ``` pip install . pytest ``` --- .gitignore | 1 + Applications/CLI/CMakeLists.txt | 47 ------------------- Applications/CLI/ogs_python_module.py | 19 -------- Applications/CMakeLists.txt | 4 ++ Applications/Python/CMakeLists.txt | 21 +++++++++ Applications/Python/OpenGeoSys/__init__.py | 1 + .../{CLI => Python}/ogs_python_module.cpp | 2 +- Tests/Python/test_cli.py | 18 +++++++ pyproject.toml | 16 +++++++ .../cmake/DependenciesExternalProject.cmake | 2 +- setup.py | 19 ++++++-- 11 files changed, 78 insertions(+), 72 deletions(-) delete mode 100755 Applications/CLI/ogs_python_module.py create mode 100644 Applications/Python/CMakeLists.txt create mode 100644 Applications/Python/OpenGeoSys/__init__.py rename Applications/{CLI => Python}/ogs_python_module.cpp (99%) create mode 100644 Tests/Python/test_cli.py diff --git a/.gitignore b/.gitignore index a023a253f49..d3e5862f91c 100644 --- a/.gitignore +++ b/.gitignore @@ -42,4 +42,5 @@ nohup.out # Python build /_skbuild *.egg-info/ +/wheelhouse .ipynb_checkpoints diff --git a/Applications/CLI/CMakeLists.txt b/Applications/CLI/CMakeLists.txt index 742e4be2a96..4d221c96d5d 100644 --- a/Applications/CLI/CMakeLists.txt +++ b/Applications/CLI/CMakeLists.txt @@ -45,31 +45,6 @@ if(OGS_USE_PYTHON) $<$:PRIVATE OGS_BUILD_SHARED_LIBS> ) - - # Create OpenGeoSys python module - # https://pybind11.readthedocs.io/en/stable/compiling.html#building-with-cmake - if(OGS_BUILD_PYTHON_MODULE) - pybind11_add_module( - OpenGeoSys MODULE ogs_python_module.cpp - CommandLineArgumentParser.cpp - ) - - # lld linker strips out PyInit_OpenGeoSys symbol. Use standard linker. - get_target_property(_link_options OpenGeoSys LINK_OPTIONS) - if(_link_options) - list(REMOVE_ITEM _link_options -fuse-ld=lld) - set_target_properties( - OpenGeoSys PROPERTIES LINK_OPTIONS "${_link_options}" - ) - endif() - - target_link_libraries( - OpenGeoSys PRIVATE ApplicationsLib BaseLib GitInfoLib tclap - pybind11::pybind11 - ) - - install(TARGETS OpenGeoSys LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) - endif() endif() ogs_add_executable(ogs ogs.cpp CommandLineArgumentParser.cpp) @@ -94,28 +69,6 @@ target_link_libraries( add_test(NAME ogs_no_args COMMAND ogs) set_tests_properties(ogs_no_args PROPERTIES WILL_FAIL TRUE LABELS "default") -if(OGS_BUILD_PYTHON_MODULE AND NOT (WIN32 AND "${CMAKE_BUILD_TYPE}" STREQUAL - "Release") -) - # TODO: does not work on Windows Release - add_test( - NAME ogs_python_module - COMMAND - ${Python_EXECUTABLE} - ${CMAKE_CURRENT_SOURCE_DIR}/ogs_python_module.py - ${CMAKE_SOURCE_DIR}/Tests/Data/Parabolic/LiquidFlow/Flux/cube_1e3_calculatesurfaceflux.prj - ) - set_tests_properties( - ogs_python_module - PROPERTIES - LABELS - "default" - ENVIRONMENT_MODIFICATION - PYTHONPATH=path_list_append:$:${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR} - DISABLED - $> - ) -endif() # ---- Installation ---- install(TARGETS ogs RUNTIME DESTINATION bin) diff --git a/Applications/CLI/ogs_python_module.py b/Applications/CLI/ogs_python_module.py deleted file mode 100755 index 8f553584fdd..00000000000 --- a/Applications/CLI/ogs_python_module.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python3 - -# The OpenGeoSys python library will be created in the lib folder of the build -# directory -## export PYTHONPATH=$PYTHONPATH:your-build_directory-here/lib - -import sys -import tempfile -import OpenGeoSys - -arguments = ["", sys.argv[1], "-o " + tempfile.mkdtemp()] - -print("Python OpenGeoSys.init ...") -OpenGeoSys.initialize(arguments) -print("Python OpenGeoSys.executeSimulation ...") -OpenGeoSys.executeSimulation() -print("Python OpenGeoSys.finalize() ...") -OpenGeoSys.finalize() -print("Python world.") diff --git a/Applications/CMakeLists.txt b/Applications/CMakeLists.txt index 386e8b3f40d..ce44d2681a9 100644 --- a/Applications/CMakeLists.txt +++ b/Applications/CMakeLists.txt @@ -19,3 +19,7 @@ endif() # OGS_BUILD_CLI if(OGS_USE_INSITU) add_subdirectory(InSituLib) endif() + +if(OGS_BUILD_PYTHON_MODULE) + add_subdirectory(Python) +endif() diff --git a/Applications/Python/CMakeLists.txt b/Applications/Python/CMakeLists.txt new file mode 100644 index 00000000000..0d6bb59652c --- /dev/null +++ b/Applications/Python/CMakeLists.txt @@ -0,0 +1,21 @@ +# Create OpenGeoSys python module +# https://pybind11.readthedocs.io/en/stable/compiling.html#building-with-cmake +pybind11_add_module( + _cli MODULE ogs_python_module.cpp ../CLI/CommandLineArgumentParser.cpp +) + +# lld linker strips out PyInit_OpenGeoSys symbol. Use standard linker. +get_target_property(_link_options _cli LINK_OPTIONS) +if(_link_options) + list(REMOVE_ITEM _link_options -fuse-ld=lld) + set_target_properties(_cli PROPERTIES LINK_OPTIONS "${_link_options}") +endif() + +target_link_libraries(_cli PRIVATE ApplicationsLib BaseLib GitInfoLib tclap) +target_include_directories(_cli PRIVATE ../CLI) + +set(_cli_dest ${CMAKE_INSTALL_LIBDIR}) +if(SKBUILD) + set(_cli_dest .) +endif() +install(TARGETS _cli LIBRARY DESTINATION ${_cli_dest}) diff --git a/Applications/Python/OpenGeoSys/__init__.py b/Applications/Python/OpenGeoSys/__init__.py new file mode 100644 index 00000000000..2ad97321e99 --- /dev/null +++ b/Applications/Python/OpenGeoSys/__init__.py @@ -0,0 +1 @@ +from ._cli import * diff --git a/Applications/CLI/ogs_python_module.cpp b/Applications/Python/ogs_python_module.cpp similarity index 99% rename from Applications/CLI/ogs_python_module.cpp rename to Applications/Python/ogs_python_module.cpp index 9abbe7a727d..c3e6f190a87 100644 --- a/Applications/CLI/ogs_python_module.cpp +++ b/Applications/Python/ogs_python_module.cpp @@ -122,7 +122,7 @@ void finalize() } /// python module name is OpenGeoSys -PYBIND11_MODULE(OpenGeoSys, m) +PYBIND11_MODULE(_cli, m) { m.doc() = "pybind11 ogs example plugin"; m.def("initialize", &initOGS, "init OGS"); diff --git a/Tests/Python/test_cli.py b/Tests/Python/test_cli.py new file mode 100644 index 00000000000..7517dfa739b --- /dev/null +++ b/Tests/Python/test_cli.py @@ -0,0 +1,18 @@ +import OpenGeoSys +import tempfile +import os + + +def test_init(): + arguments = [ + "", + f"{os.path.abspath(os.path.dirname(__file__))}/../Data/Parabolic/LiquidFlow/Flux/cube_1e3_calculatesurfaceflux.prj", + "-o " + tempfile.mkdtemp(), + ] + + print("Python OpenGeoSys.init ...") + OpenGeoSys.initialize(arguments) + print("Python OpenGeoSys.executeSimulation ...") + OpenGeoSys.executeSimulation() + print("Python OpenGeoSys.finalize() ...") + OpenGeoSys.finalize() diff --git a/pyproject.toml b/pyproject.toml index 5c7d0579bbd..754315d212c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,3 +6,19 @@ requires = [ "ninja; platform_system!='Windows'", ] build-backend = "setuptools.build_meta" + +[tool.pytest.ini_options] +testpaths = ["Tests"] +norecursedirs = ["Tests/Data"] + +[tool.cibuildwheel] +archs = "auto64" +build = "cp310-*" +skip = "*musllinux*" +test-extras = "test" +test-command = "pytest {project}/Tests/Python" +build-verbosity = "1" + +[tool.cibuildwheel.linux] +manylinux-x86_64-image = "manylinux2014" +manylinux-aarch64-image = "manylinux2014" diff --git a/scripts/cmake/DependenciesExternalProject.cmake b/scripts/cmake/DependenciesExternalProject.cmake index ec14fe15943..5a9c922131b 100644 --- a/scripts/cmake/DependenciesExternalProject.cmake +++ b/scripts/cmake/DependenciesExternalProject.cmake @@ -202,7 +202,7 @@ if(OGS_USE_MPI) set(HDF5_PREFER_PARALLEL ON) list(APPEND _hdf5_options "-DHDF5_ENABLE_PARALLEL=ON") endif() -if(WIN32) +if(WIN32 OR HDF5_USE_STATIC_LIBRARIES) set(HDF5_USE_STATIC_LIBRARIES ON) list(APPEND _hdf5_options "-DBUILD_SHARED_LIBS=OFF") endif() diff --git a/setup.py b/setup.py index 58e18d9ad73..f709460ae90 100644 --- a/setup.py +++ b/setup.py @@ -2,13 +2,24 @@ from setuptools import find_packages setup( - name="ogs", + name="OpenGeoSys", version="6.4.2", description="OpenGeoSys", author="OpenGeoSys Community", license="BSD-3-Clause", - packages=find_packages(where="."), - package_dir={"": "."}, - cmake_args=["-DOGS_BUILD_PROCESSES=SteadyStateDiffusion", "-DOGS_BUILD_UTILS=OFF"], + packages=find_packages(where="Applications/Python"), + package_dir={"": "Applications/Python"}, + cmake_install_dir="Applications/Python/OpenGeoSys", + extras_require={"test": ["pytest"]}, + cmake_args=[ + "-DOGS_BUILD_PROCESSES=LiquidFlow", + "-DOGS_BUILD_UTILS=OFF", + "-DHDF5_USE_STATIC_LIBRARIES=ON", + "-DOGS_BUILD_HDF5=ON", + "-DOGS_USE_PYTHON=OFF", # not possible because manylinux does not provide libpythonX.Y.so + "-DOGS_BUILD_PYTHON_MODULE=ON", + "-DOGS_BUILD_TESTING=OFF", + "-DOGS_INSTALL_DEPENDENCIES=OFF", # otherwise auditwheel fails + ], python_requires=">=3.6", ) From 85385aae8ecb2bca288ae10379f0625d78695fb3 Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Tue, 6 Sep 2022 17:26:30 +0200 Subject: [PATCH 03/36] [cmake] OGS_BUILD_PYTHON_MODULE is now an independent option. Was coupled to OGS_USE_PYTHON before. --- CMakeLists.txt | 7 ++----- scripts/cmake/Dependencies.cmake | 2 +- scripts/cmake/PythonSetup.cmake | 8 +++++++- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7c4027cd17f..556122f6a8b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,12 +4,9 @@ cmake_minimum_required(VERSION 3.22) project(OGS-6) option(OGS_USE_PYTHON "Interface with Python" ON) -include(CMakeDependentOption) -cmake_dependent_option( - OGS_BUILD_PYTHON_MODULE "Should the OGS Python module be built?" ON - "OGS_USE_PYTHON" OFF -) +option(OGS_BUILD_PYTHON_MODULE "Should the OGS Python module be built?" ON) +include(CMakeDependentOption) include(scripts/cmake/DownloadCpmCache.cmake) include(scripts/cmake/CPM.cmake) include(scripts/cmake/CMakeSetup.cmake) diff --git a/scripts/cmake/Dependencies.cmake b/scripts/cmake/Dependencies.cmake index 5c174b73f02..16e59865072 100644 --- a/scripts/cmake/Dependencies.cmake +++ b/scripts/cmake/Dependencies.cmake @@ -87,7 +87,7 @@ if(tetgen_ADDED) list(APPEND DISABLE_WARNINGS_TARGETS tet tetgen) endif() -if(OGS_USE_PYTHON) +if(OGS_USE_PYTHON OR OGS_BUILD_PYTHON_MODULE) CPMAddPackage( NAME pybind11 GITHUB_REPOSITORY pybind/pybind11 GIT_TAG f1abf5d9159b805674197f6bc443592e631c9130 diff --git a/scripts/cmake/PythonSetup.cmake b/scripts/cmake/PythonSetup.cmake index 833bb5718ca..14ac317021d 100644 --- a/scripts/cmake/PythonSetup.cmake +++ b/scripts/cmake/PythonSetup.cmake @@ -38,9 +38,15 @@ endif() set(_python_componets Interpreter) if(OGS_USE_PYTHON) + list(APPEND _python_componets Development.Embed) +endif() +if(OGS_BUILD_PYTHON_MODULE) + list(APPEND _python_componets Development.Module) +endif() +if(OGS_USE_PYTHON OR OGS_BUILD_PYTHON_MODULE) set(CMAKE_REQUIRE_FIND_PACKAGE_Python TRUE) - list(APPEND _python_componets Development) endif() + find_package( Python ${ogs.minimum_version.python}${_python_version_max} COMPONENTS ${_python_componets} From 61b1a2a6f82d5923bb7a2c9314a5594fa473dc1f Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Wed, 7 Sep 2022 08:56:30 +0200 Subject: [PATCH 04/36] [wheel] Moved CMake options to preset. --- CMakePresets.json | 14 ++++++++++++++ setup.py | 11 +---------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/CMakePresets.json b/CMakePresets.json index 703fc056a2c..7bb0e9709bd 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -187,6 +187,20 @@ "OGS_USE_PYTHON": "OFF", "OGS_USE_UNITY_BUILDS": "OFF" } + }, + { + "name": "wheel", + "inherits": "release", + "cacheVariables": { + "OGS_BUILD_PROCESSES": "LiquidFlow", + "OGS_BUILD_UTILS": "OFF", + "HDF5_USE_STATIC_LIBRARIES": "ON", + "OGS_BUILD_HDF5": "ON", + "OGS_USE_PYTHON": "OFF", + "OGS_BUILD_PYTHON_MODULE": "ON", + "OGS_BUILD_TESTING": "OFF", + "OGS_INSTALL_DEPENDENCIES": "OFF" + } } ], "buildPresets": [ diff --git a/setup.py b/setup.py index f709460ae90..59908b1de0b 100644 --- a/setup.py +++ b/setup.py @@ -11,15 +11,6 @@ package_dir={"": "Applications/Python"}, cmake_install_dir="Applications/Python/OpenGeoSys", extras_require={"test": ["pytest"]}, - cmake_args=[ - "-DOGS_BUILD_PROCESSES=LiquidFlow", - "-DOGS_BUILD_UTILS=OFF", - "-DHDF5_USE_STATIC_LIBRARIES=ON", - "-DOGS_BUILD_HDF5=ON", - "-DOGS_USE_PYTHON=OFF", # not possible because manylinux does not provide libpythonX.Y.so - "-DOGS_BUILD_PYTHON_MODULE=ON", - "-DOGS_BUILD_TESTING=OFF", - "-DOGS_INSTALL_DEPENDENCIES=OFF", # otherwise auditwheel fails - ], + cmake_args=["--preset wheel"], python_requires=">=3.6", ) From 0566d251e99910e71ade90c766227b884f54c8db Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Wed, 7 Sep 2022 10:50:12 +0200 Subject: [PATCH 05/36] [wheel] Added ogs binary entrypoint. Implementation from https://github.com/scikit-build/cmake-python-distributions --- Applications/Python/OpenGeoSys/__init__.py | 14 ++++++++++++++ CMakePresets.json | 4 ++++ Tests/Python/__init__.py | 11 +++++++++++ Tests/Python/test_cli.py | 22 ++++++++++++++++++++-- setup.py | 7 ++++++- 5 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 Tests/Python/__init__.py diff --git a/Applications/Python/OpenGeoSys/__init__.py b/Applications/Python/OpenGeoSys/__init__.py index 2ad97321e99..7631e8c3369 100644 --- a/Applications/Python/OpenGeoSys/__init__.py +++ b/Applications/Python/OpenGeoSys/__init__.py @@ -1 +1,15 @@ +import os +import subprocess +import sys + from ._cli import * + +OGS_BIN_DIR = os.path.join(os.path.join(os.path.dirname(__file__), "bin")) + + +def _program(name, args): + return subprocess.call([os.path.join(OGS_BIN_DIR, name)] + args) + + +def ogs(): + raise SystemExit(_program("ogs", sys.argv[1:])) diff --git a/CMakePresets.json b/CMakePresets.json index 7bb0e9709bd..3b710a650c6 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -284,6 +284,10 @@ { "name": "ci-no-unity", "configurePreset": "ci-no-unity" + }, + { + "name": "wheel", + "configurePreset": "wheel" } ], "testPresets": [ diff --git a/Tests/Python/__init__.py b/Tests/Python/__init__.py new file mode 100644 index 00000000000..fd1659acb19 --- /dev/null +++ b/Tests/Python/__init__.py @@ -0,0 +1,11 @@ +import sys + +from contextlib import contextmanager + + +@contextmanager +def push_argv(argv): + old_argv = sys.argv + sys.argv = argv + yield + sys.argv = old_argv diff --git a/Tests/Python/test_cli.py b/Tests/Python/test_cli.py index 7517dfa739b..7f6c624269d 100644 --- a/Tests/Python/test_cli.py +++ b/Tests/Python/test_cli.py @@ -1,9 +1,12 @@ -import OpenGeoSys import tempfile import os +import pytest + +import OpenGeoSys -def test_init(): + +def test_module(): arguments = [ "", f"{os.path.abspath(os.path.dirname(__file__))}/../Data/Parabolic/LiquidFlow/Flux/cube_1e3_calculatesurfaceflux.prj", @@ -16,3 +19,18 @@ def test_init(): OpenGeoSys.executeSimulation() print("Python OpenGeoSys.finalize() ...") OpenGeoSys.finalize() + + +from . import push_argv + + +def _run(program, args): + func = getattr(OpenGeoSys, program) + args = ["%s.py" % program] + args + with push_argv(args), pytest.raises(SystemExit) as excinfo: + func() + assert 0 == excinfo.value.code + + +def test_binaries(): + _run("ogs", ["--version"]) diff --git a/setup.py b/setup.py index 59908b1de0b..eaa57d94967 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,11 @@ package_dir={"": "Applications/Python"}, cmake_install_dir="Applications/Python/OpenGeoSys", extras_require={"test": ["pytest"]}, - cmake_args=["--preset wheel"], + cmake_args=["--preset wheel", "-B ."], python_requires=">=3.6", + entry_points={ + "console_scripts": [ + "ogs=OpenGeoSys:ogs", + ] + }, ) From 39ce500aed533f6fc2ef1ff67c95449bfe9eaff1 Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Wed, 7 Sep 2022 11:25:52 +0200 Subject: [PATCH 06/36] [ci] Added job build wheels linux. --- .gitlab-ci.yml | 1 + scripts/ci/jobs/build-wheels.yml | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 scripts/ci/jobs/build-wheels.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e4fe9bb3d18..b92c02dcc36 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,6 +37,7 @@ include: - local: "/scripts/ci/jobs/build-win.yml" - local: "/scripts/ci/jobs/build-mac.yml" - local: "/scripts/ci/jobs/build-container.yml" + - local: "/scripts/ci/jobs/build-wheels.yml" - local: "/scripts/ci/jobs/jupyter.yml" - local: "/scripts/ci/jobs/code-quality.yml" - local: "/scripts/ci/jobs/code-coverage.yml" diff --git a/scripts/ci/jobs/build-wheels.yml b/scripts/ci/jobs/build-wheels.yml new file mode 100644 index 00000000000..0c29c74387e --- /dev/null +++ b/scripts/ci/jobs/build-wheels.yml @@ -0,0 +1,9 @@ +build wheels linux: + stage: build + needs: [meta] + tags: [envinf3-shell] + script: + - pipx run cibuildwheel + artifacts: + paths: + - wheelhouse/ From 791ae864b7135917e53cc0aa6766ca9a9ba3e91b Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Wed, 7 Sep 2022 14:20:14 +0200 Subject: [PATCH 07/36] [wheel] Get ogs version from env or git. [ci] Upload wheel to GitLab registry. --- Applications/Python/OpenGeoSys/__init__.py | 1 + pyproject.toml | 1 + scripts/ci/jobs/build-wheels.yml | 6 ++++ setup.py | 32 +++++++++++++++++++++- 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/Applications/Python/OpenGeoSys/__init__.py b/Applications/Python/OpenGeoSys/__init__.py index 7631e8c3369..628678b353d 100644 --- a/Applications/Python/OpenGeoSys/__init__.py +++ b/Applications/Python/OpenGeoSys/__init__.py @@ -11,5 +11,6 @@ def _program(name, args): return subprocess.call([os.path.join(OGS_BIN_DIR, name)] + args) +# Binary entrypoints def ogs(): raise SystemExit(_program("ogs", sys.argv[1:])) diff --git a/pyproject.toml b/pyproject.toml index 754315d212c..9c3d7b3b2c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,3 +22,4 @@ build-verbosity = "1" [tool.cibuildwheel.linux] manylinux-x86_64-image = "manylinux2014" manylinux-aarch64-image = "manylinux2014" +environment-pass = ["OGS_VERSION"] diff --git a/scripts/ci/jobs/build-wheels.yml b/scripts/ci/jobs/build-wheels.yml index 0c29c74387e..6bcab571755 100644 --- a/scripts/ci/jobs/build-wheels.yml +++ b/scripts/ci/jobs/build-wheels.yml @@ -4,6 +4,12 @@ build wheels linux: tags: [envinf3-shell] script: - pipx run cibuildwheel + - > + pipx run twine upload + --repository-url https://gitlab.opengeosys.org/api/v4/projects/${CI_PROJECT_ID}/packages/pypi + --username gitlab-ci-token + --password ${CI_JOB_TOKEN} + wheelhouse/* artifacts: paths: - wheelhouse/ diff --git a/setup.py b/setup.py index eaa57d94967..f95e450d35f 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,39 @@ from skbuild import setup from setuptools import find_packages +import os +import re +import subprocess + + +def get_version(): + git_version = "" + if "OGS_VERSION" in os.environ: + git_version = os.environ["OGS_VERSION"] + else: + git_version = subprocess.run( + ["git describe --tags"], + capture_output=True, + text=True, + shell=True, + ).stdout.strip() + + if re.match("\d+\.\d+\.\d+-\d+-g\w+", git_version): + # Make it PEP 440 compliant + # e.g. 6.4.2-1140-g85bbc8b4e1 -> 6.4.2.dev1140 + m = re.match(".+?(?=-g[\w]*$)", git_version) # strip out commit hash + if m: + return m.group(0).replace("-", ".dev") # insert dev + else: + print("WARNING: Could not get ogs version!") + exit(1) + else: + return git_version + + setup( name="OpenGeoSys", - version="6.4.2", + version=get_version(), description="OpenGeoSys", author="OpenGeoSys Community", license="BSD-3-Clause", From 850d10ae840349994917762e01359bc2fb3b6e67 Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Thu, 8 Sep 2022 13:16:43 +0200 Subject: [PATCH 08/36] [wheel] Enabled MFront and optimized size with shared libs. --- Applications/Python/CMakeLists.txt | 6 +----- Applications/Python/OpenGeoSys/__init__.py | 8 +++++++- CMakePresets.json | 5 ++++- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/Applications/Python/CMakeLists.txt b/Applications/Python/CMakeLists.txt index 0d6bb59652c..98801e4cf44 100644 --- a/Applications/Python/CMakeLists.txt +++ b/Applications/Python/CMakeLists.txt @@ -14,8 +14,4 @@ endif() target_link_libraries(_cli PRIVATE ApplicationsLib BaseLib GitInfoLib tclap) target_include_directories(_cli PRIVATE ../CLI) -set(_cli_dest ${CMAKE_INSTALL_LIBDIR}) -if(SKBUILD) - set(_cli_dest .) -endif() -install(TARGETS _cli LIBRARY DESTINATION ${_cli_dest}) +install(TARGETS _cli LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) diff --git a/Applications/Python/OpenGeoSys/__init__.py b/Applications/Python/OpenGeoSys/__init__.py index 628678b353d..d25cddd08e7 100644 --- a/Applications/Python/OpenGeoSys/__init__.py +++ b/Applications/Python/OpenGeoSys/__init__.py @@ -2,7 +2,13 @@ import subprocess import sys -from ._cli import * +try: + from .lib64._cli import * +except ImportError: + try: + from .lib._cli import * + except ImportError: + print("ERROR: could not import OpenGeoSys Python module!") OGS_BIN_DIR = os.path.join(os.path.join(os.path.dirname(__file__), "bin")) diff --git a/CMakePresets.json b/CMakePresets.json index 3b710a650c6..517f699d1b3 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -199,7 +199,10 @@ "OGS_USE_PYTHON": "OFF", "OGS_BUILD_PYTHON_MODULE": "ON", "OGS_BUILD_TESTING": "OFF", - "OGS_INSTALL_DEPENDENCIES": "OFF" + "OGS_INSTALL_DEPENDENCIES": "OFF", + "OGS_USE_PIP": "OFF", + "OGS_USE_MFRONT": "ON", + "BUILD_SHARED_LIBS": "ON" } } ], From 34d166828092a68223837e0b3780e8165c1503b8 Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Thu, 8 Sep 2022 15:09:32 +0200 Subject: [PATCH 09/36] [T] Fixed tclap exceptions due to duplicate flags. --- Applications/Utils/MeshEdit/appendLinesAlongPolyline.cpp | 2 +- .../MeshGeoTools/computeSurfaceNodeIDsInPolygonalRegion.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Applications/Utils/MeshEdit/appendLinesAlongPolyline.cpp b/Applications/Utils/MeshEdit/appendLinesAlongPolyline.cpp index 0774cc64529..1b6625b70fe 100644 --- a/Applications/Utils/MeshEdit/appendLinesAlongPolyline.cpp +++ b/Applications/Utils/MeshEdit/appendLinesAlongPolyline.cpp @@ -50,7 +50,7 @@ int main(int argc, char* argv[]) "the name of the geometry file"); cmd.add(geoFileArg); - TCLAP::ValueArg gmsh_path_arg("g", "gmsh-path", + TCLAP::ValueArg gmsh_path_arg("", "gmsh-path", "the path to the gmsh binary", false, "", "path as string"); cmd.add(gmsh_path_arg); diff --git a/Applications/Utils/MeshGeoTools/computeSurfaceNodeIDsInPolygonalRegion.cpp b/Applications/Utils/MeshGeoTools/computeSurfaceNodeIDsInPolygonalRegion.cpp index b628eceb948..355c00e3269 100644 --- a/Applications/Utils/MeshGeoTools/computeSurfaceNodeIDsInPolygonalRegion.cpp +++ b/Applications/Utils/MeshGeoTools/computeSurfaceNodeIDsInPolygonalRegion.cpp @@ -94,7 +94,7 @@ int main(int argc, char* argv[]) true, "", "file name of input geometry"); cmd.add(geo_in); - TCLAP::ValueArg gmsh_path_arg("g", "gmsh-path", + TCLAP::ValueArg gmsh_path_arg("", "gmsh-path", "the path to the gmsh binary", false, "", "path as string"); cmd.add(gmsh_path_arg); From ca159582aae6f95be5820f8f3e9377981cab7870 Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Thu, 8 Sep 2022 15:10:08 +0200 Subject: [PATCH 10/36] [T] Added correct --version information. --- Applications/Utils/GeoTools/addDataToRaster.cpp | 10 +++++++++- Applications/Utils/GeoTools/createRaster.cpp | 11 +++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/Applications/Utils/GeoTools/addDataToRaster.cpp b/Applications/Utils/GeoTools/addDataToRaster.cpp index f21fc93f952..92e3ef9d337 100644 --- a/Applications/Utils/GeoTools/addDataToRaster.cpp +++ b/Applications/Utils/GeoTools/addDataToRaster.cpp @@ -24,6 +24,7 @@ #include "GeoLib/AABB.h" #include "GeoLib/Point.h" #include "GeoLib/Raster.h" +#include "InfoLib/GitInfo.h" double compute2DGaussBellCurveValues(GeoLib::Point const& point, GeoLib::AABB const& aabb) @@ -51,7 +52,14 @@ double computeSinXSinY(GeoLib::Point const& point, GeoLib::AABB const& aabb) int main(int argc, char* argv[]) { - TCLAP::CmdLine cmd("Add values to raster.", ' ', "0.1"); + TCLAP::CmdLine cmd( + "Add values to raster.\n\n" + "OpenGeoSys-6 software, version " + + GitInfoLib::GitInfo::ogs_version + + ".\n" + "Copyright (c) 2012-2022, OpenGeoSys Community " + "(http://www.opengeosys.org)", + ' ', GitInfoLib::GitInfo::ogs_version); TCLAP::ValueArg out_raster_arg( "o", diff --git a/Applications/Utils/GeoTools/createRaster.cpp b/Applications/Utils/GeoTools/createRaster.cpp index d58e7c29036..c9a521c53d5 100644 --- a/Applications/Utils/GeoTools/createRaster.cpp +++ b/Applications/Utils/GeoTools/createRaster.cpp @@ -20,11 +20,18 @@ #include "GeoLib/AABB.h" #include "GeoLib/Point.h" #include "GeoLib/Raster.h" +#include "InfoLib/GitInfo.h" int main(int argc, char* argv[]) { - TCLAP::CmdLine cmd("Create a raster where every pixel is zero.", ' ', - "0.1"); + TCLAP::CmdLine cmd( + "computeSurfaceNodeIDsInPolygonalRegion\n\n" + "OpenGeoSys-6 software, version " + + GitInfoLib::GitInfo::ogs_version + + ".\n" + "Copyright (c) 2012-2022, OpenGeoSys Community " + "(http://www.opengeosys.org)", + ' ', GitInfoLib::GitInfo::ogs_version); TCLAP::ValueArg output_arg("o", "output", "Name of the output raster (*.asc)", From 26e78e47aa6f3c9b33fb9df663e4198a11932c71 Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Thu, 8 Sep 2022 15:10:42 +0200 Subject: [PATCH 11/36] [wheel] Added missing binary entry points for all tools. --- Applications/Python/OpenGeoSys/__init__.py | 69 +++++++++++++++++++++- CMakePresets.json | 1 - Tests/Python/test_cli.py | 5 +- setup.py | 14 +++-- 4 files changed, 79 insertions(+), 10 deletions(-) diff --git a/Applications/Python/OpenGeoSys/__init__.py b/Applications/Python/OpenGeoSys/__init__.py index d25cddd08e7..120592366f0 100644 --- a/Applications/Python/OpenGeoSys/__init__.py +++ b/Applications/Python/OpenGeoSys/__init__.py @@ -12,11 +12,74 @@ OGS_BIN_DIR = os.path.join(os.path.join(os.path.dirname(__file__), "bin")) +binaries_list = [ + "addDataToRaster", + "AddElementQuality", + "AddFaultToVoxelGrid", + "AddLayer", + "appendLinesAlongPolyline", + "AssignRasterDataToMesh", + "checkMesh", + "ComputeNodeAreasFromSurfaceMesh", + "computeSurfaceNodeIDsInPolygonalRegion", + "constructMeshesFromGeometry", + "convertGEO", + "convertToLinearMesh", + "convertVtkDataArrayToVtkDataArray", + "CreateBoundaryConditionsAlongPolylines", + "createIntermediateRasters", + "createLayeredMeshFromRasters", + "createMeshElemPropertiesFromASCRaster", + "createNeumannBc", + "createQuadraticMesh", + "createRaster", + "editMaterialID", + "ExtractBoundary", + "ExtractMaterials", + "ExtractSurface", + "generateGeometry", + "generateMatPropsFromMatID", + "generateStructuredMesh", + "geometryToGmshGeo", + "GMSH2OGS", + "GocadSGridReader", + "GocadTSurfaceReader", + "identifySubdomains", + "IntegrateBoreholesIntoMesh", + "Layers2Grid", + "MapGeometryToMeshSurface", + "Mesh2Raster", + "MoveGeometry", + "MoveMesh", + "moveMeshNodes", + "mpmetis", + "NodeReordering", + "ogs", + "OGS2VTK", + "partmesh", + "PVD2XDMF", + "queryMesh", + "Raster2Mesh", + "RemoveGhostData", + "removeMeshElements", + "ResetPropertiesInPolygonalRegion", + "reviseMesh", + "scaleProperty", + "swapNodeCoordinateAxes", + "TecPlotTools", + "tetgen", + "TIN2VTK", + "VTK2OGS", + "VTK2TIN", + "vtkdiff", + "Vtu2Grid", +] + def _program(name, args): return subprocess.call([os.path.join(OGS_BIN_DIR, name)] + args) -# Binary entrypoints -def ogs(): - raise SystemExit(_program("ogs", sys.argv[1:])) +FUNC_TEMPLATE = """def {0}(): raise SystemExit(_program("{0}", sys.argv[1:]))""" +for f in binaries_list: + exec(FUNC_TEMPLATE.format(f)) diff --git a/CMakePresets.json b/CMakePresets.json index 517f699d1b3..5eabcc4085b 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -193,7 +193,6 @@ "inherits": "release", "cacheVariables": { "OGS_BUILD_PROCESSES": "LiquidFlow", - "OGS_BUILD_UTILS": "OFF", "HDF5_USE_STATIC_LIBRARIES": "ON", "OGS_BUILD_HDF5": "ON", "OGS_USE_PYTHON": "OFF", diff --git a/Tests/Python/test_cli.py b/Tests/Python/test_cli.py index 7f6c624269d..e2b60fc4fd4 100644 --- a/Tests/Python/test_cli.py +++ b/Tests/Python/test_cli.py @@ -33,4 +33,7 @@ def _run(program, args): def test_binaries(): - _run("ogs", ["--version"]) + ignore_list = ["moveMeshNodes", "mpmetis", "tetgen"] # have no --version cli flag + for f in OpenGeoSys.binaries_list: + if f not in ignore_list: + _run(f, ["--version"]) diff --git a/setup.py b/setup.py index f95e450d35f..2cb3ad792f5 100644 --- a/setup.py +++ b/setup.py @@ -4,6 +4,7 @@ import os import re import subprocess +import sys def get_version(): @@ -31,6 +32,13 @@ def get_version(): return git_version +sys.path.append("Applications/Python") +from OpenGeoSys import binaries_list + +console_scripts = [] +for b in binaries_list: + console_scripts.append(f"{b}=OpenGeoSys:{b}") + setup( name="OpenGeoSys", version=get_version(), @@ -43,9 +51,5 @@ def get_version(): extras_require={"test": ["pytest"]}, cmake_args=["--preset wheel", "-B ."], python_requires=">=3.6", - entry_points={ - "console_scripts": [ - "ogs=OpenGeoSys:ogs", - ] - }, + entry_points={"console_scripts": console_scripts}, ) From 3dded39521aa5ac2d1d67562344df1b8afeaf17a Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Thu, 8 Sep 2022 15:14:27 +0200 Subject: [PATCH 12/36] [ci] Added mac jobs for wheels. --- scripts/ci/jobs/build-wheels.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/scripts/ci/jobs/build-wheels.yml b/scripts/ci/jobs/build-wheels.yml index 6bcab571755..b08f6981978 100644 --- a/scripts/ci/jobs/build-wheels.yml +++ b/scripts/ci/jobs/build-wheels.yml @@ -13,3 +13,24 @@ build wheels linux: artifacts: paths: - wheelhouse/ + +build wheels mac: + stage: build + needs: [meta] + tags: + - mac + - ${ARCHITECTURE} + script: + - CMAKE_OSX_DEPLOYMENT_TARGET=10.15 pipx run cibuildwheel + - > + pipx run twine upload + --repository-url https://gitlab.opengeosys.org/api/v4/projects/${CI_PROJECT_ID}/packages/pypi + --username gitlab-ci-token + --password ${CI_JOB_TOKEN} + wheelhouse/* + artifacts: + paths: + - wheelhouse/ + parallel: + matrix: + - ARCHITECTURE: ["amd64", "arm64"] From 7957a454206767ac60ec8305d478a38e47d8457a Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Thu, 8 Sep 2022 15:52:33 +0200 Subject: [PATCH 13/36] [wheel] Build for different Python versions. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9c3d7b3b2c7..e9b2b352a81 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ norecursedirs = ["Tests/Data"] [tool.cibuildwheel] archs = "auto64" -build = "cp310-*" +build = "cp3*" skip = "*musllinux*" test-extras = "test" test-command = "pytest {project}/Tests/Python" From 71b9cac855f02b3e0b734d9d5ae8201d2a2bd428 Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Thu, 8 Sep 2022 20:34:26 +0200 Subject: [PATCH 14/36] [cmake] Bump pybind11 to 2.10.0. --- scripts/cmake/Dependencies.cmake | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/cmake/Dependencies.cmake b/scripts/cmake/Dependencies.cmake index 16e59865072..5a37639f28a 100644 --- a/scripts/cmake/Dependencies.cmake +++ b/scripts/cmake/Dependencies.cmake @@ -89,8 +89,7 @@ endif() if(OGS_USE_PYTHON OR OGS_BUILD_PYTHON_MODULE) CPMAddPackage( - NAME pybind11 GITHUB_REPOSITORY pybind/pybind11 - GIT_TAG f1abf5d9159b805674197f6bc443592e631c9130 + NAME pybind11 GITHUB_REPOSITORY pybind/pybind11 VERSION 2.10.0 ) endif() From 6c5d4aebbb5b41dcd1705482058881e90b8d931d Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Thu, 8 Sep 2022 22:15:55 +0200 Subject: [PATCH 15/36] [wheel] Dont' build 3.6 and 3.7 wheels on macOS. Is not supported for CMAKE_OSX_DEPLOYMENT_TARGET=10.15 --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index e9b2b352a81..69021cbd4a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,3 +23,6 @@ build-verbosity = "1" manylinux-x86_64-image = "manylinux2014" manylinux-aarch64-image = "manylinux2014" environment-pass = ["OGS_VERSION"] + +[tool.cibuildwheel.macos] +skip = ["cp36-*", "cp37-*", "cp38-*x86_64"] From 91ed5d1650f0ae91c71877c8f1fa536b0e2921c6 Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Fri, 9 Sep 2022 13:16:32 +0200 Subject: [PATCH 16/36] [cmake] Added missing dependency in Python _cli module. --- Applications/Python/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Applications/Python/CMakeLists.txt b/Applications/Python/CMakeLists.txt index 98801e4cf44..ef45e964522 100644 --- a/Applications/Python/CMakeLists.txt +++ b/Applications/Python/CMakeLists.txt @@ -11,7 +11,7 @@ if(_link_options) set_target_properties(_cli PROPERTIES LINK_OPTIONS "${_link_options}") endif() -target_link_libraries(_cli PRIVATE ApplicationsLib BaseLib GitInfoLib tclap) +target_link_libraries(_cli PRIVATE ApplicationsLib BaseLib CMakeInfoLib GitInfoLib tclap) target_include_directories(_cli PRIVATE ../CLI) install(TARGETS _cli LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) From ac90b262a93eeb49787489353bb381348464ab6c Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Fri, 9 Sep 2022 13:18:58 +0200 Subject: [PATCH 17/36] [cmake] Enable BUILD_SHARED_LIBS=ON on Windows. --- scripts/cmake/Dependencies.cmake | 1 + scripts/cmake/MetisSetup.cmake | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/cmake/Dependencies.cmake b/scripts/cmake/Dependencies.cmake index 5a37639f28a..f891daaf0e8 100644 --- a/scripts/cmake/Dependencies.cmake +++ b/scripts/cmake/Dependencies.cmake @@ -181,6 +181,7 @@ if(OGS_BUILD_SWMM) CPMAddPackage( NAME SWMMInterface GITHUB_REPOSITORY ufz/SwmmInterface GIT_TAG 141e05ae1f419918799d7bf9178ebcd97feb1ed3 + OPTIONS "BUILD_SHARED_LIBS OFF" ) if(SWMMInterface_ADDED) target_include_directories( diff --git a/scripts/cmake/MetisSetup.cmake b/scripts/cmake/MetisSetup.cmake index 6c6fde90cae..73131168b26 100644 --- a/scripts/cmake/MetisSetup.cmake +++ b/scripts/cmake/MetisSetup.cmake @@ -10,7 +10,10 @@ include(${GKLIB_PATH}/GKlibSystem.cmake) # Metis library file(GLOB _metis_sources ${metis_SOURCE_DIR}/libmetis/*.c) -ogs_add_library(ogs_metis ${GKlib_sources} ${_metis_sources}) +if(WIN32) + set(_metis_static STATIC) +endif() +ogs_add_library(ogs_metis ${_metis_static} ${GKlib_sources} ${_metis_sources}) target_compile_definitions(ogs_metis PUBLIC USE_GKREGEX) target_include_directories( ogs_metis PUBLIC ${metis_SOURCE_DIR}/GKlib ${metis_SOURCE_DIR}/include From 7512578a23f6103eb3e847a9e9ef72f1ada3b786 Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Fri, 9 Sep 2022 21:51:16 +0200 Subject: [PATCH 18/36] [wheel] Win builds. --- Applications/Python/OpenGeoSys/__init__.py | 36 +++++++++++++--------- CMakePresets.json | 21 +++++++++++++ pyproject.toml | 9 ++++-- setup.py | 11 +++++-- 4 files changed, 57 insertions(+), 20 deletions(-) diff --git a/Applications/Python/OpenGeoSys/__init__.py b/Applications/Python/OpenGeoSys/__init__.py index 120592366f0..d4b704a8f3e 100644 --- a/Applications/Python/OpenGeoSys/__init__.py +++ b/Applications/Python/OpenGeoSys/__init__.py @@ -1,17 +1,8 @@ import os +import platform import subprocess import sys -try: - from .lib64._cli import * -except ImportError: - try: - from .lib._cli import * - except ImportError: - print("ERROR: could not import OpenGeoSys Python module!") - -OGS_BIN_DIR = os.path.join(os.path.join(os.path.dirname(__file__), "bin")) - binaries_list = [ "addDataToRaster", "AddElementQuality", @@ -75,11 +66,26 @@ "Vtu2Grid", ] +if "PEP517_BUILD_BACKEND" not in os.environ: + if platform.system() == "Windows": + os.add_dll_directory(os.path.join(os.path.dirname(__file__), "bin")) + from .lib._cli import * + else: + try: + from .lib64._cli import * + except ImportError: + try: + from .lib._cli import * + except ImportError: + print("ERROR: could not import OpenGeoSys Python module!") + + OGS_BIN_DIR = os.path.join(os.path.join(os.path.dirname(__file__), "bin")) + -def _program(name, args): - return subprocess.call([os.path.join(OGS_BIN_DIR, name)] + args) + def _program(name, args): + return subprocess.call([os.path.join(OGS_BIN_DIR, name)] + args) -FUNC_TEMPLATE = """def {0}(): raise SystemExit(_program("{0}", sys.argv[1:]))""" -for f in binaries_list: - exec(FUNC_TEMPLATE.format(f)) + FUNC_TEMPLATE = """def {0}(): raise SystemExit(_program("{0}", sys.argv[1:]))""" + for f in binaries_list: + exec(FUNC_TEMPLATE.format(f)) diff --git a/CMakePresets.json b/CMakePresets.json index 5eabcc4085b..500fcc83850 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -202,6 +202,23 @@ "OGS_USE_PIP": "OFF", "OGS_USE_MFRONT": "ON", "BUILD_SHARED_LIBS": "ON" + }, + "condition": { + "type": "notEquals", + "lhs": "${hostSystemName}", + "rhs": "Windows" + } + }, + { + "name": "wheel-win", + "inherits": "wheel", + "cacheVariables": { + "OGS_USE_MFRONT": "OFF" + }, + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Windows" } } ], @@ -290,6 +307,10 @@ { "name": "wheel", "configurePreset": "wheel" + }, + { + "name": "wheel-win", + "configurePreset": "wheel-win" } ], "testPresets": [ diff --git a/pyproject.toml b/pyproject.toml index 69021cbd4a1..768f363e681 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,10 @@ [build-system] requires = [ "setuptools>=42", - "scikit-build>=0.15.0", + "scikit-build @ git+https://github.com/bilke/scikit-build/@disable-cmake-install-check#egg=scikit-build ; platform_system == 'Windows'", + "scikit-build>=0.15.0 ; platform_system != 'Windows'", "cmake>=3.22", - "ninja; platform_system!='Windows'", + "ninja ; platform_system != 'Windows'", ] build-backend = "setuptools.build_meta" @@ -17,6 +18,7 @@ build = "cp3*" skip = "*musllinux*" test-extras = "test" test-command = "pytest {project}/Tests/Python" +# test-command = "" build-verbosity = "1" [tool.cibuildwheel.linux] @@ -26,3 +28,6 @@ environment-pass = ["OGS_VERSION"] [tool.cibuildwheel.macos] skip = ["cp36-*", "cp37-*", "cp38-*x86_64"] + +[tool.cibuildwheel.windows] +skip = ["cp36-*", "cp37-*"] diff --git a/setup.py b/setup.py index 2cb3ad792f5..e5a8c44f2da 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ def get_version(): git_version = os.environ["OGS_VERSION"] else: git_version = subprocess.run( - ["git describe --tags"], + ["git", "describe", "--tags"], capture_output=True, text=True, shell=True, @@ -32,13 +32,18 @@ def get_version(): return git_version -sys.path.append("Applications/Python") +sys.path.append(os.path.join("Applications", "Python")) from OpenGeoSys import binaries_list console_scripts = [] for b in binaries_list: console_scripts.append(f"{b}=OpenGeoSys:{b}") +import platform +cmake_preset = "wheel" +if platform.system() == "Windows": + cmake_preset += "-win" + setup( name="OpenGeoSys", version=get_version(), @@ -49,7 +54,7 @@ def get_version(): package_dir={"": "Applications/Python"}, cmake_install_dir="Applications/Python/OpenGeoSys", extras_require={"test": ["pytest"]}, - cmake_args=["--preset wheel", "-B ."], + cmake_args=[f"--preset {cmake_preset}", "-B ."], python_requires=">=3.6", entry_points={"console_scripts": console_scripts}, ) From 7e96be7e88a8e8424531bb1c209686c525b77ac6 Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Fri, 9 Sep 2022 21:53:32 +0200 Subject: [PATCH 19/36] [ci] Added build wheels win job. --- CMakePresets.json | 3 ++- scripts/ci/jobs/build-wheels.yml | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/CMakePresets.json b/CMakePresets.json index 500fcc83850..3f2871df967 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -213,7 +213,8 @@ "name": "wheel-win", "inherits": "wheel", "cacheVariables": { - "OGS_USE_MFRONT": "OFF" + "OGS_USE_MFRONT": "OFF", + "OGS_BUILD_PROCESS_TH2M": "OFF" }, "condition": { "type": "equals", diff --git a/scripts/ci/jobs/build-wheels.yml b/scripts/ci/jobs/build-wheels.yml index b08f6981978..213db165ae9 100644 --- a/scripts/ci/jobs/build-wheels.yml +++ b/scripts/ci/jobs/build-wheels.yml @@ -34,3 +34,22 @@ build wheels mac: parallel: matrix: - ARCHITECTURE: ["amd64", "arm64"] + +build wheels win: + stage: build + needs: [meta] + tags: + - windows + extends: + - .vs2019-environment + script: + - pipx run cibuildwheel + - > + pipx run twine upload + --repository-url https://gitlab.opengeosys.org/api/v4/projects/${CI_PROJECT_ID}/packages/pypi + --username gitlab-ci-token + --password ${CI_JOB_TOKEN} + wheelhouse/* + artifacts: + paths: + - wheelhouse/ From c925801e332dff9e23967d8c90b55227e7b03afb Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Tue, 13 Sep 2022 08:41:36 +0200 Subject: [PATCH 20/36] [cmake] Allow Python 3.11 wheel builds. --- pyproject.toml | 2 +- scripts/cmake/PythonSetup.cmake | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 768f363e681..acb60b62466 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ norecursedirs = ["Tests/Data"] [tool.cibuildwheel] archs = "auto64" build = "cp3*" -skip = "*musllinux*" +skip = ["*musllinux*"] test-extras = "test" test-command = "pytest {project}/Tests/Python" # test-command = "" diff --git a/scripts/cmake/PythonSetup.cmake b/scripts/cmake/PythonSetup.cmake index 14ac317021d..ef6be751483 100644 --- a/scripts/cmake/PythonSetup.cmake +++ b/scripts/cmake/PythonSetup.cmake @@ -1,6 +1,9 @@ # cmake-lint: disable=C0103 -set(_python_version_max "...<3.11") +if(OGS_USE_PYTHON) + set(_python_version_max "...<3.11") +endif() + if(OGS_USE_PIP) set(Python_ROOT_DIR ${PROJECT_BINARY_DIR}/.venv) set(CMAKE_REQUIRE_FIND_PACKAGE_Python TRUE) From 4f5d8a36a229f1b7545f134326794e64d497f071 Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Tue, 13 Sep 2022 14:38:26 +0200 Subject: [PATCH 21/36] [py] Renamed wheels package to ogs and coupling module to ogs.simulator. --- Applications/Python/CMakeLists.txt | 18 +-------------- .../Python/ogs.simulator/CMakeLists.txt | 21 ++++++++++++++++++ .../{ => ogs.simulator}/ogs_python_module.cpp | 3 ++- .../Python/{OpenGeoSys => ogs}/__init__.py | 11 ---------- Tests/Python/test_cli.py | 22 +++---------------- Tests/Python/test_simlator.py | 20 +++++++++++++++++ pyproject.toml | 3 +-- scripts/cmake/ProjectSetup.cmake | 8 +++++-- setup.py | 19 +++++++++------- 9 files changed, 65 insertions(+), 60 deletions(-) create mode 100644 Applications/Python/ogs.simulator/CMakeLists.txt rename Applications/Python/{ => ogs.simulator}/ogs_python_module.cpp (98%) rename Applications/Python/{OpenGeoSys => ogs}/__init__.py (87%) create mode 100644 Tests/Python/test_simlator.py diff --git a/Applications/Python/CMakeLists.txt b/Applications/Python/CMakeLists.txt index ef45e964522..667d1aff703 100644 --- a/Applications/Python/CMakeLists.txt +++ b/Applications/Python/CMakeLists.txt @@ -1,17 +1 @@ -# Create OpenGeoSys python module -# https://pybind11.readthedocs.io/en/stable/compiling.html#building-with-cmake -pybind11_add_module( - _cli MODULE ogs_python_module.cpp ../CLI/CommandLineArgumentParser.cpp -) - -# lld linker strips out PyInit_OpenGeoSys symbol. Use standard linker. -get_target_property(_link_options _cli LINK_OPTIONS) -if(_link_options) - list(REMOVE_ITEM _link_options -fuse-ld=lld) - set_target_properties(_cli PROPERTIES LINK_OPTIONS "${_link_options}") -endif() - -target_link_libraries(_cli PRIVATE ApplicationsLib BaseLib CMakeInfoLib GitInfoLib tclap) -target_include_directories(_cli PRIVATE ../CLI) - -install(TARGETS _cli LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) +add_subdirectory(ogs.simulator) diff --git a/Applications/Python/ogs.simulator/CMakeLists.txt b/Applications/Python/ogs.simulator/CMakeLists.txt new file mode 100644 index 00000000000..dce167cd59f --- /dev/null +++ b/Applications/Python/ogs.simulator/CMakeLists.txt @@ -0,0 +1,21 @@ +# Create OpenGeoSys python module +# https://pybind11.readthedocs.io/en/stable/compiling.html#building-with-cmake +pybind11_add_module( + simulator MODULE ogs_python_module.cpp + ../../CLI/CommandLineArgumentParser.cpp +) + +# lld linker strips out PyInit_OpenGeoSys symbol. Use standard linker. +get_target_property(_link_options simulator LINK_OPTIONS) +if(_link_options) + list(REMOVE_ITEM _link_options -fuse-ld=lld) + set_target_properties(simulator PROPERTIES LINK_OPTIONS "${_link_options}") +endif() + +target_link_libraries( + simulator PRIVATE ApplicationsLib BaseLib CMakeInfoLib GitInfoLib tclap +) +target_include_directories(simulator PRIVATE ../../CLI) + +# Install into root dir (in Python module, enables 'import ogs.simulator') +install(TARGETS simulator LIBRARY DESTINATION .) diff --git a/Applications/Python/ogs_python_module.cpp b/Applications/Python/ogs.simulator/ogs_python_module.cpp similarity index 98% rename from Applications/Python/ogs_python_module.cpp rename to Applications/Python/ogs.simulator/ogs_python_module.cpp index c3e6f190a87..ab4103c7c65 100644 --- a/Applications/Python/ogs_python_module.cpp +++ b/Applications/Python/ogs.simulator/ogs_python_module.cpp @@ -122,8 +122,9 @@ void finalize() } /// python module name is OpenGeoSys -PYBIND11_MODULE(_cli, m) +PYBIND11_MODULE(simulator, m) { + m.attr("__name__") = "ogs.simulator"; m.doc() = "pybind11 ogs example plugin"; m.def("initialize", &initOGS, "init OGS"); m.def("currentTime", ¤tTime, "get current OGS time"); diff --git a/Applications/Python/OpenGeoSys/__init__.py b/Applications/Python/ogs/__init__.py similarity index 87% rename from Applications/Python/OpenGeoSys/__init__.py rename to Applications/Python/ogs/__init__.py index d4b704a8f3e..2aca7ae9d52 100644 --- a/Applications/Python/OpenGeoSys/__init__.py +++ b/Applications/Python/ogs/__init__.py @@ -69,23 +69,12 @@ if "PEP517_BUILD_BACKEND" not in os.environ: if platform.system() == "Windows": os.add_dll_directory(os.path.join(os.path.dirname(__file__), "bin")) - from .lib._cli import * - else: - try: - from .lib64._cli import * - except ImportError: - try: - from .lib._cli import * - except ImportError: - print("ERROR: could not import OpenGeoSys Python module!") OGS_BIN_DIR = os.path.join(os.path.join(os.path.dirname(__file__), "bin")) - def _program(name, args): return subprocess.call([os.path.join(OGS_BIN_DIR, name)] + args) - FUNC_TEMPLATE = """def {0}(): raise SystemExit(_program("{0}", sys.argv[1:]))""" for f in binaries_list: exec(FUNC_TEMPLATE.format(f)) diff --git a/Tests/Python/test_cli.py b/Tests/Python/test_cli.py index e2b60fc4fd4..52a50e2acb6 100644 --- a/Tests/Python/test_cli.py +++ b/Tests/Python/test_cli.py @@ -3,29 +3,13 @@ import pytest -import OpenGeoSys - - -def test_module(): - arguments = [ - "", - f"{os.path.abspath(os.path.dirname(__file__))}/../Data/Parabolic/LiquidFlow/Flux/cube_1e3_calculatesurfaceflux.prj", - "-o " + tempfile.mkdtemp(), - ] - - print("Python OpenGeoSys.init ...") - OpenGeoSys.initialize(arguments) - print("Python OpenGeoSys.executeSimulation ...") - OpenGeoSys.executeSimulation() - print("Python OpenGeoSys.finalize() ...") - OpenGeoSys.finalize() - +import ogs from . import push_argv def _run(program, args): - func = getattr(OpenGeoSys, program) + func = getattr(ogs, program) args = ["%s.py" % program] + args with push_argv(args), pytest.raises(SystemExit) as excinfo: func() @@ -34,6 +18,6 @@ def _run(program, args): def test_binaries(): ignore_list = ["moveMeshNodes", "mpmetis", "tetgen"] # have no --version cli flag - for f in OpenGeoSys.binaries_list: + for f in ogs.binaries_list: if f not in ignore_list: _run(f, ["--version"]) diff --git a/Tests/Python/test_simlator.py b/Tests/Python/test_simlator.py new file mode 100644 index 00000000000..bb41cad5e08 --- /dev/null +++ b/Tests/Python/test_simlator.py @@ -0,0 +1,20 @@ +import tempfile +import os + +import pytest +import ogs.simulator as sim + + +def test_simulator(): + arguments = [ + "", + f"{os.path.abspath(os.path.dirname(__file__))}/../Data/Parabolic/LiquidFlow/Flux/cube_1e3_calculatesurfaceflux.prj", + "-o " + tempfile.mkdtemp(), + ] + + print("Python OpenGeoSys.init ...") + sim.initialize(arguments) + print("Python OpenGeoSys.executeSimulation ...") + sim.executeSimulation() + print("Python OpenGeoSys.finalize() ...") + sim.finalize() diff --git a/pyproject.toml b/pyproject.toml index acb60b62466..5dc4a1ce709 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,13 +15,12 @@ norecursedirs = ["Tests/Data"] [tool.cibuildwheel] archs = "auto64" build = "cp3*" -skip = ["*musllinux*"] test-extras = "test" test-command = "pytest {project}/Tests/Python" -# test-command = "" build-verbosity = "1" [tool.cibuildwheel.linux] +skip = ["*musllinux*", "cp36-*"] manylinux-x86_64-image = "manylinux2014" manylinux-aarch64-image = "manylinux2014" environment-pass = ["OGS_VERSION"] diff --git a/scripts/cmake/ProjectSetup.cmake b/scripts/cmake/ProjectSetup.cmake index 597971cecf2..16bd5894c4d 100644 --- a/scripts/cmake/ProjectSetup.cmake +++ b/scripts/cmake/ProjectSetup.cmake @@ -40,8 +40,12 @@ endif() file(RELATIVE_PATH relDir ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR} ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR} ) -list(APPEND CMAKE_INSTALL_RPATH ${BASEPOINT} ${BASEPOINT}/${relDir}) -list(APPEND CMAKE_BUILD_RPATH ${BASEPOINT} ${BASEPOINT}/${relDir}) +list(APPEND CMAKE_INSTALL_RPATH ${BASEPOINT} ${BASEPOINT}/${relDir} + ${BASEPOINT}/${CMAKE_INSTALL_LIBDIR} # Python modules +) +list(APPEND CMAKE_BUILD_RPATH ${BASEPOINT} ${BASEPOINT}/${relDir} + ${BASEPOINT}/${CMAKE_INSTALL_LIBDIR} # Python modules +) # Some external dependencies always use lib instead of lib64, Fix for # lib64-based systems, e.g. OpenSUSE: diff --git a/setup.py b/setup.py index e5a8c44f2da..93a30713680 100644 --- a/setup.py +++ b/setup.py @@ -2,6 +2,7 @@ from setuptools import find_packages import os +import platform import re import subprocess import sys @@ -12,8 +13,11 @@ def get_version(): if "OGS_VERSION" in os.environ: git_version = os.environ["OGS_VERSION"] else: + git_describe_cmd = ["git describe --tags"] + if platform.system() == "Windows": + git_describe_cmd = ["git", "describe", "--tags"] git_version = subprocess.run( - ["git", "describe", "--tags"], + git_describe_cmd, capture_output=True, text=True, shell=True, @@ -33,28 +37,27 @@ def get_version(): sys.path.append(os.path.join("Applications", "Python")) -from OpenGeoSys import binaries_list +from ogs import binaries_list console_scripts = [] for b in binaries_list: - console_scripts.append(f"{b}=OpenGeoSys:{b}") + console_scripts.append(f"{b}=ogs:{b}") -import platform cmake_preset = "wheel" if platform.system() == "Windows": cmake_preset += "-win" setup( - name="OpenGeoSys", + name="ogs", version=get_version(), - description="OpenGeoSys", + description="OpenGeoSys Python Module", author="OpenGeoSys Community", license="BSD-3-Clause", packages=find_packages(where="Applications/Python"), package_dir={"": "Applications/Python"}, - cmake_install_dir="Applications/Python/OpenGeoSys", + cmake_install_dir="Applications/Python/ogs", extras_require={"test": ["pytest"]}, cmake_args=[f"--preset {cmake_preset}", "-B ."], - python_requires=">=3.6", + python_requires=">=3.7", entry_points={"console_scripts": console_scripts}, ) From 3469b061d5e3022ef09093b4ba3b656e8073bfb3 Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Tue, 13 Sep 2022 14:57:02 +0200 Subject: [PATCH 22/36] [ci] Upload release wheels. --- scripts/ci/jobs/release.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/scripts/ci/jobs/release.yml b/scripts/ci/jobs/release.yml index 81bba47813a..bb2581ec1ca 100644 --- a/scripts/ci/jobs/release.yml +++ b/scripts/ci/jobs/release.yml @@ -9,3 +9,24 @@ release: release: tag_name: "$CI_COMMIT_TAG" description: "Created using the GitLab release-cli." + +publish wheels: + stage: release + needs: ["build wheels linux", "build wheels mac", "build wheels win"] + tags: [envinf, shell] + rules: + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + variables: + PYPI_PASSWORD: "${TEST_PYPI_TOKEN}" + PYPI_REPO: testpypi + - if: $CI_COMMIT_TAG + variables: + PYPI_PASSWORD: "${PYPI_TOKEN}" + PYPI_REPO: pypi + script: + - > + pipx run twine upload + --repository ${PYPI_REPO} + --username __token__ + --password ${PYPI_PASSWORD} + wheelhouse/* From a563fa8d3121be89f5bb2541788a27e6387fc333 Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Tue, 13 Sep 2022 15:37:15 +0200 Subject: [PATCH 23/36] [wheel] Enable all processes. --- CMakePresets.json | 1 - 1 file changed, 1 deletion(-) diff --git a/CMakePresets.json b/CMakePresets.json index 3f2871df967..583bde18d31 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -192,7 +192,6 @@ "name": "wheel", "inherits": "release", "cacheVariables": { - "OGS_BUILD_PROCESSES": "LiquidFlow", "HDF5_USE_STATIC_LIBRARIES": "ON", "OGS_BUILD_HDF5": "ON", "OGS_USE_PYTHON": "OFF", From c286aa7931ce32935f51a7ee1c015370e4754ca2 Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Tue, 13 Sep 2022 22:26:44 +0200 Subject: [PATCH 24/36] [ML] Fixed linking issue under macOS when BUILD_SHARED_LIBS=ON. Linker error was: Undefined symbols for architecture arm64: "MaterialLib::Solids::Ehlers::StateVariables<2>::getEquivalentPlasticStrain() const", referenced from: vtable for MaterialLib::Solids::Ehlers::StateVariables<2> in unity_0_cxx.cxx.o "MaterialLib::Solids::Ehlers::StateVariables<3>::getEquivalentPlasticStrain() const", referenced from: vtable for MaterialLib::Solids::Ehlers::StateVariables<3> in unity_0_cxx.cxx.o --- MaterialLib/SolidModels/Ehlers.cpp | 3 +++ MaterialLib/SolidModels/Ehlers.h | 10 ++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/MaterialLib/SolidModels/Ehlers.cpp b/MaterialLib/SolidModels/Ehlers.cpp index 28adfbd6cae..83995b28955 100644 --- a/MaterialLib/SolidModels/Ehlers.cpp +++ b/MaterialLib/SolidModels/Ehlers.cpp @@ -842,6 +842,9 @@ SolidEhlers::getInternalVariables() const template class SolidEhlers<2>; template class SolidEhlers<3>; +template struct StateVariables<2>; +template struct StateVariables<3>; + template <> MathLib::KelvinVector::KelvinMatrixType<3> sOdotS<3>( MathLib::KelvinVector::KelvinVectorType<3> const& v) diff --git a/MaterialLib/SolidModels/Ehlers.h b/MaterialLib/SolidModels/Ehlers.h index 9c9d3ec2bfb..d984878cf13 100644 --- a/MaterialLib/SolidModels/Ehlers.h +++ b/MaterialLib/SolidModels/Ehlers.h @@ -24,11 +24,10 @@ #include "BaseLib/Error.h" #include "MathLib/KelvinVector.h" +#include "MechanicsBase.h" #include "NumLib/NewtonRaphson.h" #include "ParameterLib/Parameter.h" -#include "MechanicsBase.h" - namespace MaterialLib { namespace Solids @@ -250,8 +249,8 @@ struct StateVariables Damage damage_prev; ///< \copydoc damage #ifndef NDEBUG - friend std::ostream& operator<<( - std::ostream& os, StateVariables const& m) + friend std::ostream& operator<<(std::ostream& os, + StateVariables const& m) { os << "State:\n" << "eps_p_D: " << m.eps_p.D << "\n" @@ -358,6 +357,9 @@ class SolidEhlers final : public MechanicsBase extern template class SolidEhlers<2>; extern template class SolidEhlers<3>; + +extern template struct StateVariables<2>; +extern template struct StateVariables<3>; } // namespace Ehlers } // namespace Solids } // namespace MaterialLib From 81c35a6fd128eee0ee196de9e3292b5e9e0dcba2 Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Tue, 13 Sep 2022 22:35:49 +0200 Subject: [PATCH 25/36] [wheel] Include README. --- setup.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/setup.py b/setup.py index 93a30713680..a9cd2b4b9f7 100644 --- a/setup.py +++ b/setup.py @@ -47,10 +47,17 @@ def get_version(): if platform.system() == "Windows": cmake_preset += "-win" +from pathlib import Path + +this_directory = Path(__file__).parent +long_description = (this_directory / "README.md").read_text() + setup( name="ogs", version=get_version(), description="OpenGeoSys Python Module", + long_description=long_description, + long_description_content_type="text/markdown", author="OpenGeoSys Community", license="BSD-3-Clause", packages=find_packages(where="Applications/Python"), From f9c6e4e17e2407c021885a6b6d8a611a5cc6aadb Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Wed, 14 Sep 2022 12:56:58 +0200 Subject: [PATCH 26/36] [ci] Refactor wheel jobs to use template. --- scripts/ci/jobs/build-wheels.yml | 53 +++++++++----------------------- 1 file changed, 15 insertions(+), 38 deletions(-) diff --git a/scripts/ci/jobs/build-wheels.yml b/scripts/ci/jobs/build-wheels.yml index 213db165ae9..b43eb0402b7 100644 --- a/scripts/ci/jobs/build-wheels.yml +++ b/scripts/ci/jobs/build-wheels.yml @@ -1,55 +1,32 @@ -build wheels linux: +.wheels_template: &wheels_template stage: build needs: [meta] - tags: [envinf3-shell] script: - pipx run cibuildwheel - - > - pipx run twine upload - --repository-url https://gitlab.opengeosys.org/api/v4/projects/${CI_PROJECT_ID}/packages/pypi - --username gitlab-ci-token - --password ${CI_JOB_TOKEN} - wheelhouse/* + rules: + - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + - if: $CI_COMMIT_TAG artifacts: paths: - wheelhouse/ +build wheels linux: + tags: [envinf3-shell] + <<: *wheels_template + build wheels mac: - stage: build - needs: [meta] tags: - mac - ${ARCHITECTURE} - script: - - CMAKE_OSX_DEPLOYMENT_TARGET=10.15 pipx run cibuildwheel - - > - pipx run twine upload - --repository-url https://gitlab.opengeosys.org/api/v4/projects/${CI_PROJECT_ID}/packages/pypi - --username gitlab-ci-token - --password ${CI_JOB_TOKEN} - wheelhouse/* - artifacts: - paths: - - wheelhouse/ + variables: + CMAKE_OSX_DEPLOYMENT_TARGET: "10.15" parallel: matrix: - ARCHITECTURE: ["amd64", "arm64"] + <<: *wheels_template build wheels win: - stage: build - needs: [meta] - tags: - - windows - extends: - - .vs2019-environment - script: - - pipx run cibuildwheel - - > - pipx run twine upload - --repository-url https://gitlab.opengeosys.org/api/v4/projects/${CI_PROJECT_ID}/packages/pypi - --username gitlab-ci-token - --password ${CI_JOB_TOKEN} - wheelhouse/* - artifacts: - paths: - - wheelhouse/ + tags: [windows] + extends: + - .vs2019-environment + <<: *wheels_template From dd0afe59f0172629b0856745f6702de91f810bb2 Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Wed, 14 Sep 2022 20:17:06 +0200 Subject: [PATCH 27/36] [ci] Run jupyter job on envinf2 and envinf3. --- scripts/ci/jobs/jupyter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/jobs/jupyter.yml b/scripts/ci/jobs/jupyter.yml index 875e30ff859..039cd85185c 100644 --- a/scripts/ci/jobs/jupyter.yml +++ b/scripts/ci/jobs/jupyter.yml @@ -1,7 +1,7 @@ # Built for Sandy Bridge (envinf1) and newer build jupyter: stage: build - tags: [envinf, shell] + tags: [envinf23, shell] needs: [meta] extends: - .container-maker-setup From a06d04465eece7fde3c61867e433f25a8213dc83 Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Wed, 14 Sep 2022 20:17:08 +0200 Subject: [PATCH 28/36] [ci] Run clang-format only on .h and .cpp files. --- scripts/ci/jobs/pre-commit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci/jobs/pre-commit.yml b/scripts/ci/jobs/pre-commit.yml index 292e826aeab..10672766a16 100644 --- a/scripts/ci/jobs/pre-commit.yml +++ b/scripts/ci/jobs/pre-commit.yml @@ -20,7 +20,7 @@ clang-format: needs: [ci_images] allow_failure: true script: - - git clang-format ${CI_MERGE_REQUEST_TARGET_BRANCH_SHA} + - git clang-format --extensions "h,cpp" ${CI_MERGE_REQUEST_TARGET_BRANCH_SHA} - if [[ $(git diff) ]]; then exit 1; fi after_script: - git diff From d65f121aed9079fd6d91e3656b6a7e2474e04ad9 Mon Sep 17 00:00:00 2001 From: GitLab Runner Date: Wed, 14 Sep 2022 20:44:20 +0200 Subject: [PATCH 29/36] [versions] Updated cpm cache info. --- web/data/versions.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/data/versions.json b/web/data/versions.json index d064e9cdf8d..45f0d34dee4 100644 --- a/web/data/versions.json +++ b/web/data/versions.json @@ -71,8 +71,8 @@ } }, "cpm": { - "package_file_id": 119, - "package_file_sha256": "7d98b148e6d24acd72d17d2503f3ce3bab74029da5b3328788b2d4c379e9dcac" + "package_file_id": 182, + "package_file_sha256": "00d7dea24754ad415e7003535b36a7d5b4e7224701341f5ca587f93e42b63563" }, "ext": { "cache_hash": "e6f3f1f4c29c6c5f096f89785e6e245bdf39ac1a" From eec595dec3e655566c236ecb28c32a8f225dae27 Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Thu, 15 Sep 2022 09:05:01 +0200 Subject: [PATCH 30/36] [wheel] Removed unnecessary else statements. --- setup.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index a9cd2b4b9f7..e607d50f4a5 100644 --- a/setup.py +++ b/setup.py @@ -29,11 +29,10 @@ def get_version(): m = re.match(".+?(?=-g[\w]*$)", git_version) # strip out commit hash if m: return m.group(0).replace("-", ".dev") # insert dev - else: - print("WARNING: Could not get ogs version!") - exit(1) - else: - return git_version + print("ERROR: Could not get ogs version!") + exit(1) + + return git_version sys.path.append(os.path.join("Applications", "Python")) From b28f26aefec5529cb7e37b38de3212c2d665a486 Mon Sep 17 00:00:00 2001 From: Lars Bilke Date: Thu, 15 Sep 2022 17:41:11 +0200 Subject: [PATCH 31/36] [web] Added docs on pip install usage and wheel development. --- .../devguide/advanced/python-wheel/index.md | 75 +++++++++++++++++++ .../userguide/basics/introduction/index.md | 52 ++++++++++++- 2 files changed, 124 insertions(+), 3 deletions(-) create mode 100644 web/content/docs/devguide/advanced/python-wheel/index.md diff --git a/web/content/docs/devguide/advanced/python-wheel/index.md b/web/content/docs/devguide/advanced/python-wheel/index.md new file mode 100644 index 00000000000..91154b43ab4 --- /dev/null +++ b/web/content/docs/devguide/advanced/python-wheel/index.md @@ -0,0 +1,75 @@ ++++ +date = "2022-02-09T11:00:13+01:00" +title = "Python wheel development" +author = "Lars Bilke" +weight = 1068 + +[menu] + [menu.devguide] + parent = "advanced" ++++ + +## Local setup + +Python wheel builds are driven by [scikit-build](https://scikit-build.readthedocs.io/en/latest/) which basically is a `setuptools`-wrapper for CMake-based projects. + +The entrypoint is `setup.py` in the root directory. It uses the `wheel` CMake preset (or `wheel-win` on Windows). + +You can locally develop and test with the following setup: + +```bash +# Create a virtual environment inside your source directory +python3 -m venv .venv +# Activate the environment +source .venv/bin/activate +# Install (build) the local Python project +pip install -v .[test] +... +Successfully installed ogs-6.4.2.dev1207 +``` + +The `pip install`-step starts a new CMake-based ogs build in `_skbuild`-subdirectory (inside the source code) using the `wheel`-preset. When the CMake build is done it installs the wheel into the activated virtual environment and you can interact with it, e.g.: + +```bash +# Run python tests +pytest +============================================== test session starts =============================================== +platform darwin -- Python 3.10.6, pytest-7.1.3, pluggy-1.0.0 +rootdir: ~/code/ogs/ogs, configfile: pyproject.toml, testpaths: Tests +collected 2 items + +Tests/Python/test_cli.py . [ 50%] +Tests/Python/test_simlator.py . [100%] + +=============================================== 2 passed in 0.55s ================================================ + +# Start the python interpreter +python3 +>>> import ogs.simulator as sim +>>> sim.initialize(["", "--help"]) +``` + +If you make modifications you need to run `pip install .[test]` again (or for temporary modifications you can directly edit inside the virtual environment, e.g. in `.venv/lib/python3.10/site-packages/ogs`). + +The contents of `_skbuild/[platform-specific]/cmake-install` will make up the wheel. + +## CI + +For generating the various wheels for different Python versions and platforms [`cibuildwheel`](https://cibuildwheel.readthedocs.io/en/stable/) is used. + +You can test it locally with, e.g. only building for Python 3.10: + +```bash +CIBW_BUILD="cp310*" pipx run cibuildwheel +``` + +Please note that on Linux `cibuildwheel` runs the builds inside [manylinux](https://github.com/pypa/manylinux) Docker containers. On other platforms the build happens with native tools. See the [cibuildwheel docs](https://cibuildwheel.readthedocs.io/en/stable/#how-it-works) for more information. + +Wheels are generated in the `wheelhouse/`-folder. + +`cibuildwheel` is configured in `pyproject.toml`: + +```toml +[tool.cibuildwheel] +... +``` diff --git a/web/content/docs/userguide/basics/introduction/index.md b/web/content/docs/userguide/basics/introduction/index.md index 208d6437359..d808b4c07ce 100644 --- a/web/content/docs/userguide/basics/introduction/index.md +++ b/web/content/docs/userguide/basics/introduction/index.md @@ -15,14 +15,60 @@ weight = 1 post = "Download, install and run an OGS benchmark in 5 minutes! No development setup required." +++ -## Download +## Installation -Download the latest release of OpenGeoSys from the [Releases](/releases)-page. Be sure to pick the correct file for your operating system. +
-## Installation +Download the latest release of OpenGeoSys from the [Releases](/releases)-page. Be sure to pick the correct file for your operating system. OGS itself is a simple executable file so you can put it anywhere you like. For convenience you may put into a location which is in your `PATH`-environment variable which allows you to start the executable without specifying its full file path. +
+ +### Alternative: Install via `pip` + +You can also install ogs via Python's [`pip`-tool](https://packaging.python.org/en/latest/tutorials/installing-packages/): + +```bat +pip install ogs +``` + +If you install into an activated [virtual environment](https://docs.python.org/3/library/venv.html) then ogs and its tools are automatically also in the `PATH`. Otherwise `pip` will print instructions which directory needs to be added to the `PATH`. + +
+ +
+ +
+ +Install via Python's [`pip`-tool](https://packaging.python.org/en/latest/tutorials/installing-packages/): + +```bash +pip install ogs +``` + +You may want to set up and activate a [virtual environment](https://docs.python.org/3/library/venv.html) before. + +You could also use [`pipx`](https://pypa.github.io/pipx/) to install into an isolated environment. + +
+ +
+ +See Linux tab! + +
+ +
+ +### Limitations of the `pip`-based installation + +- Serial config only! For PETSc-support please use a [Singularity container]({{< relref "container" >}}). +- No embedded Python interpreter, i.e. no Python boundary conditions! +- A Python (3.8 - 3.11) installation with `pip` is required. + +
+ ## Download benchmarks You can download the latest benchmark files from GitLab: From ce05ebb88786da9b2a737c89ba16e23a5b25f346 Mon Sep 17 00:00:00 2001 From: Christoph Lehmann Date: Fri, 16 Sep 2022 07:48:53 +0200 Subject: [PATCH 32/36] [App] Moved Python code for CLI tools --- Applications/Python/ogs/__init__.py | 80 ------------------- Applications/Python/ogs/_internal/__init__.py | 0 .../provide_ogs_cli_tools_via_wheel.py | 80 +++++++++++++++++++ 3 files changed, 80 insertions(+), 80 deletions(-) create mode 100644 Applications/Python/ogs/_internal/__init__.py create mode 100644 Applications/Python/ogs/_internal/provide_ogs_cli_tools_via_wheel.py diff --git a/Applications/Python/ogs/__init__.py b/Applications/Python/ogs/__init__.py index 2aca7ae9d52..e69de29bb2d 100644 --- a/Applications/Python/ogs/__init__.py +++ b/Applications/Python/ogs/__init__.py @@ -1,80 +0,0 @@ -import os -import platform -import subprocess -import sys - -binaries_list = [ - "addDataToRaster", - "AddElementQuality", - "AddFaultToVoxelGrid", - "AddLayer", - "appendLinesAlongPolyline", - "AssignRasterDataToMesh", - "checkMesh", - "ComputeNodeAreasFromSurfaceMesh", - "computeSurfaceNodeIDsInPolygonalRegion", - "constructMeshesFromGeometry", - "convertGEO", - "convertToLinearMesh", - "convertVtkDataArrayToVtkDataArray", - "CreateBoundaryConditionsAlongPolylines", - "createIntermediateRasters", - "createLayeredMeshFromRasters", - "createMeshElemPropertiesFromASCRaster", - "createNeumannBc", - "createQuadraticMesh", - "createRaster", - "editMaterialID", - "ExtractBoundary", - "ExtractMaterials", - "ExtractSurface", - "generateGeometry", - "generateMatPropsFromMatID", - "generateStructuredMesh", - "geometryToGmshGeo", - "GMSH2OGS", - "GocadSGridReader", - "GocadTSurfaceReader", - "identifySubdomains", - "IntegrateBoreholesIntoMesh", - "Layers2Grid", - "MapGeometryToMeshSurface", - "Mesh2Raster", - "MoveGeometry", - "MoveMesh", - "moveMeshNodes", - "mpmetis", - "NodeReordering", - "ogs", - "OGS2VTK", - "partmesh", - "PVD2XDMF", - "queryMesh", - "Raster2Mesh", - "RemoveGhostData", - "removeMeshElements", - "ResetPropertiesInPolygonalRegion", - "reviseMesh", - "scaleProperty", - "swapNodeCoordinateAxes", - "TecPlotTools", - "tetgen", - "TIN2VTK", - "VTK2OGS", - "VTK2TIN", - "vtkdiff", - "Vtu2Grid", -] - -if "PEP517_BUILD_BACKEND" not in os.environ: - if platform.system() == "Windows": - os.add_dll_directory(os.path.join(os.path.dirname(__file__), "bin")) - - OGS_BIN_DIR = os.path.join(os.path.join(os.path.dirname(__file__), "bin")) - - def _program(name, args): - return subprocess.call([os.path.join(OGS_BIN_DIR, name)] + args) - - FUNC_TEMPLATE = """def {0}(): raise SystemExit(_program("{0}", sys.argv[1:]))""" - for f in binaries_list: - exec(FUNC_TEMPLATE.format(f)) diff --git a/Applications/Python/ogs/_internal/__init__.py b/Applications/Python/ogs/_internal/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Applications/Python/ogs/_internal/provide_ogs_cli_tools_via_wheel.py b/Applications/Python/ogs/_internal/provide_ogs_cli_tools_via_wheel.py new file mode 100644 index 00000000000..2aca7ae9d52 --- /dev/null +++ b/Applications/Python/ogs/_internal/provide_ogs_cli_tools_via_wheel.py @@ -0,0 +1,80 @@ +import os +import platform +import subprocess +import sys + +binaries_list = [ + "addDataToRaster", + "AddElementQuality", + "AddFaultToVoxelGrid", + "AddLayer", + "appendLinesAlongPolyline", + "AssignRasterDataToMesh", + "checkMesh", + "ComputeNodeAreasFromSurfaceMesh", + "computeSurfaceNodeIDsInPolygonalRegion", + "constructMeshesFromGeometry", + "convertGEO", + "convertToLinearMesh", + "convertVtkDataArrayToVtkDataArray", + "CreateBoundaryConditionsAlongPolylines", + "createIntermediateRasters", + "createLayeredMeshFromRasters", + "createMeshElemPropertiesFromASCRaster", + "createNeumannBc", + "createQuadraticMesh", + "createRaster", + "editMaterialID", + "ExtractBoundary", + "ExtractMaterials", + "ExtractSurface", + "generateGeometry", + "generateMatPropsFromMatID", + "generateStructuredMesh", + "geometryToGmshGeo", + "GMSH2OGS", + "GocadSGridReader", + "GocadTSurfaceReader", + "identifySubdomains", + "IntegrateBoreholesIntoMesh", + "Layers2Grid", + "MapGeometryToMeshSurface", + "Mesh2Raster", + "MoveGeometry", + "MoveMesh", + "moveMeshNodes", + "mpmetis", + "NodeReordering", + "ogs", + "OGS2VTK", + "partmesh", + "PVD2XDMF", + "queryMesh", + "Raster2Mesh", + "RemoveGhostData", + "removeMeshElements", + "ResetPropertiesInPolygonalRegion", + "reviseMesh", + "scaleProperty", + "swapNodeCoordinateAxes", + "TecPlotTools", + "tetgen", + "TIN2VTK", + "VTK2OGS", + "VTK2TIN", + "vtkdiff", + "Vtu2Grid", +] + +if "PEP517_BUILD_BACKEND" not in os.environ: + if platform.system() == "Windows": + os.add_dll_directory(os.path.join(os.path.dirname(__file__), "bin")) + + OGS_BIN_DIR = os.path.join(os.path.join(os.path.dirname(__file__), "bin")) + + def _program(name, args): + return subprocess.call([os.path.join(OGS_BIN_DIR, name)] + args) + + FUNC_TEMPLATE = """def {0}(): raise SystemExit(_program("{0}", sys.argv[1:]))""" + for f in binaries_list: + exec(FUNC_TEMPLATE.format(f)) From ccf3ea0bb124bbf303bec82b17f3c3c0c308b2a7 Mon Sep 17 00:00:00 2001 From: Christoph Lehmann Date: Fri, 16 Sep 2022 07:50:36 +0200 Subject: [PATCH 33/36] [App] Use recommended subprocess method --- .../Python/ogs/_internal/provide_ogs_cli_tools_via_wheel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Applications/Python/ogs/_internal/provide_ogs_cli_tools_via_wheel.py b/Applications/Python/ogs/_internal/provide_ogs_cli_tools_via_wheel.py index 2aca7ae9d52..6399fd881b7 100644 --- a/Applications/Python/ogs/_internal/provide_ogs_cli_tools_via_wheel.py +++ b/Applications/Python/ogs/_internal/provide_ogs_cli_tools_via_wheel.py @@ -73,7 +73,7 @@ OGS_BIN_DIR = os.path.join(os.path.join(os.path.dirname(__file__), "bin")) def _program(name, args): - return subprocess.call([os.path.join(OGS_BIN_DIR, name)] + args) + return subprocess.run([os.path.join(OGS_BIN_DIR, name)] + args).returncode FUNC_TEMPLATE = """def {0}(): raise SystemExit(_program("{0}", sys.argv[1:]))""" for f in binaries_list: From f86edadd20be2ebbb466450faee14eaab4998c32 Mon Sep 17 00:00:00 2001 From: Christoph Lehmann Date: Fri, 16 Sep 2022 10:56:10 +0200 Subject: [PATCH 34/36] Reflect moved Python code in setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e607d50f4a5..177b162c830 100644 --- a/setup.py +++ b/setup.py @@ -36,11 +36,11 @@ def get_version(): sys.path.append(os.path.join("Applications", "Python")) -from ogs import binaries_list +from ogs._internal.provide_ogs_cli_tools_via_wheel import binaries_list console_scripts = [] for b in binaries_list: - console_scripts.append(f"{b}=ogs:{b}") + console_scripts.append(f"{b}=ogs._internal.provide_ogs_cli_tools_via_wheel:{b}") cmake_preset = "wheel" if platform.system() == "Windows": From fb2d813a6c01f9ef702b129052da215b1c7ac52f Mon Sep 17 00:00:00 2001 From: Christoph Lehmann Date: Fri, 16 Sep 2022 13:01:28 +0200 Subject: [PATCH 35/36] [T] Adjusted Python CLI tool test. --- Tests/Python/test_cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tests/Python/test_cli.py b/Tests/Python/test_cli.py index 52a50e2acb6..c9d23bb8d79 100644 --- a/Tests/Python/test_cli.py +++ b/Tests/Python/test_cli.py @@ -3,13 +3,13 @@ import pytest -import ogs +import ogs._internal.provide_ogs_cli_tools_via_wheel as ogs_cli_wheel from . import push_argv def _run(program, args): - func = getattr(ogs, program) + func = getattr(ogs_cli_wheel, program) args = ["%s.py" % program] + args with push_argv(args), pytest.raises(SystemExit) as excinfo: func() @@ -18,6 +18,6 @@ def _run(program, args): def test_binaries(): ignore_list = ["moveMeshNodes", "mpmetis", "tetgen"] # have no --version cli flag - for f in ogs.binaries_list: + for f in ogs_cli_wheel.binaries_list: if f not in ignore_list: _run(f, ["--version"]) From dfd918f52122657424d0664e38c856e98fa0b53b Mon Sep 17 00:00:00 2001 From: Christoph Lehmann Date: Mon, 19 Sep 2022 08:10:39 +0200 Subject: [PATCH 36/36] [App] Corrected OGS_BIN_DIR --- .../Python/ogs/_internal/provide_ogs_cli_tools_via_wheel.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Applications/Python/ogs/_internal/provide_ogs_cli_tools_via_wheel.py b/Applications/Python/ogs/_internal/provide_ogs_cli_tools_via_wheel.py index 6399fd881b7..ea18d343753 100644 --- a/Applications/Python/ogs/_internal/provide_ogs_cli_tools_via_wheel.py +++ b/Applications/Python/ogs/_internal/provide_ogs_cli_tools_via_wheel.py @@ -67,10 +67,10 @@ ] if "PEP517_BUILD_BACKEND" not in os.environ: - if platform.system() == "Windows": - os.add_dll_directory(os.path.join(os.path.dirname(__file__), "bin")) + OGS_BIN_DIR = os.path.join(os.path.join(os.path.dirname(__file__), "..", "bin")) - OGS_BIN_DIR = os.path.join(os.path.join(os.path.dirname(__file__), "bin")) + if platform.system() == "Windows": + os.add_dll_directory(OGS_BIN_DIR) def _program(name, args): return subprocess.run([os.path.join(OGS_BIN_DIR, name)] + args).returncode