From d05f2c44e1f05db5cb25523e278e9def2cb757c8 Mon Sep 17 00:00:00 2001 From: Zohar Malamant Date: Tue, 3 Nov 2020 18:58:19 +0100 Subject: [PATCH] pip-installable libres --- .github/workflows/python-bdist.yml | 135 ++++++++++++ .github/workflows/testing.yml | 193 +++++++++++------- .gitignore | 5 + .libecl_version | 1 - CMakeLists.txt | 185 ++++++----------- Jenkinsfile | 7 +- README.md | 145 ++++++------- ci/github/build_linux_wheel.sh | 21 ++ ci/jenkins/testkomodo.sh | 26 +-- lib/CMakeLists.txt | 131 +++++------- lib/analysis/analysis_module.cpp | 61 ++++-- lib/enkf/site_config.cpp | 20 +- pyproject.toml | 2 + python/CMakeLists.txt | 11 - python/cmake/Modules/add_python_package.cmake | 42 ---- python/cmake/Modules/add_python_test.cmake | 38 ---- python/cmake/Modules/init_python.cmake | 187 ----------------- python/ert_logger/CMakeLists.txt | 5 - python/job_runner/CMakeLists.txt | 12 -- python/job_runner/io/CMakeLists.txt | 6 - python/job_runner/job_dispatch.py | 20 ++ python/job_runner/reporting/CMakeLists.txt | 10 - python/job_runner/util/CMakeLists.txt | 5 - python/res/CMakeLists.txt | 23 --- python/res/__init__.py | 91 ++++----- python/res/analysis/CMakeLists.txt | 9 - python/res/analysis/analysis_module.py | 4 + python/res/analysis/enums/CMakeLists.txt | 10 - python/res/config/CMakeLists.txt | 14 -- python/res/enkf/CMakeLists.txt | 60 ------ python/res/enkf/config/CMakeLists.txt | 13 -- python/res/enkf/data/CMakeLists.txt | 12 -- python/res/enkf/enums/CMakeLists.txt | 18 -- python/res/enkf/export/CMakeLists.txt | 14 -- python/res/enkf/observations/CMakeLists.txt | 11 - python/res/enkf/plot/CMakeLists.txt | 15 -- python/res/enkf/plot_data/CMakeLists.txt | 15 -- python/res/enkf/util/CMakeLists.txt | 7 - python/res/fm/CMakeLists.txt | 12 -- python/res/fm/ecl/CMakeLists.txt | 18 -- python/res/fm/rms/CMakeLists.txt | 9 - python/res/fm/shell/CMakeLists.txt | 7 - python/res/fm/templating/CMakeLists.txt | 6 - python/res/job_queue/CMakeLists.txt | 26 --- python/res/res_lib_info_build.py.in | 8 - python/res/res_lib_info_install.py.in | 8 - python/res/sched/CMakeLists.txt | 8 - python/res/simulator/CMakeLists.txt | 7 - python/res/test/CMakeLists.txt | 7 - python/res/test/synthesizer/CMakeLists.txt | 10 - python/res/util/CMakeLists.txt | 16 -- python/res/util/enums/CMakeLists.txt | 8 - python/res/util/stat.py | 1 + python/tests/global/__init__.py | 0 python/tests/global/test_import.py | 26 --- python/tests/job_runner/test_job_dispatch.py | 84 ++++---- .../res/analysis/test_analysis_module.py | 3 +- python/tests/res/config/test_config.py | 10 +- python/tests/res/fm/test_rms_config.py | 5 +- python/tests/res/fm/test_rms_run.py | 4 - .../tests/res/job_queue/test_workflow_job.py | 11 +- python/tests/res/util/test_res_log.py | 18 +- python/tests/share/CMakeLists.txt | 6 - requirements.txt | 23 ++- setup.py | 92 +++++++++ share/ert/site-config | 2 +- testjenkins.sh | 82 ++++---- 67 files changed, 790 insertions(+), 1311 deletions(-) create mode 100644 .github/workflows/python-bdist.yml delete mode 100644 .libecl_version create mode 100755 ci/github/build_linux_wheel.sh create mode 100644 pyproject.toml delete mode 100644 python/CMakeLists.txt delete mode 100644 python/cmake/Modules/add_python_package.cmake delete mode 100644 python/cmake/Modules/add_python_test.cmake delete mode 100644 python/cmake/Modules/init_python.cmake delete mode 100644 python/ert_logger/CMakeLists.txt delete mode 100644 python/job_runner/CMakeLists.txt delete mode 100644 python/job_runner/io/CMakeLists.txt create mode 100644 python/job_runner/job_dispatch.py delete mode 100644 python/job_runner/reporting/CMakeLists.txt delete mode 100644 python/job_runner/util/CMakeLists.txt delete mode 100644 python/res/CMakeLists.txt delete mode 100644 python/res/analysis/CMakeLists.txt delete mode 100644 python/res/analysis/enums/CMakeLists.txt delete mode 100644 python/res/config/CMakeLists.txt delete mode 100644 python/res/enkf/CMakeLists.txt delete mode 100644 python/res/enkf/config/CMakeLists.txt delete mode 100644 python/res/enkf/data/CMakeLists.txt delete mode 100644 python/res/enkf/enums/CMakeLists.txt delete mode 100644 python/res/enkf/export/CMakeLists.txt delete mode 100644 python/res/enkf/observations/CMakeLists.txt delete mode 100644 python/res/enkf/plot/CMakeLists.txt delete mode 100644 python/res/enkf/plot_data/CMakeLists.txt delete mode 100644 python/res/enkf/util/CMakeLists.txt delete mode 100644 python/res/fm/CMakeLists.txt delete mode 100644 python/res/fm/ecl/CMakeLists.txt delete mode 100644 python/res/fm/rms/CMakeLists.txt delete mode 100644 python/res/fm/shell/CMakeLists.txt delete mode 100644 python/res/fm/templating/CMakeLists.txt delete mode 100644 python/res/job_queue/CMakeLists.txt delete mode 100644 python/res/res_lib_info_build.py.in delete mode 100644 python/res/res_lib_info_install.py.in delete mode 100644 python/res/sched/CMakeLists.txt delete mode 100644 python/res/simulator/CMakeLists.txt delete mode 100644 python/res/test/CMakeLists.txt delete mode 100644 python/res/test/synthesizer/CMakeLists.txt delete mode 100644 python/res/util/CMakeLists.txt delete mode 100644 python/res/util/enums/CMakeLists.txt delete mode 100644 python/tests/global/__init__.py delete mode 100644 python/tests/global/test_import.py delete mode 100644 python/tests/share/CMakeLists.txt create mode 100644 setup.py diff --git a/.github/workflows/python-bdist.yml b/.github/workflows/python-bdist.yml new file mode 100644 index 0000000000..ffb1709d42 --- /dev/null +++ b/.github/workflows/python-bdist.yml @@ -0,0 +1,135 @@ +name: Python Binary Distribution + +on: [push] + +jobs: + build-test-cmake: + name: CMake + + strategy: + fail-fast: false + matrix: + os: ['ubuntu-latest', 'macos-latest'] + + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + + - name: Install Ubuntu dependencies + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt-get update -y + sudo apt-get install -y valgrind + + - name: Build libecl + run: | + git clone https://github.com/equinor/libecl + mkdir libecl/build + cmake -S libecl -B libecl/build + sudo cmake --build libecl/build --target install + sudo rm -rf libecl + + - name: Build libres + run: | + mkdir cmake-build + cmake -S . -B cmake-build \ + -DBUILD_TESTS=ON \ + -DRES_VERSION=1.2.3 \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo + cmake --build cmake-build + + - name: Run tests + run: | + cd cmake-build + export PATH=$PWD/bin:$PATH + ctest --output-on-failure + + + build-test-wheel: + name: Python + + strategy: + fail-fast: false + matrix: + os: ['ubuntu-latest', 'macos-latest'] + python: ['3.6', '3.7', '3.8'] + + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + + - name: Build Linux Wheel + uses: docker://quay.io/pypa/manylinux2010_x86_64 + with: + entrypoint: /github/workspace/ci/github/build_linux_wheel.sh + args: ${{ matrix.python }} + if: matrix.os == 'ubuntu-latest' + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + + - name: Build macOS Wheel + run: pip wheel . --no-deps -w dist + if: matrix.os == 'macos-latest' + + - name: Upload wheel as artifact + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.os }} Python ${{ matrix.python }} wheel + path: dist/* + + - name: Install libres + run: pip install dist/* + + - name: Run Python tests + run: | + # pytest adds every directory up-to and including python/ into sys.path, + # meaning that "import res" will import python/res and not the installed + # one. This doesn't work because the libecl.so library only exists in + # site-packages, so we copy directories required by the tests out into its + # own temporary directory. + mkdir test-run; cd test-run + mkdir -p {.git,python} + ln -s {..,$PWD}/bin + ln -s {..,$PWD}/lib + ln -s {..,$PWD}/test-data + ln -s {..,$PWD}/share + cp -R {..,$PWD}/python/tests + + # Env vars + export ECL_SKIP_SIGNAL=1 + export ERT_SHOW_BACKTRACE=1 + + # Run tests + python -m pip install -r ../test_requirements.txt + python -m pytest python/tests + + + publish: + name: Publish to PyPI + runs-on: ubuntu-latest + needs: [build-test-wheel] + + # If this is a tagged release + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + + steps: + - name: Get wheels + uses: actions/download-artifact@v2 + with: + path: artifacts + + - name: Move to dist/ + run: | + mkdir dist + find artifacts -name "*.whl" -exec mv '{}' dist/ \; + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@v1.3.1 + with: + user: statoil-travis + password: ${{ secrets.pypi_password }} diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index cecbc19f33..fd53e9d6a6 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -1,94 +1,135 @@ name: Libres testing -on: [pull_request] +on: [push, pull_request] -env: - INSTALL_DIR: ${{ github.workspace }}/install - ERT_SHOW_BACKTRACE: 1 jobs: - build: + build-test-cmake: + name: CMake - runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - python-version: [3.6, 3.7, 3.8] - os: [ubuntu-latest] - include: - - python-version: 3.6 - os: macos-latest + os: ['ubuntu-latest', 'macos-latest'] + + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} + - name: Install Ubuntu dependencies if: matrix.os == 'ubuntu-latest' run: | - sudo add-apt-repository ppa:opm/ppa - sudo apt-get update - sudo apt-get install libopm-simulators liblapack-dev valgrind - - name: Setup environment - run: | - echo "${{ env.INSTALL_DIR}}/bin" >> $GITHUB_PATH - echo "LD_LIBRARY_PATH=${{ env.INSTALL_DIR }}/lib:${{ env.INSTALL_DIR }}/lib64" >> $GITHUB_ENV - echo "DYLD_LIBRARY_PATH=${{ env.INSTALL_DIR }}/lib:${{ env.INSTALL_DIR }}/lib64" >> $GITHUB_ENV - echo "PYTHONPATH=${{ env.INSTALL_DIR }}/lib/python${{ matrix.python-version }}/site-packages:${{ env.INSTALL_DIR }}/lib/python${{ matrix.python-version }}/dist-packages" >> $GITHUB_ENV - - name: Install dependencies + sudo apt-get update -y + sudo apt-get install -y valgrind + + - name: Build libecl run: | - pip install -r requirements.txt - pip install -r test_requirements.txt - source .libecl_version - git clone https://github.com/equinor/libecl - pushd libecl - git fetch - git checkout tags/$LIBECL_VERSION - pip install -r requirements.txt - mkdir build - pushd build - cmake .. -DENABLE_PYTHON=ON \ - -DBUILD_APPLICATIONS=ON \ - -DINSTALL_CWRAP=OFF \ - -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \ - -DCMAKE_PREFIX_PATH=$INSTALL_DIR - make - make install - popd;popd - - name: Build and install libres + git clone https://github.com/equinor/libecl + mkdir libecl/build + cmake -S libecl -B libecl/build + sudo cmake --build libecl/build --target install + sudo rm -rf libecl + + - name: Build libres run: | - mkdir build - pushd build - ulimit -n 1024 - cmake .. -DBUILD_TESTS=ON \ - -DENABLE_PYTHON=ON \ - -DBUILD_APPLICATIONS=ON \ - -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \ - -DCMAKE_PREFIX_PATH=$INSTALL_DIR \ - -DCMAKE_INSTALL_NAME_DIR=$INSTALL_DIR/lib \ - -DCMAKE_C_FLAGS="-Werror -Wall -Wno-incompatible-pointer-types" \ - -DCMAKE_CXX_FLAGS="-Werror -Wall -Wno-unused-result -Wno-reorder \ - -Wno-sign-compare -Wno-unknown-pragmas \ - -Wno-unused-variable -Wno-parentheses \ - -Wno-unused-function -Wno-unused-but-set-variable \ - -Wno-unknown-warning-option -Wno-missing-braces \ - -Wno-varargs -Wno-sometimes-uninitialized \ - -Wno-tautological-compare" - make - make install - popd - - name: Run ctest + mkdir cmake-build + cmake -S . -B cmake-build \ + -DBUILD_TESTS=ON \ + -DRES_VERSION=1.2.3 \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo + cmake --build cmake-build + + - name: Run tests run: | - pushd build - set -e; python -c "import res"; set +e - ctest --output-on-failure $TEST_SUITE - popd - - name: Run pytest + cd cmake-build + export PATH=$PWD/bin:$PATH + ctest --output-on-failure + + + build-test-wheel: + name: Python + + strategy: + fail-fast: false + matrix: + os: ['ubuntu-latest', 'macos-latest'] + python: ['3.6', '3.7', '3.8'] + + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v2 + + - name: Build Linux Wheel + uses: docker://quay.io/pypa/manylinux2010_x86_64 + with: + entrypoint: /github/workspace/ci/github/build_linux_wheel.sh + args: ${{ matrix.python }} + if: matrix.os == 'ubuntu-latest' + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + + - name: Build macOS Wheel + run: pip wheel . --no-deps -w dist + if: matrix.os == 'macos-latest' + + - name: Upload wheel as artifact + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.os }} Python ${{ matrix.python }} wheel + path: dist/* + + - name: Install libres + run: pip install dist/* + + - name: Run Python tests run: | - export LD_LIBRARY_PATH=${{ env.INSTALL_DIR }}/lib:${{ env.INSTALL_DIR }}/lib64" - export DYLD_LIBRARY_PATH=${{ env.INSTALL_DIR }}/lib:${{ env.INSTALL_DIR }}/lib64" - pushd python - pytest + # pytest adds every directory up-to and including python/ into sys.path, + # meaning that "import res" will import python/res and not the installed + # one. This doesn't work because the libecl.so library only exists in + # site-packages, so we copy directories required by the tests out into its + # own temporary directory. + mkdir test-run; cd test-run + mkdir -p {.git,python} + ln -s {..,$PWD}/bin + ln -s {..,$PWD}/lib + ln -s {..,$PWD}/test-data + ln -s {..,$PWD}/share + cp -R {..,$PWD}/python/tests + + # Env vars + export ECL_SKIP_SIGNAL=1 + export ERT_SHOW_BACKTRACE=1 + + # Run tests + python -m pip install -r ../test_requirements.txt + python -m pytest python/tests + + + publish: + name: Publish to PyPI + runs-on: ubuntu-latest + needs: [build-test-wheel] + + # If this is a tagged release + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + + steps: + - name: Get wheels + uses: actions/download-artifact@v2 + with: + path: artifacts + + - name: Move to dist/ + run: | + mkdir dist + find artifacts -name "*.whl" -exec mv '{}' dist/ \; + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@v1.3.1 + with: + user: statoil-travis + password: ${{ secrets.pypi_password }} diff --git a/.gitignore b/.gitignore index 65306b0b57..a095c31738 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,8 @@ scratch.sparsebundle *.DS_Store __res_lib_path.py __res_lib_info.py + +*.egg-info +/_skbuild +/python/res/_version.py +.libs diff --git a/.libecl_version b/.libecl_version deleted file mode 100644 index 2a8af50a67..0000000000 --- a/.libecl_version +++ /dev/null @@ -1 +0,0 @@ -export LIBECL_VERSION=2.9.3rc0 diff --git a/CMakeLists.txt b/CMakeLists.txt index 207913c706..31b5c7f195 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,14 @@ -cmake_minimum_required(VERSION 2.8.12) +cmake_minimum_required(VERSION 3.6.1) project(res C CXX) +set(CMAKE_CXX_STANDARD 14) + +if(NOT SKBUILD) + message(WARNING "This CMakeLists.txt file should not be used directly.\n" + "Use 'pip install ${CMAKE_SOURCE_DIR}' to install this as a Python package.\n" + "Refer to the README for more information.") +endif() + if(NOT DEFINED CMAKE_MACOSX_RPATH) # There is some weirdness around this variable, the default value is different depending on # the cmake version, see policy CMP0042. @@ -14,98 +22,49 @@ enable_testing() #----------------------------------------------------------------- -set( RES_VERSION_MAJOR 0 ) -set( RES_VERSION_MINOR 0 ) -set( RES_VERSION_MICRO "not-available" ) - -# If the micro version is not integer, that should be interpreted as a -# development version leading towards version MAJOR.MINOR.0 +set(RES_VERSION "0.0.0-unset" CACHE STRING "libres version") -execute_process(COMMAND date "+%Y-%m-%d %H:%M:%S" OUTPUT_VARIABLE RES_BUILD_TIME ) -string(STRIP ${RES_BUILD_TIME} RES_BUILD_TIME) - -find_package(Git) - -if(GIT_FOUND) - execute_process( - COMMAND ${GIT_EXECUTABLE} "--git-dir=${CMAKE_SOURCE_DIR}/.git" status - RESULT_VARIABLE RESULT - OUTPUT_QUIET - ERROR_QUIET - ) - - if(NOT "${RESULT}" STREQUAL 0) - set(GIT_FOUND OFF) - endif() -endif() - -if(GIT_FOUND) - execute_process( - COMMAND ${GIT_EXECUTABLE} "--git-dir=${CMAKE_SOURCE_DIR}/.git" rev-parse HEAD - OUTPUT_VARIABLE GIT_COMMIT - OUTPUT_STRIP_TRAILING_WHITESPACE - ) - - execute_process( - COMMAND ${GIT_EXECUTABLE} "--git-dir=${CMAKE_SOURCE_DIR}/.git" rev-parse --short HEAD - OUTPUT_VARIABLE GIT_COMMIT_SHORT - OUTPUT_STRIP_TRAILING_WHITESPACE - ) - - execute_process( - COMMAND ${GIT_EXECUTABLE} "--git-dir=${CMAKE_SOURCE_DIR}/.git" describe --tags - OUTPUT_VARIABLE GIT_TAG - OUTPUT_STRIP_TRAILING_WHITESPACE - ) - - string( REGEX REPLACE "^([^.]+)\\.([^.]+)\\.(.+)$" "\\1" RES_VERSION_MAJOR "${GIT_TAG}") - string( REGEX REPLACE "^([^.]+)\\.([^.]+)\\.(.+)$" "\\2" RES_VERSION_MINOR "${GIT_TAG}") - string( REGEX REPLACE "^([^.]+)\\.([^.]+)\\.(.+)$" "\\3" RES_VERSION_MICRO "${GIT_TAG}") -else() - set( GIT_COMMIT "unknown (git not found!)") - set( GIT_COMMIT_SHORT "unknown (git not found!)") - message( WARNING "Git not found. Build will not contain correct version info." ) -endif() +string( REGEX REPLACE "^([^.]+)\\.([^.]+)\\.(.+)$" "\\1" RES_VERSION_MAJOR "${RES_VERSION}") +string( REGEX REPLACE "^([^.]+)\\.([^.]+)\\.(.+)$" "\\2" RES_VERSION_MINOR "${RES_VERSION}") +string( REGEX REPLACE "^([^.]+)\\.([^.]+)\\.(.+)$" "\\3" RES_VERSION_MICRO "${RES_VERSION}") message( STATUS "libres version: ${RES_VERSION_MAJOR}.${RES_VERSION_MINOR}.${RES_VERSION_MICRO}" ) list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Modules) find_package(CXX11Features) +set(CMAKE_C_STANDARD 99) #----------------------------------------------------------------- option( BUILD_TESTS "Should the tests be built" OFF) -option( ENABLE_PYTHON "Enable the python wrappers" ON ) option( USE_RPATH "Should we embed path to libraries" ON ) option( ERT_LSF_SUBMIT_TEST "Build and run tests of LSF submit" OFF) -set( SHARE_DIR "share/ert") -# If the SITE_CONFIG_FILE is not set as -DSITE_CONFIG_FILE switch when invoking -# cmake we set it here to location of the to-be-installed site-config file. That -# implies that when testing the binary will have embedded a path to non-existent -# or stale version of the site configuration file. Irrespective of the value of -# SITE_CONFIG_FILE the environment variable ERT_SITE_CONFIG will be set to point -# to the in-source version of the site-config file for testing. -# -# The variable SITE_CONFIG_FILE can be set by the user, if that is set it will -# take presedence. If the user has not explicitly set the SITE_CONFIG_FILE variable -# we will use the default path based on the install prefix. -set( SITE_CONFIG_FILE "" CACHE FILEPATH "Path to global ERT Configuration file") -set( __SITE_CONFIG_FILE "${CMAKE_INSTALL_PREFIX}/${SHARE_DIR}/site-config") -if (SITE_CONFIG_FILE) - set( __SITE_CONFIG_FILE ${SITE_CONFIG_FILE}) -endif() -message(STATUS "The path ${__SITE_CONFIG_FILE} will be compiled into the libres library as the location of the site configuration file") - -if (USE_RPATH) +if(NOT SKBUILD) + # We let setuptools handle installing share/ert normally. + + set( SHARE_DIR "share/ert") + # If the SITE_CONFIG_FILE is not set as -DSITE_CONFIG_FILE switch when invoking + # cmake we set it here to location of the to-be-installed site-config file. That + # implies that when testing the binary will have embedded a path to non-existent + # or stale version of the site configuration file. Irrespective of the value of + # SITE_CONFIG_FILE the environment variable ERT_SITE_CONFIG will be set to point + # to the in-source version of the site-config file for testing. + # + # The variable SITE_CONFIG_FILE can be set by the user, if that is set it will + # take presedence. If the user has not explicitly set the SITE_CONFIG_FILE variable + # we will use the default path based on the install prefix. + set( SITE_CONFIG_FILE "${CMAKE_INSTALL_PREFIX}/${SHARE_DIR}/site-config" CACHE FILEPATH "Path to global ERT Configuration file") + message(STATUS "The path ${SITE_CONFIG_FILE} will be compiled into the libres library as the location of the site configuration file") + + if (USE_RPATH) SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}") set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) -endif () + endif () +endif() #----------------------------------------------------------------- -find_package( ecl REQUIRED ) - set(ERT_LSF_LIB_PATH "" CACHE FILEPATH "Path to search for the LSF libraries") set(ERT_LSF_INCLUDE_PATH "" CACHE FILEPATH "Path to search for the LSF header files") find_path(LSF_HEADER_PATH lsf/lsf.h PATHS ${ERT_LSF_INCLUDE_PATH}) @@ -124,34 +83,11 @@ if (EXISTS ${EQUINOR_TESTDATA_ROOT}) message(STATUS "Linking testdata: ${LINK} -> ${EQUINOR_TESTDATA_ROOT}") endif() -file(COPY bin/job_dispatch.py DESTINATION ${PROJECT_BINARY_DIR}/bin ) +file(COPY python/job_runner/job_dispatch.py + DESTINATION ${PROJECT_BINARY_DIR}/bin + FILE_PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ) set( ERT_ROOT "${PROJECT_BINARY_DIR}" ) -#----------------------------------------------------------------- - -if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) - message(STATUS "Setting build type to 'RelWithDebInfo' as none was specified.") - set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING - "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." FORCE) - # Set the possible values of build type for cmake-gui - set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" - "MinSizeRel" "RelWithDebInfo") -endif() - -if (MSVC) - add_definitions( "/W3 /D_CRT_SECURE_NO_WARNINGS /wd4996" ) -else () - set(CMAKE_C_FLAGS "-std=gnu99 -Wall -Wno-unknown-pragmas ${CMAKE_C_FLAGS}") -endif() -if (NOT BUILD_SHARED_LIBS) - message(WARNING "Building python - forcing shared libs.") - set(BUILD_SHARED_LIBS ON) -endif () - -set(INSTALL_GROUP "" - CACHE STRING "Group to install as - blank to install as current group") -set(LIBRARY_OUTPUT_PATH ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}) -set(EXECUTABLE_OUTPUT_PATH ${CMAKE_CURRENT_BINARY_DIR}/bin) #----------------------------------------------------------------- find_package(Threads) @@ -163,6 +99,16 @@ endif () include(CheckFunctionExists) check_function_exists( regexec ERT_HAVE_REGEXP ) +#----------------------------------------------------------------- + +# This is to enable our dynamic lookup hack +if (APPLE) + set(CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS} -undefined dynamic_lookup") + set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS} -undefined dynamic_lookup") + set(CMAKE_SHARED_MODULE_CREATE_C_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS} -undefined dynamic_lookup") + set(CMAKE_SHARED_MODULE_CREATE_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS} -undefined dynamic_lookup") +endif () + #----------------------------------------------------------------- # install_example() is a small utility function which is used to install an # example file. If a file with the same name already exists in the installation @@ -187,30 +133,23 @@ endfunction() add_subdirectory(lib) -add_subdirectory(python) -foreach(share_dir workflows gui forward-models) - install(DIRECTORY "share/ert/${share_dir}" DESTINATION ${SHARE_DIR}) -endforeach() -install_example("share/ert/site-config" ${SHARE_DIR}) -install(PROGRAMS bin/job_dispatch.py DESTINATION bin ) +if(NOT SKBUILD) + foreach(share_dir workflows gui forward-models) + install(DIRECTORY "share/ert/${share_dir}" DESTINATION ${SHARE_DIR}) + endforeach() + install_example("share/ert/site-config" ${SHARE_DIR}) + install(PROGRAMS bin/job_dispatch.py DESTINATION bin ) -foreach (script ecl100 ecl300 flow rms) - install(PROGRAMS "share/ert/forward-models/res/script/${script}" DESTINATION "${SHARE_DIR}/forward-models/res/script") -endforeach() + foreach (script ecl100 ecl300 flow rms) + install(PROGRAMS "share/ert/forward-models/res/script/${script}" DESTINATION "${SHARE_DIR}/forward-models/res/script") + endforeach() -foreach (script careful_copy_file copy_directory copy_file delete_directory delete_file make_directory move_file symlink) - install(PROGRAMS "share/ert/shell_scripts/${script}" DESTINATION "${SHARE_DIR}/shell_scripts") -endforeach() + foreach (script careful_copy_file copy_directory copy_file delete_directory delete_file make_directory move_file symlink) + install(PROGRAMS "share/ert/shell_scripts/${script}" DESTINATION "${SHARE_DIR}/shell_scripts") + endforeach() -foreach (script template_render) + foreach (script template_render) install(PROGRAMS "share/ert/forward-models/templating/script/${script}" DESTINATION "${SHARE_DIR}/forward-models/templating/script") -endforeach() - -install(EXPORT res-config - DESTINATION share/cmake/res - NAMESPACE res::) -export(TARGETS res - NAMESPACE res:: - FILE resConfig.cmake) -export(PACKAGE res) + endforeach() +endif() diff --git a/Jenkinsfile b/Jenkinsfile index 09ef3fd949..d2e1269b89 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,5 +1,5 @@ pipeline { - agent { label 'si-build' } + agent { label 'scout-ci7' } environment { WORKING_DIR = sh(script: 'mktemp -d', , returnStdout: true).trim() } @@ -10,9 +10,10 @@ pipeline { sh 'sh testjenkins.sh setup' } } - stage('build ecl') { + stage('build libres') { steps { - sh 'sh testjenkins.sh build_ecl' + sh 'sh testjenkins.sh build_libecl' + sh 'sh testjenkins.sh build_libres' } } stage('build res') { diff --git a/README.md b/README.md index 3944028395..967eae1215 100644 --- a/README.md +++ b/README.md @@ -1,110 +1,95 @@ # libres [![Libres testing](https://github.com/equinor/libres/workflows/Libres%20testing/badge.svg)](https://github.com/equinor/libres/actions?query=workflow%3A%22Libres+testing%22) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) -`libres` is part of the `ERT` project: _[Ensemble based Reservoir Tool](https://github.com/Equinor/ert)_. +*libres* is part of the `ERT` project: _[Ensemble based Reservoir Tool](https://github.com/Equinor/ert)_. It is now available in PyPI: -## Building libres +``` sh +$ pip install equinor-libres +``` -### 1. Build libecl -Build and install [libecl](https://github.com/Equinor/libecl) at the version that `.libecl_version` specifies. When configuring -`libecl` you should used the option `-DCMAKE_INSTALL_PREFIX` to tell ``cmake`` -where to install `libecl`. The value passed to `CMAKE_INSTALL_PREFIX` will be -needed when running cmake to configure `libres` in point 4 below. For now let us -assume that the prefix `/local/ert/install` was used. - - -### 2. Install Python dependencies +or, for the latest development version (requires GCC/clang and `Python.h`): +``` sh +$ pip install git+https://github.com/equinor/libres.git@master ``` -pip install -r requirements.txt -``` -### 3. Update environment variables -To ensure that the build system correctly finds the `ecl` Python package you -need to set the environment variables `PYTHONPATH` and `LD_LIBRARY_PATH` to -include the `libecl` installation: - +## Development + +*libres* is meant to be installed using `setup.py`, directly or using `pip +install ./`. The `CMakeLists.txt` exists, but is used by `setup.py` to generate +the `libres` C library and by Github Actions to run C tests. + +### Building + +Use the following commands to start developing from a clean virtualenv ``` -bash% export LD_LIBRARY_PATH=/local/ert/install/lib64:$LD_LIBRARY_PATH -bash% export PYTHONPATH=/local/ert/install/lib/python3.6/site-packages:$PYTHONPATH +$ pip install -r requirements.txt +$ python setup.py develop ``` -Observe that path components `lib64` and `lib/python3.6/site-packages` will -depend on your Python version and which Linux distribution you are using. The -example given here is for RedHat based distributions. +Alternatively, `pip install -e .` will also setup `libres` for development, but +it will be more difficult to recompile the C library. +[scikit-build](https://scikit-build.readthedocs.io/en/latest/index.html) is used +for compiling the C library. It creates a directory named `_skbuild` which is +reused upon future invocations of either `python setup.py develop`, or `python +setup.py build_ext`. The latter only rebuilds the C library. In some cases this +directory must be removed in order for compilation to succeed. -### 4. Run `cmake` to configure `libres` +The C library files get installed into `python/res/.libs`, which is where the +`res` module will look for them. -When running `cmake` you must tell `cmake` where the `libecl` code is located -with `-DCMAKE_PREFIX_PATH`, i.e. in addition to other possible arguments you -must at least add: +### Testing Python code +Install the required testing packages and run tests. ``` --DCMAKE_PREFIX_PATH=/local/ert/install +$ pip install -r test_requirements.txt +$ pytest python/tests ``` -in addition you probably want to pass `-DCMAKE_INSTALL_PREFIX` to configure where -the `libres` distribuion should be installed, normally that will be the same -location where you have already installed `libecl`. - +You may need to do `export PYTHONPATH=$PWD/python` for `pytest` to find your +local development installation. -### 5. Run `make` to compile `libres` +### Testing C code -After you have run cmake you should run `make` and `make install` to build and install `libres`: +Install [*ecl*](https://github.com/Equinor/ecl) using CMake as a C library. Then: +``` sh +$ mkdir build +$ cd build +$ cmake .. -DBUILD_TESTS=ON +$ cmake --build . +$ ctest --output-on-failure ``` -bash% make -bash% make install -``` - -### 6. postinstall configuration +## Configuration -#### 6.1. The `site_config` file +### The `site_config` file As part of the installation process `libres` will install a file called -`site-config` in `$prefix/share/ert/site-config`; when ert starts this file will -be loaded before the users personal config file. For more extensive use of `ert` -it might be benefical to customize the `site-config` file to your personal site. -There are three possible ways to do this: - -1. You can just edit the installed file manually - `libres` will not install - it's version of the `site-config` file if one is already present. - -2. The path to `site-config` file is *compiled into* the `libres` library, if - you pass the `cmake` option `-DSITE_CONFIG_FILE=/path/to/site/config` when - configuring `cmake`; that way your own personal `site-config` file is built - in. - -3. If you set the environment variable `ERT_SITE_CONFIG` to point to an - alternative file that will be used when bootstrapping. This can be a handy - way to debug the `site-config` settings. - -For a start you can probably just use the shipped default version of the -`site-config` file. - - -#### 6.2 Forward models - -The `libres` code contains basic functionality for forward models to run the -reservoir simulators Eclipse/flow and the geomodelling program RMS. Exactly how -these programs depend on the setup on your site and you must make some -modifications to two files installed with `libres`: - -##### 6.2.1. Eclipse/flow configuration +`site-config` in `share/ert/site-config`; when ert starts this file will be +loaded before the users personal config file. For more extensive use of `ert` it +might be benefical to customize the `site-config` file to your personal site. -In the Python distribution installed by `libres` there is a file: -`res/fm/ecl/ecl_config.yml` which is used to configure the eclipse/flow versions -are available at the location. You should edit this file to correspond to the -conditions at your site; alternatively you can store an alternative -configuration file elsewhere and set the environment variable `ECL_SITE_CONFIG` -to point to the alternative file. +To customize, you need to set the environment variable `ERT_SITE_CONFIG` to +point to an alternative file that will be used. + +### 6.2 Forward models +`libres` contains basic functionality for forward models to run the reservoir +simulators Eclipse/flow and the geomodelling program RMS. Exactly how these +programs depend on the setup on your site and you must make some modifications +to two files installed with `libres`: -##### 6.2.2. RMS configuration +#### 6.2.1. Eclipse/flow configuration + +In the Python distribution installed by `libres` there is a file +`res/fm/ecl/ecl_config.yml` which is used to configure the eclipse/flow versions +are available at the location. You can provide an alternative configuration file +by setting the environment variable `ECL_SITE_CONFIG`. + +#### 6.2.2. RMS configuration In the Python distribution installed by `libres` there is a file: `res/fm/rms/rms_config.yml` which contains some site specific RMS configuration. -You should update this file with your local path to the `rms` wrapper script -supplied by `Roxar`; alternatively you can store an alternative configuration -file elseswhere and set the environment variable `RMS_SITE_CONFIG` to point to -the alternative file. +You should provide an alternative file with your local path to the `rms` wrapper +script supplied by _Roxar_ by setting the environment variable `RMS_SITE_CONFIG` +to point to the alternative file. diff --git a/ci/github/build_linux_wheel.sh b/ci/github/build_linux_wheel.sh new file mode 100755 index 0000000000..110ef65735 --- /dev/null +++ b/ci/github/build_linux_wheel.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -euo pipefail + +case "$1" in + 3.6) pyver=cp36-cp36m ;; + 3.7) pyver=cp37-cp37m ;; + 3.8) pyver=cp38-cp38 ;; + 3.9) pyver=cp39-cp39 ;; + *) + echo "Unknown Python version $1" + exit 1 + ;; +esac + +# Install dependencies +yum install -y lapack-devel blas-devel + +# Build wheel +cd /github/workspace +/opt/python/$pyver/bin/pip wheel . --no-deps -w wheelhouse +auditwheel repair wheelhouse/* -w dist diff --git a/ci/jenkins/testkomodo.sh b/ci/jenkins/testkomodo.sh index b555c3be1b..3268c6ce76 100644 --- a/ci/jenkins/testkomodo.sh +++ b/ci/jenkins/testkomodo.sh @@ -1,9 +1,3 @@ -_build_libres () { - echo "Building libres" - cmake -S . -B cmake-build -DBUILD_TESTS=ON - cmake --build cmake-build -} - copy_test_files () { mkdir $CI_TEST_ROOT/.git mkdir -p $CI_TEST_ROOT/python/res/fm/rms/ @@ -17,18 +11,7 @@ copy_test_files () { ln -s {$PWD,$CI_TEST_ROOT}/python/res/fm/rms/rms_config.yml } -install_package () { - local PWD=$(pwd) - export ERT_SITE_CONFIG=$PWD/share/ert/site-config - export PYTHONPATH=$CI_SOURCE_ROOT/cmake-build/lib/python3.6/site-packages:${PYTHONPATH:-} - export LD_LIBRARY_PATH=$CI_SOURCE_ROOT/cmake-build/lib:$CI_SOURCE_ROOT/cmake-build/lib64:${LD_LIBRARY_PATH:-} -} - start_tests () { - echo "Running ctest" - cd $CI_SOURCE_ROOT/cmake-build - ctest --output-on-failure - echo "Running pytest" cd $CI_TEST_ROOT/python export ECL_SKIP_SIGNAL=ON @@ -43,14 +26,9 @@ start_tests () { } run_tests () { - ci_install_cmake - _build_libres - if [ ! -z "${CI_PR_RUN:-}" ] + if [[ ! -z "${CI_PR_RUN:-}" ]] then - install_package - else - #removing built libs in order to ensure we are using libs from komodo - rm -r cmake-build/lib64 + pip install . fi copy_test_files diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 88d005bdd7..f0546d4baa 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -9,7 +9,15 @@ else () message(STATUS "LSF not found") endif () -add_library(res res_util/res_log.cpp +if(NOT SKBUILD) + # Link directly to libecl if not using `pip install` + find_package(ecl REQUIRED) + set(ECL ecl) + set(RES res) +endif() + +add_library(res SHARED + res_util/res_log.cpp res_util/log.cpp res_util/es_testdata.cpp res_util/arg_pack.cpp @@ -184,32 +192,31 @@ add_library(res res_util/res_log.cpp #----------------------------------------------------------------- -add_library(rml_enkf SHARED analysis/modules/rml_enkf_config.c - analysis/modules/rml_enkf.c - analysis/modules/rml_enkf_common.c - analysis/modules/rml_enkf_log.c - ) - -target_link_libraries(rml_enkf PRIVATE res ecl ${CMAKE_DL_LIBS}) -target_include_directories(rml_enkf PRIVATE analysis/modules) +add_library(rml_enkf SHARED + analysis/modules/rml_enkf_config.c + analysis/modules/rml_enkf.c + analysis/modules/rml_enkf_common.c + analysis/modules/rml_enkf_log.c) +target_link_libraries(rml_enkf PRIVATE ${RES} ${ECL} ${CMAKE_DL_LIBS}) +target_include_directories(rml_enkf PRIVATE ${ECL_INCLUDE_DIRS} include analysis/modules) set_target_properties(rml_enkf PROPERTIES VERSION 1.0 - SOVERSION 1.0 - PREFIX "") - -set( ies_source analysis/modules/ies_enkf.c - analysis/modules/ies_enkf_config.c - analysis/modules/ies_enkf_data.c) - -add_library(ies MODULE ${ies_source}) -target_link_libraries(ies PRIVATE res ecl ${CMAKE_DL_LIBS}) -target_include_directories(ies PRIVATE analysis/modules) - -add_library(std_enkf_debug SHARED analysis/modules/std_enkf_debug.c) -target_link_libraries(std_enkf_debug PRIVATE res ecl ${CMAKE_DL_LIBS}) -target_include_directories(std_enkf_debug PRIVATE modules) + SOVERSION 1.0 + PREFIX "") + +add_library(ies SHARED + analysis/modules/ies_enkf.c + analysis/modules/ies_enkf_config.c + analysis/modules/ies_enkf_data.c) +target_link_libraries(ies PRIVATE ${RES} ${ECL} ${CMAKE_DL_LIBS}) +target_include_directories(ies PRIVATE ${ECL_INCLUDE_DIRS} include analysis/modules) + +add_library(std_enkf_debug SHARED + analysis/modules/std_enkf_debug.c) +target_link_libraries(std_enkf_debug PRIVATE ${RES} ${ECL} ${CMAKE_DL_LIBS}) +target_include_directories(std_enkf_debug PRIVATE ${ECL_INCLUDE_DIRS} include modules) set_target_properties(std_enkf_debug PROPERTIES VERSION 1.0 - SOVERSION 1.0 - PREFIX "") + SOVERSION 1.0 + PREFIX "") #----------------------------------------------------------------- @@ -218,31 +225,21 @@ set_target_properties(std_enkf_debug PROPERTIES VERSION 1.0 target_compile_definitions(res PRIVATE -DINTERNAL_LINK) find_package(LAPACK REQUIRED) -target_link_libraries( res PUBLIC ecl ${LAPACK_LIBRARIES} ${LAPACK_LINKER_FLAGS}) +target_link_libraries(res PUBLIC ${ECL} ${LAPACK_LIBRARIES} ${LAPACK_LINKER_FLAGS}) target_include_directories(res PUBLIC $ $ PRIVATE $ PRIVATE $ - ) - - -set_target_properties( res PROPERTIES VERSION ${RES_VERSION_MAJOR}.${RES_VERSION_MINOR} SOVERSION ${RES_VERSION_MAJOR} ) + PRIVATE "${ECL_INCLUDE_DIRS}") target_compile_definitions(res PRIVATE - -DGIT_COMMIT=${GIT_COMMIT} - -DGIT_COMMIT_SHORT=${GIT_COMMIT_SHORT} - -DRES_VERSION_MAJOR=${RES_VERSION_MAJOR} - -DRES_VERSION_MINOR=${RES_VERSION_MINOR} - -DRES_VERSION_MICRO=${RES_VERSION_MICRO} - -DCOMPILE_TIME_STAMP="${RES_BUILD_TIME}" - -DSITE_CONFIG_FILE=\"${__SITE_CONFIG_FILE}\" -) - - - - -install(DIRECTORY include/ DESTINATION include) + -DGIT_COMMIT=${GIT_COMMIT} + -DGIT_COMMIT_SHORT=${GIT_COMMIT_SHORT} + -DRES_VERSION_MAJOR=${RES_VERSION_MAJOR} + -DRES_VERSION_MINOR=${RES_VERSION_MINOR} + -DRES_VERSION_MICRO=${RES_VERSION_MICRO} + -DCOMPILE_TIME_STAMP="${RES_BUILD_TIME}") install(TARGETS res EXPORT res-config @@ -265,46 +262,33 @@ install(TARGETS ies if (NOT BUILD_TESTS) return () endif() + foreach(name ies_enkf_config ies_enkf_data) - add_executable(${name} analysis/modules/tests/${name}.cpp ${ies_source}) - target_include_directories(${name} PRIVATE analysis/modules) - target_link_libraries(${name} res) - add_test(NAME ${name} COMMAND ${name}) + add_executable(${name} analysis/modules/tests/${name}.cpp ${ies_source}) + target_include_directories(${name} PRIVATE analysis/modules) + target_link_libraries(${name} res ies) + add_test(NAME ${name} COMMAND ${name}) endforeach() + add_executable( ies_module analysis/modules/tests/ies_enkf_module.cpp) -target_link_libraries(ies_module res) +target_link_libraries(ies_module res ies) add_test(NAME ies_module COMMAND ies_module $ ${CMAKE_CURRENT_SOURCE_DIR}/analysis/modules/tests/test-data/poly) add_executable(ies_linalg analysis/modules/tests/ies_linalg.cpp ${ies_source}) target_include_directories(ies_linalg PRIVATE analysis/modules) -target_link_libraries(ies_linalg res) +target_link_libraries(ies_linalg res ies) add_test(NAME ies_linalg COMMAND ies_linalg ${CMAKE_CURRENT_SOURCE_DIR}/analysis/modules/tests/test-data/poly) add_executable(ies_std_compare analysis/modules/tests/ies_std_compare.cpp ${ies_source}) target_include_directories(ies_std_compare PRIVATE analysis/modules) -target_link_libraries(ies_std_compare res) +target_link_libraries(ies_std_compare res ies) add_test(NAME ies_std_compare COMMAND ies_std_compare ${CMAKE_CURRENT_SOURCE_DIR}/analysis/modules/tests/test-data/poly) add_executable(ies_iteration analysis/modules/tests/ies_iteration.cpp ${ies_source}) target_include_directories(ies_iteration PRIVATE analysis/modules) -target_link_libraries(ies_iteration res) +target_link_libraries(ies_iteration res ies) add_test(NAME ies_iteration COMMAND ies_iteration ${CMAKE_CURRENT_SOURCE_DIR}/analysis/modules/tests/test-data/poly_normal) - -# The ies library is only mentioned as a dependency indirectly through the -# generator expression, seems that is not sufficient to -# guarantee that it is indeed built in correct order; we therefor add an -# explicit dependency here. -foreach( ies_test ies_enkf_config - ies_enkf_data - ies_module - ies_linalg - ies_std_compare - ies_iteration ) - add_dependencies( ${ies_test} ies ) -endforeach() - - foreach(name ert_util_logh ert_util_arg_pack ert_util_matrix @@ -492,21 +476,22 @@ add_test(NAME job_loadFail3 ${CMAKE_CURRENT_SOURCE_DIR}/job_queue/tests/data/internalFail ${CMAKE_CURRENT_SOURCE_DIR}/job_queue/tests/data/externalFail) +set_target_properties(job_workflow_test PROPERTIES ENABLE_EXPORTS ON) add_test(NAME job_workflow_test COMMAND job_workflow_test - ${EXECUTABLE_OUTPUT_PATH} + ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/job_queue/tests/data/internal_job) add_test(NAME job_lsf_exclude_hosts_test COMMAND job_lsf_exclude_hosts_test job_program NULL LOCAL) file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/job_queue/tests/data/qsub_emulators/ - DESTINATION ${EXECUTABLE_OUTPUT_PATH}) + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) add_test(NAME job_torque_submit_test - WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMAND job_torque_submit_test dummyparam) -set_property(TEST job_torque_submit_test PROPERTY ENVIRONMENT “setenv PATH ${EXECUTABLE_OUTPUT_PATH}:$PATH”) +set_property(TEST job_torque_submit_test PROPERTY ENVIRONMENT “setenv PATH ${CMAKE_CURRENT_BINARY_DIR}:$PATH”) add_test(NAME ext_joblist_test COMMAND ext_joblist_test @@ -616,8 +601,6 @@ foreach (test enkf_active_list target_include_directories(${test} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/private-include) endforeach () - - function( add_config_test name command ) add_test( NAME ${name} COMMAND ${command} ${ARGN}) @@ -719,10 +702,6 @@ add_config_test(gen_kw_logarithmic_test gen_kw_logarithmic_test ${CMAKE_CURRENT_SOURCE_DIR}/enkf/tests/data/config/gen_kw_logarithmic/config_GEN_KW_logarithmic) - - - - add_config_test(enkf_analysis_config_analysis_load enkf_analysis_config_analysis_load ${CMAKE_CURRENT_SOURCE_DIR}/enkf/tests/data/config/analysis_load_config) # It seems necessary to set the LD_LIBRARY_PATH for the loading of the # rml_enkf module to work. Do understand why! diff --git a/lib/analysis/analysis_module.cpp b/lib/analysis/analysis_module.cpp index 142743f970..dcaf391b6b 100644 --- a/lib/analysis/analysis_module.cpp +++ b/lib/analysis/analysis_module.cpp @@ -16,6 +16,8 @@ for more details. */ +#include +#include #include #include #include @@ -61,6 +63,9 @@ struct analysis_module_struct { }; +static std::string analysis_modules_dir; + + static analysis_module_type * analysis_module_alloc_empty( const char * symbol_table , const char * lib_name) { analysis_module_type * module = (analysis_module_type*)util_malloc( sizeof * module ); @@ -138,34 +143,52 @@ static analysis_module_type * analysis_module_alloc__( const analysis_table_type } - - - static analysis_module_type * analysis_module_alloc( const char * libname , const char * table_name , bool verbose, analysis_module_load_status_enum * load_status) { analysis_module_type * module = NULL; - void * lib_handle = dlopen( libname , RTLD_NOW ); - if (lib_handle != NULL) { - analysis_table_type * analysis_table = (analysis_table_type *) dlsym( lib_handle , table_name ); - if (analysis_table != NULL) { - *load_status = LOAD_OK; - module = analysis_module_alloc__( analysis_table , table_name , libname , lib_handle ); - } else { - *load_status = LOAD_SYMBOL_TABLE_NOT_FOUND; + void * lib_handle = nullptr; + + if (libname == nullptr) { + // internal + lib_handle = dlopen(nullptr, RTLD_NOW); + } else { + // external, look for the library in /res/.libs + if (!analysis_modules_dir.empty()) { + auto lib_path = analysis_modules_dir + "/" + libname; + lib_handle = dlopen(lib_path.c_str(), RTLD_NOW); + if (lib_handle == nullptr && verbose) + fprintf(stderr, "Failed to load library:%s Error:%s\n", lib_path.c_str(), dlerror()); + } + + // external, look for library system-wide + if (lib_handle == nullptr) { + lib_handle = dlopen(libname, RTLD_NOW); + } + + // error handling + if (lib_handle == nullptr) { + *load_status = DLOPEN_FAILURE; if (verbose) - fprintf(stderr , "Failed to load symbol table:%s Error:%s \n",table_name , dlerror()); + fprintf(stderr, "Failed to load library:%s Error:%s \n", libname, dlerror()); + return NULL; } + } - if (module == NULL) - dlclose( lib_handle ); + analysis_table_type * analysis_table = (analysis_table_type *) dlsym( lib_handle , table_name ); + if (analysis_table != NULL) { + *load_status = LOAD_OK; + module = analysis_module_alloc__( analysis_table , table_name , libname , lib_handle ); } else { - *load_status = DLOPEN_FAILURE; + *load_status = LOAD_SYMBOL_TABLE_NOT_FOUND; if (verbose) - fprintf(stderr , "Failed to load library:%s Error:%s \n",libname , dlerror()); + fprintf(stderr , "Failed to load symbol table:%s Error:%s \n",table_name , dlerror()); } + if (module == NULL) + dlclose( lib_handle ); + if (module != NULL) { if (libname == NULL) module->internal = true; @@ -467,3 +490,9 @@ void * analysis_module_get_ptr( const analysis_module_type * module , const char return NULL; } + + +extern "C" +void set_analysis_modules_dir(const char * lib) { + analysis_modules_dir = lib; +} diff --git a/lib/enkf/site_config.cpp b/lib/enkf/site_config.cpp index e03fa4c5f9..b72f53974f 100644 --- a/lib/enkf/site_config.cpp +++ b/lib/enkf/site_config.cpp @@ -20,6 +20,8 @@ #include #include +#include + #include #include @@ -473,13 +475,21 @@ config_content_type * site_config_alloc_content( } -const char * site_config_get_location() { - const char * site_config = NULL; +static std::string _site_config; - #ifdef SITE_CONFIG_FILE - site_config = SITE_CONFIG_FILE; - #endif +extern "C" +void set_site_config(const char * site_config) { + auto path = realpath(site_config, NULL); + if (!path) { + perror("Failed to set default site config file"); + } else { + _site_config = path; + free(path); + } +} +const char * site_config_get_location() { + const char * site_config = _site_config.c_str(); const char * env_site_config = getenv("ERT_SITE_CONFIG"); if(env_site_config != NULL) { diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..8965069423 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,2 @@ +[build-system] +requires = ["setuptools", "setuptools_scm", "wheel", "scikit-build", "cmake", "ninja", "libecl"] diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt deleted file mode 100644 index 5aa99a9a6b..0000000000 --- a/python/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -if (NOT ENABLE_PYTHON) - return () -endif() - -set( CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules") -include(init_python) -init_python() - -add_subdirectory( res ) -add_subdirectory( job_runner ) -add_subdirectory( ert_logger ) diff --git a/python/cmake/Modules/add_python_package.cmake b/python/cmake/Modules/add_python_package.cmake deleted file mode 100644 index bb0c1a6e2d..0000000000 --- a/python/cmake/Modules/add_python_package.cmake +++ /dev/null @@ -1,42 +0,0 @@ -function(add_python_package target package_path source_files install_package) - - set(build_files "") - - foreach (file ${source_files} ) - string(SUBSTRING ${file} 0 1 first_char) - string(SUBSTRING ${file} 1 1 second_char) - if (first_char STREQUAL "/" OR second_char STREQUAL ":") - set( source_file ${file} ) - set( build_file ${file} ) - file(RELATIVE_PATH file ${CMAKE_CURRENT_BINARY_DIR} ${file}) - set(dependent_target) - else() - set( source_file ${CMAKE_CURRENT_SOURCE_DIR}/${file} ) - set( build_file ${PROJECT_BINARY_DIR}/${package_path}/${file} ) - set(dependent_target DEPENDS ${source_file}) - endif() - if("$ENV{DESTDIR}" STREQUAL "") - set( install_file ${CMAKE_INSTALL_PREFIX}/${package_path}/${file} ) - else() - set( install_file $ENV{DESTDIR}/${CMAKE_INSTALL_PREFIX}/${package_path}/${file} ) - endif() - - add_custom_command( - OUTPUT ${build_file} - COMMAND ${PYTHON_EXECUTABLE} - ARGS ${PROJECT_BINARY_DIR}/bin/cmake_pyc ${source_file} ${build_file} - ${dependent_target}) - - list(APPEND build_files ${build_file} ) - - if (install_package) - # For cmake versions >= 2.8.12 the preferred keyword is DIRECTORY. - get_filename_component( src_path ${file} PATH ) - install(FILES ${build_file} DESTINATION ${CMAKE_INSTALL_PREFIX}/${package_path}/${src_path}) - install(CODE "execute_process(COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_BINARY_DIR}/bin/cmake_pyc_file ${install_file})") - endif() - - endforeach() - add_custom_target( ${target} ALL DEPENDS ${build_files}) - -endfunction() diff --git a/python/cmake/Modules/add_python_test.cmake b/python/cmake/Modules/add_python_test.cmake deleted file mode 100644 index cb9130e1c6..0000000000 --- a/python/cmake/Modules/add_python_test.cmake +++ /dev/null @@ -1,38 +0,0 @@ -# This macro will create a ctest based on the supplied TEST_CLASS. The -# TEST_CLASS argument should correspond to a valid Python path, i.e. -# -# >> import ${TEST_CLASS} -# -# should work. The actual test is by running a small test script which -# will invoke normal Python test discovery functionality. This is a -# macro, and relevant variables must be crrectly set in calling scope -# before it is invoked: -# -# PYTHON_TEST_RUNNER: Path to executable which will load the testcase -# given by ${TEST_CLASS} and run it. -# -# -# CTEST_PYTHONPATH: Normal colon separated path variable, should at -# least include the binary root directory of the current python -# installation, but can in addition contain the path to -# additional packages. The PYTHON_TEST_RUNNER should inspect the -# $CTEST_PYTHONPATH environment variable and update sys.path -# accordingly. - -macro( addPythonTest TEST_CLASS ) - set(TEST_NAME ${TEST_CLASS}) - - add_test(NAME ${TEST_NAME} - WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/${PYTHON_INSTALL_PREFIX}" - COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_BINARY_DIR}/bin/ctest_run_python ${TEST_CLASS} ) - - set(oneValueArgs LABELS) - cmake_parse_arguments(TEST_OPTIONS "" "${oneValueArgs}" "" ${ARGN}) - if(TEST_OPTIONS_LABELS) - set_property(TEST ${TEST_NAME} PROPERTY LABELS "Python:${TEST_OPTIONS_LABELS}") - else() - set_property(TEST ${TEST_NAME} PROPERTY LABELS "Python") - endif() - - set_property(TEST ${TEST_NAME} PROPERTY ENVIRONMENT "CTEST_PYTHONPATH=${CTEST_PYTHONPATH};ERT_ROOT=${ERT_ROOT}") -endmacro( ) diff --git a/python/cmake/Modules/init_python.cmake b/python/cmake/Modules/init_python.cmake deleted file mode 100644 index 71483d8358..0000000000 --- a/python/cmake/Modules/init_python.cmake +++ /dev/null @@ -1,187 +0,0 @@ -# This macro will initialize the current cmake session for Python. The -# macro starts by looking for the Python interpreter of correct -# version. When a Python interepreter of the correct version has been -# located the macro will continue to set variables, load other cmake -# modules and generate scripts to be used in the remaining part of the -# cmake process. -# -# Variables which will be set: -# ---------------------------- -# -# PYTHON_INSTALL_PREFIX: All python packages will be located in -# ${GLOBAL_PREFIX}/${PYTHON_INSTALL_PREFIX} - this applies both -# when searching for dependencies and when installing. -# -# CTEST_PYTHONPATH: Normal ':' separated path variables which is -# passed to the test runner. Should contain the PYTHONPATH to -# all third party packages which are not in the default search -# path. The CTEST_PYTHONPATH variable will be updated by the -# python_package( ) function when searching for third party -# packages. -# -# -# New functions/macros which will be available: -# --------------------------------------------- -# -# add_python_package( ): This function will copy python source files -# to the build directory, 'compile' them and set up installation. -# -# -# add_python_test( ): Set up a test based on invoking a Python test -# class with a small python executable front end. -# -# find_python_package( ): Will search for a python package. -# -# -# New scripts generated: -# ---------------------- -# -# -# cmake_pyc: Small script which will run in-place Python compilation -# of a directory tree recursively. -# -# cmake_pyc_file: Small script which will compile one python file. -# -# ctest_run_python: Small script which will invoke one Python test class. -# -# All the generated scripts will be located in ${PROJECT_BINARY_DIR}/bin. -# -# -# Downstream projects should use this as: -# -# include( init_python ) -# init_python() -# ... - -macro(init_python) - - FIND_PACKAGE(PythonInterp) - if (NOT DEFINED PYTHON_EXECUTABLE) - message(WARNING "Python interpreter not found - Python wrappers not enabled") - set( ENABLE_PYTHON OFF PARENT_SCOPE ) - return() - endif() - - if (EXISTS "/etc/debian_version") - set( PYTHON_PACKAGE_PATH "dist-packages") - else() - set( PYTHON_PACKAGE_PATH "site-packages") - endif() - - set(PYTHON_INSTALL_PREFIX "lib/python${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR}/${PYTHON_PACKAGE_PATH}" CACHE STRING "Subdirectory to install Python modules in") - set(CTEST_PYTHONPATH ${PROJECT_BINARY_DIR}/${PYTHON_INSTALL_PREFIX}) - configure_python_env( ) - include(add_python_test) - include(add_python_package) -endmacro() - - - -# The function configure_python_env( ) will generate three small -# Python scripts which will be located in ${PROJECT_BINARY_DIR}/bin -# and will be used when 'compiling' and testing Python code. The -# function will be called from the init_python() macro. - -function( configure_python_env ) - -FILE(WRITE "${PROJECT_BINARY_DIR}/bin/ctest_run_python" -"import sys -import os -from unittest import TextTestRunner - - -def runTestCase(tests, verbosity=0): - test_result = TextTestRunner(verbosity=verbosity).run(tests) - - if len(test_result.errors) or len(test_result.failures): - test_result.printErrors() - sys.exit(1) - - -# This will update both the internal sys.path load order and the -# environment variable PYTHONPATH with the following list: -# -# cwd:CTEST_PYTHONPATH:PYTHONPATH - -def update_path(): - path_list = [os.getcwd()] - - if 'CTEST_PYTHONPATH' in os.environ: - ctest_pythonpath = os.environ['CTEST_PYTHONPATH'] - for path in ctest_pythonpath.split(':'): - path_list.append( path ) - - for path in reversed(path_list): - sys.path.insert(0 , path) - - if 'PYTHONPATH' in os.environ: - pythonpath = os.environ['PYTHONPATH'] - for path in pythonpath.split(':'): - path_list.append( path ) - - os.environ['PYTHONPATH'] = ':'.join( path_list ) - - - -if __name__ == '__main__': - update_path( ) - from ecl.util.test import ErtTestRunner - for test_class in sys.argv[1:]: - tests = ErtTestRunner.getTestsFromTestClass(test_class) - - # Set verbosity to 2 to see which test method in a class that fails. - runTestCase(tests, verbosity=0) -") - -#----------------------------------------------------------------- - -FILE(WRITE "${PROJECT_BINARY_DIR}/bin/cmake_pyc" -" -import py_compile -import os -import os.path -import sys -import shutil - - -src_file = sys.argv[1] -target_file = sys.argv[2] - -(target_path , tail) = os.path.split( target_file ) -if not os.path.exists( target_path ): - try: - os.makedirs( target_path ) - except: - # When running make with multiple processes there might be a - # race to create this directory. - pass - -shutil.copyfile( src_file , target_file ) -shutil.copystat( src_file , target_file ) -try: - py_compile.compile( target_file , doraise = True) -except Exception as error: - sys.exit('py_compile(%s) failed:%s' % (target_file , error)) -") - -#----------------------------------------------------------------- - -FILE(WRITE "${PROJECT_BINARY_DIR}/bin/cmake_pyc_file" -" -import py_compile -import os -import sys -import os.path - -# Small 'python compiler' used in the build system for ert. - -for file in sys.argv[1:]: - try: - py_compile.compile( file , doraise = True ) - except Exception as error: - sys.exit('py_compile(%s) failed:%s' % (file , error)) -") - - - -endfunction() diff --git a/python/ert_logger/CMakeLists.txt b/python/ert_logger/CMakeLists.txt deleted file mode 100644 index c74f2e5121..0000000000 --- a/python/ert_logger/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(PYTHON_SOURCES - __init__.py -) - -add_python_package("python.ert_logger" ${PYTHON_INSTALL_PREFIX}/ert_logger "${PYTHON_SOURCES}" True) diff --git a/python/job_runner/CMakeLists.txt b/python/job_runner/CMakeLists.txt deleted file mode 100644 index 5044dbcca6..0000000000 --- a/python/job_runner/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -set(PYTHON_SOURCES - __init__.py - cli.py - job.py - runner.py -) - -add_python_package("python.job_runner" ${PYTHON_INSTALL_PREFIX}/job_runner "${PYTHON_SOURCES}" True) - -add_subdirectory( io ) -add_subdirectory( reporting ) -add_subdirectory( util ) diff --git a/python/job_runner/io/CMakeLists.txt b/python/job_runner/io/CMakeLists.txt deleted file mode 100644 index ce9b6d6bff..0000000000 --- a/python/job_runner/io/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(PYTHON_SOURCES - __init__.py -) - - -add_python_package("python.job_runner.io" ${PYTHON_INSTALL_PREFIX}/job_runner/io "${PYTHON_SOURCES}" True) diff --git a/python/job_runner/job_dispatch.py b/python/job_runner/job_dispatch.py new file mode 100644 index 0000000000..488529a386 --- /dev/null +++ b/python/job_runner/job_dispatch.py @@ -0,0 +1,20 @@ +import os +import signal +import sys + +from job_runner.cli import main as job_runner_main + + +def sigterm_handler(_signo, _stack_frame): + signal.signal(signal.SIGTERM, signal.SIG_DFL) + os.kill(0, signal.SIGTERM) + + +def main(): + os.nice(19) + signal.signal(signal.SIGTERM, sigterm_handler) + job_runner_main(sys.argv) + + +if __name__ == "__main__": + main() diff --git a/python/job_runner/reporting/CMakeLists.txt b/python/job_runner/reporting/CMakeLists.txt deleted file mode 100644 index dfa9ee61ff..0000000000 --- a/python/job_runner/reporting/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -set(PYTHON_SOURCES - __init__.py - file.py - interactive.py - network.py - message.py -) - - -add_python_package("python.job_runner.reporting" ${PYTHON_INSTALL_PREFIX}/job_runner/reporting "${PYTHON_SOURCES}" True) diff --git a/python/job_runner/util/CMakeLists.txt b/python/job_runner/util/CMakeLists.txt deleted file mode 100644 index 0d122b70f1..0000000000 --- a/python/job_runner/util/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -set(PYTHON_SOURCES - __init__.py -) - -add_python_package("python.job_runner.util" ${PYTHON_INSTALL_PREFIX}/job_runner/util "${PYTHON_SOURCES}" True) diff --git a/python/res/CMakeLists.txt b/python/res/CMakeLists.txt deleted file mode 100644 index 79f7a59b43..0000000000 --- a/python/res/CMakeLists.txt +++ /dev/null @@ -1,23 +0,0 @@ -set(PYTHON_SOURCES - __init__.py -) -add_python_package("python.res" ${PYTHON_INSTALL_PREFIX}/res "${PYTHON_SOURCES}" True) - -set( ERT_SITE_INIT_TARGET "" CACHE STRING "Name of optional external site init module for top level res package") -if (EXISTS ${ERT_SITE_INIT_TARGET}) - message(STATUS "Using res site init module: ${ERT_SITE_INIT_TARGET}") - install(FILES ${ERT_SITE_INIT_TARGET} DESTINATION ${PYTHON_INSTALL_PREFIX}/res RENAME res_site_init.py) -endif() -add_subdirectory(util) -add_subdirectory(analysis) -add_subdirectory(config) -add_subdirectory(enkf) -add_subdirectory(job_queue) -add_subdirectory(sched) -add_subdirectory(test) -add_subdirectory(simulator) -add_subdirectory(fm) - -configure_file(res_lib_info_build.py.in ${PROJECT_BINARY_DIR}/${PYTHON_INSTALL_PREFIX}/res/__res_lib_info.py ) -configure_file(res_lib_info_install.py.in ${PROJECT_BINARY_DIR}/${PYTHON_INSTALL_PREFIX}/res_lib_info_install.py ) -install(FILES ${PROJECT_BINARY_DIR}/${PYTHON_INSTALL_PREFIX}/res_lib_info_install.py DESTINATION ${PYTHON_INSTALL_PREFIX}/res RENAME __res_lib_info.py) diff --git a/python/res/__init__.py b/python/res/__init__.py index 9aa06383b6..62a91f8b1e 100644 --- a/python/res/__init__.py +++ b/python/res/__init__.py @@ -15,77 +15,74 @@ # for more details. """ Ert - Ensemble Reservoir Tool - a package for reservoir modeling. - -The res package itself has no code, but contains several subpackages: - -res.job_queue: - -The res package is based on wrapping the libriaries from the ERT C -code with ctypes; an essential part of ctypes approach is to load the -shared libraries with the ctypes.CDLL() function. The ctypes.CDLL() -function uses the standard methods of the operating system, -i.e. standard locations configured with ld.so.conf and the environment -variable LD_LIBRARY_PATH. - """ import os.path import sys +import platform +import ecl import warnings warnings.filterwarnings(action="always", category=DeprecationWarning, module=r"res|ert") -from cwrap import load as cwrapload from cwrap import Prototype -try: - import ert_site_init -except ImportError: - pass +from ._version import version as __version__ + -res_lib_path = None -ert_so_version = "" -__version__ = "0.0.0" +def _load_lib(): + import ctypes + # Find and dlopen libres + lib_path = os.path.join(os.path.dirname(__file__), ".libs") + if not os.path.isdir(lib_path): + lib_path = "" -# 1. Try to load the __res_lib_info module; this module has been -# configured by cmake during the build configuration process. The -# module should contain the variable lib_path pointing to the -# directory with shared object files. -try: - from .__res_lib_info import ResLibInfo + if platform.system() == "Linux": + lib_path = os.path.join(lib_path, "libres.so") + elif platform.system() == "Darwin": + lib_path = os.path.join(lib_path, "libres.dylib") + else: + raise NotImplementedError("Invalid platform") - res_lib_path = ResLibInfo.lib_path - ert_so_version = ResLibInfo.so_version - __version__ = ResLibInfo.__version__ -except ImportError: - pass -except AttributeError: - pass + lib = ctypes.CDLL(lib_path, ctypes.RTLD_GLOBAL) + # Configure site_config to be a ctypes.CFUNCTION with type: + # void set_site_config(char *); + site_config = lib.set_site_config + site_config.restype = None + site_config.argtypes = (ctypes.c_char_p,) -# Check that the final ert_lib_path setting corresponds to an existing -# directory. -if res_lib_path: - if not os.path.isabs(res_lib_path): - res_lib_path = os.path.abspath( - os.path.join(os.path.dirname(__file__), res_lib_path) - ) + # Find share/ert + from pathlib import Path - if not os.path.isdir(res_lib_path): - res_lib_path = None + path = Path(__file__).parent + for p in path.parents: + npath = p / "share" / "ert" / "site-config" + if npath.is_file(): + path = npath + break + else: + raise ImportError("Could not find `share/ert/site-config`") + # Set site-config to point to [PREFIX]/share/ert/site-config + site_config(str(path).encode("utf-8")) -# This load() function is *the* function actually loading shared -# libraries. + # Configure set_analysis_modules_dir to be a ctypes.CFUNCTION with type: + # void set_analysis_modules_dir(char *); + set_analysis_modules_dir = lib.set_analysis_modules_dir + set_analysis_modules_dir.restype = None + set_analysis_modules_dir.argtypes = (ctypes.c_char_p,) + # Set analysis modules dir to be [CURRENT DIR]/.libs + path = os.path.join(os.path.dirname(__file__), ".libs") + set_analysis_modules_dir(path.encode("utf-8")) -def load(name): - return cwrapload(name, path=res_lib_path, so_version=ert_so_version) + return lib class ResPrototype(Prototype): - lib = load("libres") + lib = _load_lib() def __init__(self, prototype, bind=True): super(ResPrototype, self).__init__(ResPrototype.lib, prototype, bind=bind) diff --git a/python/res/analysis/CMakeLists.txt b/python/res/analysis/CMakeLists.txt deleted file mode 100644 index 0cf15aa6bb..0000000000 --- a/python/res/analysis/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -set(PYTHON_SOURCES - __init__.py - analysis_module.py - linalg.py -) - -add_python_package("python.res.analysis." ${PYTHON_INSTALL_PREFIX}/res/analysis/ "${PYTHON_SOURCES}" True) - -add_subdirectory(enums) diff --git a/python/res/analysis/analysis_module.py b/python/res/analysis/analysis_module.py index e9601c4efe..7d37e120b3 100644 --- a/python/res/analysis/analysis_module.py +++ b/python/res/analysis/analysis_module.py @@ -17,7 +17,9 @@ from cwrap import BaseCClass from ecl.util.util.rng import RandomNumberGenerator from res import ResPrototype +from os import path +import res from res.util import Matrix @@ -163,6 +165,8 @@ def free(self): self._free() def __repr__(self): + if not self: + return repr(None) nm = self.name() tn = self.getTableName() ln = self.getLibName() diff --git a/python/res/analysis/enums/CMakeLists.txt b/python/res/analysis/enums/CMakeLists.txt deleted file mode 100644 index ef72456829..0000000000 --- a/python/res/analysis/enums/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -set(PYTHON_SOURCES - __init__.py - analysis_module_options_enum.py - analysis_module_load_status_enum.py -) - -add_python_package("python.res.analysis.enums" ${PYTHON_INSTALL_PREFIX}/res/analysis/enums "${PYTHON_SOURCES}" True) - - - diff --git a/python/res/config/CMakeLists.txt b/python/res/config/CMakeLists.txt deleted file mode 100644 index 19baa2d1a8..0000000000 --- a/python/res/config/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -set(PYTHON_SOURCES - __init__.py - schema_item.py - config_error.py - config_parser.py - config_content.py - config_settings.py - content_type_enum.py - unrecognized_enum.py - config_path_elm.py -) - -add_python_package("python.res.config" ${PYTHON_INSTALL_PREFIX}/res/config "${PYTHON_SOURCES}" True) - diff --git a/python/res/enkf/CMakeLists.txt b/python/res/enkf/CMakeLists.txt deleted file mode 100644 index e1770a785e..0000000000 --- a/python/res/enkf/CMakeLists.txt +++ /dev/null @@ -1,60 +0,0 @@ -set(PYTHON_SOURCES - __init__.py - subst_config.py - res_config.py - rng_config.py - active_list.py - analysis_config.py - analysis_iter_config.py - ecl_config.py - enkf_fs.py - enkf_fs_manager.py - enkf_linalg.py - enkf_main.py - enkf_obs.py - enkf_simulation_runner.py - enkf_state.py - ensemble_config.py - ert_run_context.py - ert_template.py - ert_templates.py - ert_workflow_list.py - key_manager.py - local_config.py - local_dataset.py - local_ministep.py - local_obsdata.py - local_obsdata_node.py - local_updatestep.py - meas_block.py - meas_data.py - model_config.py - node_id.py - obs_block.py - obs_data.py - hook_manager.py - hook_workflow.py - run_arg.py - runpath_list.py - site_config.py - queue_config.py - state_map.py - summary_key_matcher.py - summary_key_set.py - forward_load_context.py - es_update.py - log_config.py - config_keys.py - enkf_defaults.py -) - -add_python_package("python.res.enkf" ${PYTHON_INSTALL_PREFIX}/res/enkf "${PYTHON_SOURCES}" True) - -add_subdirectory(config) -add_subdirectory(data) -add_subdirectory(enums) -add_subdirectory(export) -add_subdirectory(observations) -add_subdirectory(plot) -add_subdirectory(plot_data) -add_subdirectory(util) diff --git a/python/res/enkf/config/CMakeLists.txt b/python/res/enkf/config/CMakeLists.txt deleted file mode 100644 index 7e1e75e343..0000000000 --- a/python/res/enkf/config/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -set(PYTHON_SOURCES - __init__.py - enkf_config_node.py - field_type_enum.py - field_config.py - gen_data_config.py - gen_kw_config.py - summary_config.py - ext_param_config.py -) - -add_python_package("python.res.enkf.config" ${PYTHON_INSTALL_PREFIX}/res/enkf/config "${PYTHON_SOURCES}" True) - diff --git a/python/res/enkf/data/CMakeLists.txt b/python/res/enkf/data/CMakeLists.txt deleted file mode 100644 index aa55d5fa77..0000000000 --- a/python/res/enkf/data/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -set(PYTHON_SOURCES - __init__.py - enkf_node.py - field.py - gen_data.py - gen_kw.py - ext_param.py - summary.py -) - -add_python_package("python.res.enkf.data" ${PYTHON_INSTALL_PREFIX}/res/enkf/data "${PYTHON_SOURCES}" True) - diff --git a/python/res/enkf/enums/CMakeLists.txt b/python/res/enkf/enums/CMakeLists.txt deleted file mode 100644 index 5a8900c587..0000000000 --- a/python/res/enkf/enums/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ -set(PYTHON_SOURCES - __init__.py - enkf_field_file_format_enum.py - enkf_fs_type_enum.py - enkf_init_modes_enum.py - enkf_obs_impl_type_enum.py - enkf_run_enum.py - enkf_truncation_type.py - enkf_var_type_enum.py - ert_impl_type_enum.py - load_fail_type_enum.py - realization_state_enum.py - gen_data_file_type_enum.py - active_mode_enum.py - hook_runtime_enum.py -) - -add_python_package("python.res.enkf.enums" ${PYTHON_INSTALL_PREFIX}/res/enkf/enums "${PYTHON_SOURCES}" True) diff --git a/python/res/enkf/export/CMakeLists.txt b/python/res/enkf/export/CMakeLists.txt deleted file mode 100644 index fdcd667b57..0000000000 --- a/python/res/enkf/export/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -set(PYTHON_SOURCES - __init__.py - arg_loader.py - design_matrix_reader.py - gen_data_collector.py - gen_data_observation_collector.py - gen_kw_collector.py - misfit_collector.py - summary_collector.py - summary_observation_collector.py -) - -add_python_package("python.res.enkf.export" ${PYTHON_INSTALL_PREFIX}/res/enkf/export "${PYTHON_SOURCES}" True) - diff --git a/python/res/enkf/observations/CMakeLists.txt b/python/res/enkf/observations/CMakeLists.txt deleted file mode 100644 index 96b2047595..0000000000 --- a/python/res/enkf/observations/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -set(PYTHON_SOURCES - __init__.py - block_observation.py - block_data_config.py - gen_observation.py - obs_vector.py - summary_observation.py -) - -add_python_package("python.res.enkf.observations" ${PYTHON_INSTALL_PREFIX}/res/enkf/observations "${PYTHON_SOURCES}" True) - diff --git a/python/res/enkf/plot/CMakeLists.txt b/python/res/enkf/plot/CMakeLists.txt deleted file mode 100644 index c28f9a1ddf..0000000000 --- a/python/res/enkf/plot/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -set(PYTHON_SOURCES - __init__.py - block_observation_data_fetcher.py - data_fetcher.py - ensemble_block_data_fetcher.py - ensemble_data_fetcher.py - ensemble_gen_data_fetcher.py - ensemble_gen_kw_fetcher.py - observation_data_fetcher.py - observation_gen_data_fetcher.py - refcase_data_fetcher.py -) - -add_python_package("python.res.enkf.plot" ${PYTHON_INSTALL_PREFIX}/res/enkf/plot "${PYTHON_SOURCES}" True) - diff --git a/python/res/enkf/plot_data/CMakeLists.txt b/python/res/enkf/plot_data/CMakeLists.txt deleted file mode 100644 index 300a562f2b..0000000000 --- a/python/res/enkf/plot_data/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -set(PYTHON_SOURCES - __init__.py - ensemble_plot_data.py - ensemble_plot_data_vector.py - ensemble_plot_gen_data.py - ensemble_plot_gen_data_vector.py - ensemble_plot_gen_kw.py - ensemble_plot_gen_kw_vector.py - plot_block_data.py - plot_block_data_loader.py - plot_block_vector.py -) - -add_python_package("python.res.enkf.plot_data" ${PYTHON_INSTALL_PREFIX}/res/enkf/plot_data "${PYTHON_SOURCES}" True) - diff --git a/python/res/enkf/util/CMakeLists.txt b/python/res/enkf/util/CMakeLists.txt deleted file mode 100644 index 268ce291ee..0000000000 --- a/python/res/enkf/util/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -set(PYTHON_SOURCES - __init__.py - time_map.py -) - -add_python_package("python.res.enkf.util" ${PYTHON_INSTALL_PREFIX}/res/enkf/util "${PYTHON_SOURCES}" True) - diff --git a/python/res/fm/CMakeLists.txt b/python/res/fm/CMakeLists.txt deleted file mode 100644 index b1a06dcb16..0000000000 --- a/python/res/fm/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -set(PYTHON_SOURCES - __init__.py -) - - - -add_subdirectory(ecl) -add_subdirectory(rms) -add_subdirectory(shell) -add_subdirectory(templating) -add_python_package("python.res.fm" ${PYTHON_INSTALL_PREFIX}/res/fm "${PYTHON_SOURCES}" True) - diff --git a/python/res/fm/ecl/CMakeLists.txt b/python/res/fm/ecl/CMakeLists.txt deleted file mode 100644 index c3d4381835..0000000000 --- a/python/res/fm/ecl/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ -set(PYTHON_SOURCES - __init__.py - ecl_config.py - ecl_run.py - script.py -) - -set(_build_path "${PROJECT_BINARY_DIR}/${PYTHON_INSTALL_PREFIX}/res/fm/ecl") -set(_ecl100_config_file "${_build_path}/ecl100_config.yml") -set(_ecl300_config_file "${_build_path}/ecl300_config.yml") - -configure_file( ecl100_config.yml "${PROJECT_BINARY_DIR}/${PYTHON_INSTALL_PREFIX}/res/fm/ecl/ecl100_config.yml" COPYONLY) -configure_file( ecl300_config.yml "${PROJECT_BINARY_DIR}/${PYTHON_INSTALL_PREFIX}/res/fm/ecl/ecl300_config.yml" COPYONLY) - -add_python_package("python.res.fm.ecl" ${PYTHON_INSTALL_PREFIX}/res/fm/ecl "${PYTHON_SOURCES}" True) - -install_example(${_ecl100_config_file} "${PYTHON_INSTALL_PREFIX}/res/fm/ecl") -install_example(${_ecl300_config_file} "${PYTHON_INSTALL_PREFIX}/res/fm/ecl") diff --git a/python/res/fm/rms/CMakeLists.txt b/python/res/fm/rms/CMakeLists.txt deleted file mode 100644 index 97916ea264..0000000000 --- a/python/res/fm/rms/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -set(PYTHON_SOURCES - __init__.py - rms_config.py - rms_run.py -) - -add_python_package("python.res.fm.rms" ${PYTHON_INSTALL_PREFIX}/res/fm/rms "${PYTHON_SOURCES}" True) - -install_example("rms_config.yml" "${PYTHON_INSTALL_PREFIX}/res/fm/rms") diff --git a/python/res/fm/shell/CMakeLists.txt b/python/res/fm/shell/CMakeLists.txt deleted file mode 100644 index f79af93ab7..0000000000 --- a/python/res/fm/shell/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -set(PYTHON_SOURCES - __init__.py - shell.py -) - -add_python_package("python.res.fm.shell" ${PYTHON_INSTALL_PREFIX}/res/fm/shell "${PYTHON_SOURCES}" True) - diff --git a/python/res/fm/templating/CMakeLists.txt b/python/res/fm/templating/CMakeLists.txt deleted file mode 100644 index 2dc811ad97..0000000000 --- a/python/res/fm/templating/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(PYTHON_SOURCES - __init__.py - template_render.py - ) - -add_python_package("python.res.fm.templating" ${PYTHON_INSTALL_PREFIX}/res/fm/templating "${PYTHON_SOURCES}" True) diff --git a/python/res/job_queue/CMakeLists.txt b/python/res/job_queue/CMakeLists.txt deleted file mode 100644 index 015e873aec..0000000000 --- a/python/res/job_queue/CMakeLists.txt +++ /dev/null @@ -1,26 +0,0 @@ -set(PYTHON_SOURCES - __init__.py - driver.py - ert_plugin.py - ert_script.py - ext_job.py - ext_joblist.py - external_ert_script.py - forward_model.py - forward_model_status.py - function_ert_script.py - job.py - job_status_type_enum.py - run_status_type_enum.py - thread_status_type_enum.py - queue.py - job_queue_manager.py - workflow.py - workflow_job.py - workflow_joblist.py - workflow_runner.py - environment_varlist.py - job_queue_node.py -) - -add_python_package("python.res.job_queue" ${PYTHON_INSTALL_PREFIX}/res/job_queue "${PYTHON_SOURCES}" True) diff --git a/python/res/res_lib_info_build.py.in b/python/res/res_lib_info_build.py.in deleted file mode 100644 index ced56e0217..0000000000 --- a/python/res/res_lib_info_build.py.in +++ /dev/null @@ -1,8 +0,0 @@ - -class ResLibInfo(object): - lib_path = "${LIBRARY_OUTPUT_PATH}" - so_version = ".${RES_VERSION_MAJOR}.${RES_VERSION_MINOR}" - __version__ = "${RES_VERSION_MAJOR}.${RES_VERSION_MINOR}.${RES_VERSION_MICRO}" - - def __init__(self): - pass diff --git a/python/res/res_lib_info_install.py.in b/python/res/res_lib_info_install.py.in deleted file mode 100644 index ee541c2602..0000000000 --- a/python/res/res_lib_info_install.py.in +++ /dev/null @@ -1,8 +0,0 @@ - -class ResLibInfo(object): - lib_path = "../../../../${CMAKE_INSTALL_LIBDIR}" - so_version = ".${RES_VERSION_MAJOR}.${RES_VERSION_MINOR}" - __version__ = "${RES_VERSION_MAJOR}.${RES_VERSION_MINOR}.${RES_VERSION_MICRO}" - - def __init__(self): - pass diff --git a/python/res/sched/CMakeLists.txt b/python/res/sched/CMakeLists.txt deleted file mode 100644 index e01edf5937..0000000000 --- a/python/res/sched/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -set(PYTHON_SOURCES - __init__.py - history.py - history_source_enum.py -) - -add_python_package("python.res.sched" ${PYTHON_INSTALL_PREFIX}/res/sched "${PYTHON_SOURCES}" True) - diff --git a/python/res/simulator/CMakeLists.txt b/python/res/simulator/CMakeLists.txt deleted file mode 100644 index a8e5826280..0000000000 --- a/python/res/simulator/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -set(PYTHON_SOURCES - __init__.py - batch_simulator.py - batch_simulator_context.py - simulation_context.py) - -add_python_package("python.res.simulator" ${PYTHON_INSTALL_PREFIX}/res/simulator "${PYTHON_SOURCES}" True) diff --git a/python/res/test/CMakeLists.txt b/python/res/test/CMakeLists.txt deleted file mode 100644 index 7b6cc98a0b..0000000000 --- a/python/res/test/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -set(PYTHON_SOURCES - __init__.py - ert_test_context.py -) - -add_python_package("python.res.test" ${PYTHON_INSTALL_PREFIX}/res/test "${PYTHON_SOURCES}" True) -add_subdirectory(synthesizer) diff --git a/python/res/test/synthesizer/CMakeLists.txt b/python/res/test/synthesizer/CMakeLists.txt deleted file mode 100644 index 5a9d7f5cf2..0000000000 --- a/python/res/test/synthesizer/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -set(PYTHON_SOURCES - __init__.py - oil_simulator.py - perlin.py - prime_generator.py - shaped_perlin.py -) - - -add_python_package("python.res.test.synthesizer" ${PYTHON_INSTALL_PREFIX}/res/test/synthesizer "${PYTHON_SOURCES}" True) diff --git a/python/res/util/CMakeLists.txt b/python/res/util/CMakeLists.txt deleted file mode 100644 index a6837c4b0e..0000000000 --- a/python/res/util/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -set(PYTHON_SOURCES - __init__.py - log.py - matrix.py - path_format.py - res_log.py - res_version.py - stat.py - subprocess.py - substitution_list.py - ui_return.py -) - -add_python_package("python.res.util" ${PYTHON_INSTALL_PREFIX}/res/util "${PYTHON_SOURCES}" True) - -add_subdirectory(enums) diff --git a/python/res/util/enums/CMakeLists.txt b/python/res/util/enums/CMakeLists.txt deleted file mode 100644 index c413e8b447..0000000000 --- a/python/res/util/enums/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -set(PYTHON_SOURCES - __init__.py - ui_return_status_enum.py - message_level_enum.py - llsq_result_enum.py -) - -add_python_package("python.res.util.enums" ${PYTHON_INSTALL_PREFIX}/res/util/enums "${PYTHON_SOURCES}" True) diff --git a/python/res/util/stat.py b/python/res/util/stat.py index cf43d23c00..6100f3c260 100644 --- a/python/res/util/stat.py +++ b/python/res/util/stat.py @@ -20,6 +20,7 @@ from cwrap import PrototypeError from res import ResPrototype +from ecl import EclPrototype from res.util import LLSQResultEnum from res.util import Matrix diff --git a/python/tests/global/__init__.py b/python/tests/global/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/python/tests/global/test_import.py b/python/tests/global/test_import.py deleted file mode 100644 index d044cc12e0..0000000000 --- a/python/tests/global/test_import.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (C) 2017 Equinor ASA, Norway. -# -# This file is part of ERT - Ensemble based Reservoir Tool. -# -# ERT is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# ERT is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. -# -# See the GNU General Public License at -# for more details. - -import os -import sys - - -from ecl.util.test import ImportTestCase - - -class ImportRes(ImportTestCase): - def test_import(self): - self.assertTrue(self.import_package("res")) diff --git a/python/tests/job_runner/test_job_dispatch.py b/python/tests/job_runner/test_job_dispatch.py index 55b718abca..7f5747a15e 100644 --- a/python/tests/job_runner/test_job_dispatch.py +++ b/python/tests/job_runner/test_job_dispatch.py @@ -5,6 +5,10 @@ import sys import time import unittest +import importlib +from subprocess import Popen +from textwrap import dedent + import psutil from subprocess import Popen @@ -64,36 +68,31 @@ def test_terminate_jobs(self): "ert_pid": "", } - job_dispatch_script = os.path.realpath( - os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "../../../bin/job_dispatch.py", - ) - ) - with open("jobs.json", "w") as f: f.write(json.dumps(self.job_list)) - # Required to execute job_dispatch in separate process group by - # os.setsid moves the current process out of the current group - with open("job_dispatch_executer", "w") as f: - f.write( - "#!/usr/bin/env python\n" - "import os, sys\n" - "os.setsid()\n" - "os.execv(sys.argv[1], sys.argv[1:])\n" - "\n" + # macOS doesn't provide /usr/bin/setsid, so we roll our own + with open("setsid", "w") as f: + f.write( + dedent( + """\ + #!/usr/bin/env python + import os + import sys + os.setsid() + os.execvp(sys.argv[1], sys.argv[1:]) + """ ) - os.chmod( - "job_dispatch_executer", stat.S_IRWXU | stat.S_IRWXO | stat.S_IRWXG ) + os.chmod("setsid", 0o755) - current_dir = os.path.realpath(os.curdir) + job_dispatch_script = importlib.util.find_spec("job_runner.job_dispatch").origin job_dispatch_process = Popen( [ - os.path.realpath("job_dispatch_executer"), + os.getcwd() + "/setsid", + sys.executable, job_dispatch_script, - current_dir, + os.getcwd(), ] ) @@ -124,7 +123,6 @@ def test_job_dispatch_kills_itself_after_unsuccessful_job(self): @tmpdir(None) def test_job_dispatch_run_subset_specified_as_parmeter(self): - with open("dummy_executable", "w") as f: f.write( "#!/usr/bin/env python\n" @@ -208,32 +206,28 @@ def test_job_dispatch_run_subset_specified_as_parmeter(self): with open("jobs.json", "w") as f: f.write(json.dumps(self.job_list)) - current_dir = os.path.realpath(os.curdir) - - # Required to execute job_dispatch in separate process group by - # os.setsid moves the current process out of the current group - with open("job_dispatch_executer", "w") as f: + # macOS doesn't provide /usr/bin/setsid, so we roll our own + with open("setsid", "w") as f: f.write( - "#!/usr/bin/env python\n" - "import os, sys\n" - "os.setsid()\n" - "os.execv(sys.argv[1], sys.argv[1:])\n" - "\n" - ) - os.chmod("job_dispatch_executer", stat.S_IRWXU | stat.S_IRWXO | stat.S_IRWXG) - - job_dispatch_script = os.path.realpath( - os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "../../../bin/job_dispatch.py", + dedent( + """\ + #!/usr/bin/env python + import os + import sys + os.setsid() + os.execvp(sys.argv[1], sys.argv[1:]) + """ + ) ) - ) + os.chmod("setsid", 0o755) + job_dispatch_script = importlib.util.find_spec("job_runner.job_dispatch").origin job_dispatch_process = Popen( [ - os.path.realpath("job_dispatch_executer"), + os.getcwd() + "/setsid", + sys.executable, job_dispatch_script, - current_dir, + os.getcwd(), "job_B", "job_C", ] @@ -241,6 +235,6 @@ def test_job_dispatch_run_subset_specified_as_parmeter(self): job_dispatch_process.wait() - assert not os.path.isfile("{}/job_A.out".format(current_dir)) - assert os.path.isfile("{}/job_B.out".format(current_dir)) - assert os.path.isfile("{}/job_C.out".format(current_dir)) + assert not os.path.isfile("job_A.out") + assert os.path.isfile("job_B.out") + assert os.path.isfile("job_C.out") diff --git a/python/tests/res/analysis/test_analysis_module.py b/python/tests/res/analysis/test_analysis_module.py index 4a64297e6e..6a9b17a96c 100644 --- a/python/tests/res/analysis/test_analysis_module.py +++ b/python/tests/res/analysis/test_analysis_module.py @@ -25,6 +25,7 @@ from ecl.util.enums import RngAlgTypeEnum, RngInitModeEnum from ecl.util.util.rng import RandomNumberGenerator from res.util import Matrix +from os.path import basename class AnalysisModuleTest(ResTest): @@ -52,7 +53,7 @@ def test_load_status_enum(self): def test_analysis_module(self): am = self.createAnalysisModule() - self.assertEqual(am.getLibName(), self.libname) + self.assertEqual(basename(am.getLibName()), self.libname) self.assertFalse(am.getInternal()) diff --git a/python/tests/res/config/test_config.py b/python/tests/res/config/test_config.py index ab37e9474f..e6000cc300 100755 --- a/python/tests/res/config/test_config.py +++ b/python/tests/res/config/test_config.py @@ -19,19 +19,15 @@ from ecl.util.test import TestAreaContext from tests import ResTest -from res import load as resload +from res import ResPrototype from res.config import UnrecognizedEnum, SchemaItem from res.config import ContentTypeEnum, ContentItem, ContentNode from res.config import ConfigContent, ConfigParser, ConfigSettings -class TestConfigPrototype(Prototype): - lib = resload("libres") - +class TestConfigPrototype(ResPrototype): def __init__(self, prototype, bind=False): - super(TestConfigPrototype, self).__init__( - TestConfigPrototype.lib, prototype, bind=bind - ) + super(TestConfigPrototype, self).__init__(prototype, bind=bind) # Adding extra functions to the ConfigContent object for the ability diff --git a/python/tests/res/fm/test_rms_config.py b/python/tests/res/fm/test_rms_config.py index 708b578563..5879b941c6 100644 --- a/python/tests/res/fm/test_rms_config.py +++ b/python/tests/res/fm/test_rms_config.py @@ -37,10 +37,7 @@ def test_load(self): with self.assertRaises(IOError): conf = RMSConfig() - self.monkeypatch.setenv( - "RMS_SITE_CONFIG", - os.path.join(self.SOURCE_ROOT, "python/res/fm/rms/rms_config.yml"), - ) + self.monkeypatch.setenv("RMS_SITE_CONFIG", RMSConfig.DEFAULT_CONFIG_FILE) conf = RMSConfig() with self.assertRaises(OSError): diff --git a/python/tests/res/fm/test_rms_run.py b/python/tests/res/fm/test_rms_run.py index 965570348b..d7a9fca320 100644 --- a/python/tests/res/fm/test_rms_run.py +++ b/python/tests/res/fm/test_rms_run.py @@ -99,10 +99,6 @@ def tearDown(self): self.monkeypatch.undo() def test_create(self): - self.monkeypatch.setenv( - "RMS_SITE_CONFIG", - os.path.join(self.SOURCE_ROOT, "python/res/fm/rms/rms_config.yml"), - ) with self.assertRaises(OSError): r = RMSRun(0, "/project/does/not/exist", "workflow") diff --git a/python/tests/res/job_queue/test_workflow_job.py b/python/tests/res/job_queue/test_workflow_job.py index 81ae21b347..15095c65e9 100644 --- a/python/tests/res/job_queue/test_workflow_job.py +++ b/python/tests/res/job_queue/test_workflow_job.py @@ -1,21 +1,16 @@ import ecl import res.enkf # noqa +from res import ResPrototype from res.job_queue import WorkflowJob from tests import ResTest from ecl.util.test import TestAreaContext from .workflow_common import WorkflowCommon -from cwrap import Prototype - - -class _TestWorkflowJobPrototype(Prototype): - lib = res.load("libres") +class _TestWorkflowJobPrototype(ResPrototype): def __init__(self, prototype, bind=True): - super(_TestWorkflowJobPrototype, self).__init__( - _TestWorkflowJobPrototype.lib, prototype, bind=bind - ) + super(_TestWorkflowJobPrototype, self).__init__(prototype, bind=bind) class WorkflowJobTest(ResTest): diff --git a/python/tests/res/util/test_res_log.py b/python/tests/res/util/test_res_log.py index f8af4a56e8..c54a7d4ec8 100644 --- a/python/tests/res/util/test_res_log.py +++ b/python/tests/res/util/test_res_log.py @@ -20,7 +20,7 @@ def test_log(self): self.assertTrue(len(text) > 0) self.assertTrue(message in text[-1]) - def test_getFilename(self): + def test_get_filename(self): with TestAreaContext("python/res_log/log") as work_area: test_log_filename = "log_test_file.txt" ResLog.init(1, test_log_filename, True) @@ -28,19 +28,3 @@ def test_getFilename(self): ResLog.log(1, message) self.assertEqual(ResLog.getFilename(), test_log_filename) - - def test_log(self): - with TestAreaContext("python/log"): - logh = Log("logfile", MessageLevelEnum.LOG_DEBUG) - - os.mkdir("read_only") - os.chmod("read_only", 0o500) - with self.assertRaises(IOError): - logh = Log("read_only/logfile.txt", MessageLevelEnum.LOG_DEBUG) - - def test_init_perm_denied(self): - with TestAreaContext("python/res_log"): - os.mkdir("read_only") - os.chmod("read_only", 0o500) - with self.assertRaises(IOError): - ResLog.init(1, "read_only/logfile.txt", True) diff --git a/python/tests/share/CMakeLists.txt b/python/tests/share/CMakeLists.txt deleted file mode 100644 index f8205a8328..0000000000 --- a/python/tests/share/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -set(TEST_SOURCES - __init__.py - test_synthesizer.py -) - -add_python_package("python.tests.share" ${PYTHON_INSTALL_PREFIX}/tests/share "${TEST_SOURCES}" False) diff --git a/requirements.txt b/requirements.txt index b235af6262..b431a62dd4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,17 @@ -# The python wrappers to libecl is required by libres, but there is -# not yet any pip package for that, it is therefor just mentioned here -# as a comment, and must be installed by building libecl from source. -# libecl +# From pyproject.toml +ecl +setuptools_scm +wheel +scikit-build +cmake +ninja -cwrap -numpy>=1.12.0 +# From setup.py install_requires +ecl +futures +jinja2 +numpy pandas +psutil pyyaml -jinja2 requests -psutil -futures -future diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000..f11c2c9c9a --- /dev/null +++ b/setup.py @@ -0,0 +1,92 @@ +import os +import setuptools +import skbuild +from setuptools_scm import get_version + + +def get_ecl_include(): + from ecl import get_include + + return get_include() + + +def get_data_files(): + data_files = [] + for root, _, files in os.walk("share/ert"): + data_files.append((root, [os.path.join(root, name) for name in files])) + return data_files + + +version = get_version( + relative_to=__file__, + write_to="python/res/_version.py", + write_to_template='# config: utf-8\n#\nversion = "{version}"', # black-compatible version string +) + +with open("README.md") as f: + long_description = f.read() + + +skbuild.setup( + name="equinor-libres", + author="Equinor ASA", + author_email="fg_sib-scout@equinor.com", + description="Part of the Ensemble based Reservoir Tool (ERT)", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/equinor/libres", + packages=setuptools.find_packages( + where="python", + exclude=["doc", "*.tests", "*.tests.*", "tests.*", "tests"], + ), + package_dir={"": "python"}, + package_data={ + "res": [ + "fm/rms/rms_config.yml", + "fm/ecl/ecl300_config.yml", + "fm/ecl/ecl100_config.yml", + ] + }, + data_files=get_data_files(), + license="GPL-3.0", + platforms="any", + install_requires=[ + "ecl", + "futures", + "jinja2", + "numpy", + "pandas", + "psutil", + "pyyaml", + "requests", + ], + entry_points={ + "console_scripts": ["job_dispatch.py = job_runner.job_dispatch:main"] + }, + cmake_args=[ + "-DRES_VERSION=" + version, + "-DECL_INCLUDE_DIRS=" + get_ecl_include(), + # we can safely pass OSX_DEPLOYMENT_TARGET as it's ignored on + # everything not OS X. We depend on C++11, which makes our minimum + # supported OS X release 10.9 + "-DCMAKE_OSX_DEPLOYMENT_TARGET=10.9", + "-DCMAKE_INSTALL_LIBDIR=python/res/.libs", + ], + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Environment :: Other Environment", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", + "Natural Language :: English", + "Programming Language :: Python", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Physics", + "Topic :: Software Development :: Libraries", + "Topic :: Utilities", + ], + version=version, +) diff --git a/share/ert/site-config b/share/ert/site-config index 0184ecd60c..3d896bd4bd 100644 --- a/share/ert/site-config +++ b/share/ert/site-config @@ -4,7 +4,7 @@ WORKFLOW_JOB_DIRECTORY workflows/jobs/shell WORKFLOW_JOB_DIRECTORY workflows/jobs/internal/config WORKFLOW_JOB_DIRECTORY workflows/jobs/internal-gui/config -JOB_SCRIPT ../../bin/job_dispatch.py +JOB_SCRIPT job_dispatch.py INSTALL_JOB_DIRECTORY forward-models/res INSTALL_JOB_DIRECTORY forward-models/shell INSTALL_JOB_DIRECTORY forward-models/templating diff --git a/testjenkins.sh b/testjenkins.sh index b72e12d378..9cfd9a2a05 100644 --- a/testjenkins.sh +++ b/testjenkins.sh @@ -21,7 +21,8 @@ build_and_test () { rm -rf jenkinsbuild mkdir jenkinsbuild run setup - run build_ecl + run build_libecl + run build_libres run build_res run run_ctest run run_pytest_equinor @@ -29,53 +30,43 @@ build_and_test () { } setup () { - run source_build_tools run setup_variables - run clone_repos run create_directories run create_virtualenv + run source_build_tools + run clone_repos + run setup_testdir } -build_ecl () { +build_libecl () { run enable_environment - pushd $LIBECL_ROOT - python -m pip install -r requirements.txt pushd $LIBECL_BUILD - - cmake .. -DCMAKE_INSTALL_PREFIX=$INSTALL \ - -DINSTALL_ERT_LEGACY=ON \ - -DBUILD_TESTS=OFF \ - -DUSE_RPATH=ON \ - -DENABLE_PYTHON=ON + cmake .. -DCMAKE_INSTALL_PREFIX=$INSTALL make -j 6 install popd - popd } -build_res () { +build_libres () { run enable_environment - pushd $LIBRES_ROOT - python -m pip install -r requirements.txt - python -m pip install -r test_requirements.txt pushd $LIBRES_BUILD - echo "PYTHON:"$(which python) cmake .. -DEQUINOR_TESTDATA_ROOT=/project/res-testdata/ErtTestData \ -DCMAKE_PREFIX_PATH=$INSTALL \ - -DCMAKE_MODULE_PATH=$INSTALL/share/cmake/Modules \ -DCMAKE_INSTALL_PREFIX=$INSTALL \ - -DBUILD_TESTS=ON \ - -DENABLE_PYTHON=ON - + -DBUILD_TESTS=ON make -j 6 install popd - popd +} + +build_res () { + run enable_environment + pip install $LIBRES_ROOT + pip install -r test_requirements.txt } source_build_tools() { - source /opt/rh/devtoolset-7/enable - LIBECL_VERSION="" PYTHON_VERSION="3.6.4" GCC_VERSION=7.3.0 CMAKE_VERSION=3.10.2 source /prog/sdpsoft/env.sh + export PATH=/opt/rh/devtoolset-8/root/bin:$PATH python --version gcc --version cmake --version @@ -83,66 +74,69 @@ source_build_tools() { set -e } -setup_variables () { +setup_testdir() { + mkdir -p $TESTDIR/{.git,python} + ln -s {$LIBRES_ROOT,$TESTDIR}/lib + ln -s {$LIBRES_ROOT,$TESTDIR}/test-data + ln -s {$LIBRES_ROOT,$TESTDIR}/share + cp -R {$LIBRES_ROOT,$TESTDIR}/python/tests +} +setup_variables () { ENV=$WORKING_DIR/venv INSTALL=$WORKING_DIR/install LIBECL_ROOT=$WORKING_DIR/libecl - LIBECL_BUILD=$LIBECL_ROOT/build LIBRES_ROOT=$WORKING_DIR/libres LIBRES_BUILD=$LIBRES_ROOT/build - KOMODO_VERSION=bleeding + TESTDIR=$WORKING_DIR/testdir } enable_environment () { - run source_build_tools run setup_variables - source $ENV/bin/activate + run source_build_tools + export ERT_SHOW_BACKTRACE=Y - export ECL_SITE_CONFIG=/project/res/komodo/$KOMODO_VERSION/root/lib/python3.6/site-packages/res/fm/ecl/ecl_config.yml - export RMS_SITE_CONFIG=/project/res/komodo/$KOMODO_VERSION/root/lib/python3.6/site-packages/res/fm/rms/rms_config.yml - export LD_LIBRARY_PATH=$INSTALL/lib64:$LD_LIBRARY_PATH - export PYTHONPATH=$INSTALL/lib/python3.6/site-packages:$PYTHONPATH - export PYTHONFAULTHANDLER=PYTHONFAULTHANDLER + export RMS_SITE_CONFIG=/prog/res/komodo/bleeding-py36-rhel7/root/lib/python3.6/site-packages/ert_configurations/resources/rms_config.yml } create_directories () { mkdir $INSTALL - mkdir $LIBECL_BUILD - mkdir $LIBRES_BUILD } clone_repos () { + echo "Cloning into $LIBECL_ROOT" + git clone https://github.com/equinor/libecl $LIBECL_ROOT + mkdir -p $LIBECL_BUILD + echo "Cloning into $LIBRES_ROOT" git clone . $LIBRES_ROOT - - source ./.libecl_version - - echo "Cloning into $LIBECL_ROOT" - git clone -b $LIBECL_VERSION https://github.com/equinor/libecl $LIBECL_ROOT + mkdir -p $LIBRES_BUILD + ln -s /project/res-testdata/ErtTestData $LIBRES_ROOT/test-data/Equinor } create_virtualenv () { mkdir $ENV python3 -m venv $ENV source $ENV/bin/activate + pip install -U pip wheel setuptools cmake } run_ctest () { run enable_environment pushd $LIBRES_BUILD + export ERT_SITE_CONFIG=$INSTALL/share/ert/site-config ctest -j $CTEST_JARG -E Lint --output-on-failure popd } run_pytest_normal () { run enable_environment - pushd $LIBRES_ROOT/python + pushd $TESTDIR python -m pytest -m "not equinor_test" --durations=10 popd } @@ -150,7 +144,7 @@ run_pytest_normal () { run_pytest_equinor () { run enable_environment - pushd $LIBRES_ROOT/python + pushd $TESTDIR python -m pytest -m "equinor_test" --durations=10 popd }