diff --git a/.gitattributes b/.gitattributes index da75c11..8b13789 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1 @@ -iwrap/_version.py export-subst + diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..97d6def --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,72 @@ +name: build-wheel-and-publish-test-pypi + +on: + push: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + build: + name: Build distribution + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + - name: Install pypa/build + run: >- + python3 -m pip install pip setuptools wheel build + - name: Build a binary wheel and a source tarball + run: python3 -m build . + - name: Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + + publish-to-pypi: + name: Publish iWrap distribution to PyPI + if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes + needs: + - build + runs-on: ubuntu-22.04 + environment: + name: pypi + url: https://pypi.org/p/iwrap + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + publish-to-testpypi: + name: Publish iWrap distribution to TestPyPI + if: github.event_name == 'push' + needs: + - build + runs-on: ubuntu-22.04 + environment: + name: testpypi + url: https://test.pypi.org/p/iwrap + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution to TestPyPI + uses: pypa/gh-action-pypi-publish@unstable/v1 + with: + repository-url: https://test.pypi.org/legacy/ + verbose: true \ No newline at end of file diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..387e99a --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,25 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.11" + commands: + - pip install . + - pip install -r docs/requirements.txt + - cd docs && jupyter-book build . + # RTD expects the built HTML at: $READTHEDOCS_OUTPUT/html + - mkdir -p $READTHEDOCS_OUTPUT/html + - cp -a docs/_build/html/. $READTHEDOCS_OUTPUT/html/ + +# Optionally set the Python requirements used to build documentation +python: + install: + - requirements: docs/requirements.txt + +# Explicitly specify submodules +submodules: + include: all diff --git a/MANIFEST.in b/MANIFEST.in index 83f20f2..f5b84e2 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,6 +2,4 @@ recursive-include bin * recursive-include iwrap * include requirements* include README.md -include LICENSE.md -include versioneer.py -include iwrap/_version.py \ No newline at end of file +include LICENSE.md \ No newline at end of file diff --git a/Makefile b/Makefile index 68bb2ec..4e94638 100755 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ all: iwrap_build install: install_dir install_iwrap install_module docs uninstall: uninstall_module uninstall_iwrap -.PHONY: build/module/$(MODULEFILE) install_iwrap update_iwrap iwrap_build build_deps build_deps_clear help clean docs +.PHONY: build/module/$(MODULEFILE) install_iwrap update_iwrap iwrap_build build_deps build_deps_clear help clean docs test-muscle3 build-muscle3-macro clean-muscle3 check_already_installed: @@ -104,6 +104,10 @@ help: install_dir @echo -e "\tINSTALL_MOD: [$(INSTALL_MOD)]" @echo -e "Version of the package - iWrap version:" @echo -e "\tVERSION: [$(VERSION)]" + @echo -e "\n- MUSCLE3-specific targets:" + @echo -e "\ttest-muscle3 : Run MUSCLE3 integration tests" + @echo -e "\tbuild-muscle3-macro : Build MUSCLE3 macro model for tests" + @echo -e "\tclean-muscle3 : Clean MUSCLE3 test artifacts" docs: @$(SHELL_SCRIPT) @@ -111,6 +115,33 @@ docs: code-check: pylint -E ./iwrap -clean: +# MUSCLE3-specific targets +test-muscle3: + @echo "Running MUSCLE3 integration tests..." + @if [ -f tests/run-muscle3-tests.sh ]; then \ + cd tests && ./run-muscle3-tests.sh; \ + else \ + echo "MUSCLE3 test script not found. Running pytest..."; \ + $(PY_CMD) -m pytest tests/muscle3/ -v -m muscle3 || echo "pytest not available or tests failed"; \ + fi + +build-muscle3-macro: + @echo "Building MUSCLE3 macro model for tests..." + @if [ -d tests/muscle3/macro ]; then \ + $(MAKE) -C tests/muscle3/macro; \ + else \ + echo "MUSCLE3 macro directory not found at tests/muscle3/macro"; \ + fi + +clean-muscle3: + @echo "Cleaning MUSCLE3 test artifacts..." + @$(RM) -rf tests/muscle3/run_* + @$(RM) -rf tests/muscle3/*/m3_actor + @$(RM) -rf tests/muscle3/actors/*/run_* + @$(RM) -rf tests/muscle3/actors/*/*/m3_actor + @$(RM) -rf tests/integration/muscle3/run_* + @echo "MUSCLE3 test artifacts cleaned" + +clean: clean-muscle3 rm -rf dist rm -rf build diff --git a/README.md b/README.md index 9e005d4..71bc72b 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,21 @@ # iWrap +[![Read the Docs](https://img.shields.io/badge/docs-readthedocs-blue)](https://iwrap.readthedocs.io) + iWrap is a modular component generator, implemented in Python, used for creating IMAS actors from physics models. This mechanism allows to integrate physics codes written in one language (Fortran, CPP) within complex computing scenarios designed in other language (e.g. Python). It's plug-in based modular design with clear separation of concerns allows to generate various types of actors and easily change data access paradigm (from dataset descriptor for AL to direct HDC data for instance) +## Available Actor Types + +iWrap includes the following built-in actor generators: + +- **Python Actor**: Standard Python actors for straightforward Python integrations +- **MUSCLE3 Actors** (optional): High-performance multiscale coupling framework + - MUSCLE3-Python: Python code with MUSCLE3 coupling + - MUSCLE3-CPP: C++ code with MUSCLE3 coupling + - MUSCLE3-Fortran: Fortran code with MUSCLE3 coupling + For user conveniency it provides two kinds of interfaces: * user friendly graphical interface that allows non-experienced users to define an actor in intuitive way * command line interface foreseen for more advanced users that may want to e.g. automatise actor generation process using scripts. @@ -14,10 +26,39 @@ To learn about iWrap, please follow `tutorials/README.md` to generate an interac We have also prepared HTML and Docker versions, so you can choose the one that suits you best. +# Installation + +## Quick Install + +### Basic Installation (Core Only) +```bash +pip install iwrap +``` +This installs iWrap with the standard Python actor generator. + +### Installation with MUSCLE3 Support +```bash +pip install iwrap[muscle3] +``` +This enables MUSCLE3 actor generators (MUSCLE3-Python, MUSCLE3-CPP, MUSCLE3-Fortran). + +### Development Installation +```bash +pip install iwrap[all] +``` +Installs all optional dependencies including MUSCLE3. + +## Verify Installation + +List available actor types: +```bash +iwrap --list-actor-types +``` + # Configuration of working environment ## Downloading software - git clone ssh://git@git.iter.org/imex/iwrap.git + git clone https://github.com/iterorganization/iWrap.git cd iwrap git checkout diff --git a/ci/muscle3/build-macro-model.sh b/ci/muscle3/build-macro-model.sh new file mode 100755 index 0000000..1faca37 --- /dev/null +++ b/ci/muscle3/build-macro-model.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +source ci/muscle3/setup-test-env.sh $* || exit 1 +echo "executing $(basename "$0")" +cd ./tests/muscle3/macro +make all + diff --git a/ci/muscle3/make-docs.sh b/ci/muscle3/make-docs.sh new file mode 100755 index 0000000..1a0a7e0 --- /dev/null +++ b/ci/muscle3/make-docs.sh @@ -0,0 +1,30 @@ +#!/bin/bash + + +source ci/muscle3/setup-test-env.sh $@ || exit 1 + +echo "-----------Create Python Virtual ENV-------------" +# Remove Virtual env if already exists +rm -rf venv +python -m venv --system-site-packages venv +#python -m venv venv + +echo "-----------Activate Python Virtual ENV-------------" +source `pwd`/venv/bin/activate + +echo "-----------------PIP sphinx-rtd-theme------------" +pip install sphinx-rtd-theme + + +echo "--------------Report Python version--------------" +echo "Python version: `python --version`" +echo "Using python from: `which python`" + + +echo "--------------Build documentation--------------" +# PYTHONPATH set to allow sphinx find 'Read The Docs' theme +PYTHONPATH=`pwd`/venv/lib//python3.11/site-packages/:$PYTHONPATH make docs +# make docs + +echo "-----------Remove Python Virtual ENV-------------" +rm -rf venv \ No newline at end of file diff --git a/ci/muscle3/setup-test-env.sh b/ci/muscle3/setup-test-env.sh new file mode 100755 index 0000000..bc727a1 --- /dev/null +++ b/ci/muscle3/setup-test-env.sh @@ -0,0 +1,54 @@ +#!/bin/bash +echo "executing $(basename "$0")" + +print_help() { + echo -e "Usage:" + echo -e "\t $0 [--use-installed|-i] [--help|-h]" + echo + echo -e "optional arguments:" + echo -e "\t -i, --use-installed use PIP installed or already loaded instance of plugins" + echo -e "\t -h, --help show this help message and exit" +} + + +use_installed=false + +while [ $# -gt 0 ]; do + case "$1" in + --use-installed|-i) + echo "WARNING: Using pre-loaded instance of plugins!" + use_installed=true + ;; + --help|-h) + print_help + exit 0 + ;; + *) + printf "ERROR: Incorrect parameter: \"$1\"\n" + print_help + # exit 1 + esac + shift +done + +# setup system environment +source ci/muscle3/st01-system-env.sh "$@" || exit 1 + +# look if iWrap has been already loaded / configured +if `which iwrap &> /dev/null` ; then + echo "WARNING: Using pre-loaded 'iWrap' from `which iwrap`!" + +else + echo "INFO: iWrap was not loaded! Loading 'iWrap' module!" + source ci/muscle3/st02-iwrap-load.sh "$@" || exit 1 +fi + +# check if 'local' or installed version should be used +# if $use_installed ; then +# echo "WARNING: Using pre-loaded instance of plugins!" +# else +# echo "INFO: Setting up plugins from local directory!" +# source ci/muscle3/st03-m3plugins-local.sh "$@" || exit 1 +# fi + + diff --git a/ci/muscle3/st00-defs.sh b/ci/muscle3/st00-defs.sh new file mode 100755 index 0000000..506b666 --- /dev/null +++ b/ci/muscle3/st00-defs.sh @@ -0,0 +1,19 @@ +# auxiliary functions definitions + +echo "executing $(basename "$0")" +function yell () +{ + echo "$0: $*" >&2 +} + +function die () +{ + yell "$*"; exit 1 +} + +function try () +{ + "$@" || die "cannot $*" +} + + diff --git a/ci/muscle3/st01-system-env.sh b/ci/muscle3/st01-system-env.sh new file mode 100755 index 0000000..5093ac5 --- /dev/null +++ b/ci/muscle3/st01-system-env.sh @@ -0,0 +1,57 @@ +# Set up environment +source ci/muscle3/st00-defs.sh +echo "executing $(basename "$0")" +# Set up environment such that module files can be loaded +if test -f /etc/profile.d/modules.sh ;then +. /etc/profile.d/modules.sh +else +. /usr/share/Modules/init/sh +fi +module purge + +# Set up environment +echo "--------------Module load IMAS--------------" +if [ "$COMPILER_VENDOR" == "intel" ]; then # INTEL + + try module load XMLlib/3.3.1-intel-compilers-2023.2.1 + + try module load MUSCLE3/0.7.1-intel-2023b + try module load IMAS-AL-Fortran/5.4.0-intel-2023b-DD-4.0.0 + try module load IMAS-AL-Java/5.4.0-intel-2023b-DD-4.0.0 + try module load IMAS-AL-Cpp/5.4.0-intel-2023b-DD-4.0.0 + try module load IMAS-AL-Matlab/5.4.0-intel-2023b-DD-4.0.0 + try module load IMAS-Python/2.0.1-intel-2023b + try module load PyYAML/6.0.1-GCCcore-13.2.0 + try module load lxml/4.9.3-GCCcore-13.2.0 + + export CXX="icpc" + export FC="ifort" + export MPICXX="mpiicpc" + export MPIFC="mpiifort" + +else + + # GFORTRAN + try module load IMAS-AL-Fortran/5.4.0-foss-2023b-DD-4.0.0 + try module load IMAS-AL-Java/5.4.0-foss-2023b-DD-4.0.0 + try module load IMAS-AL-Cpp/5.4.0-foss-2023b-DD-4.0.0 + try module load IMAS-AL-Matlab/5.4.0-foss-2023b-DD-4.0.0 + try module load IMAS-Python/2.0.1-foss-2023b + try module load XMLlib/3.3.1-GCC-13.2.0 + try module load MUSCLE3/0.7.1-foss-2023b + try module load PyYAML/6.0.1-GCCcore-13.2.0 + try module load lxml/4.9.3-GCCcore-13.2.0 + + export CXX="g++" + export FC="gfortran" + export MPICXX="mpicxx" + export MPIFC="mpifort" +fi + +export AL_MAJOR="${AL_VERSION%.*.*}" + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +export TESTS_DIR=$( cd -- "${SCRIPT_DIR}/../../tests/muscle3" &> /dev/null && pwd ) + +echo TESTS_DIR: $TESTS_DIR diff --git a/ci/muscle3/st02-iwrap-load.sh b/ci/muscle3/st02-iwrap-load.sh new file mode 100755 index 0000000..4af1713 --- /dev/null +++ b/ci/muscle3/st02-iwrap-load.sh @@ -0,0 +1,16 @@ +# Set up environment +source ci/muscle3/st00-defs.sh +echo "executing $(basename "$0")" +export IWRAP_HOME=$(realpath "$(dirname ${BASH_SOURCE})/../..") + +export PATH=${IWRAP_HOME}/bin:${PATH} + +export PYTHONPATH=${IWRAP_HOME}:${PYTHONPATH} + + +echo "IWRAP_HOME: $IWRAP_HOME" +echo "PATH: $PATH" +echo "PYTHONPATH: $PYTHONPATH" + +echo "IWRAP setup completed successfully" + diff --git a/ci/muscle3/st03-m3plugins-local.sh b/ci/muscle3/st03-m3plugins-local.sh new file mode 100755 index 0000000..f96fe36 --- /dev/null +++ b/ci/muscle3/st03-m3plugins-local.sh @@ -0,0 +1,4 @@ + +export PYTHONPATH=${PWD}/iwrap_plugins:${PYTHONPATH} + + diff --git a/ci/muscle3/static-code-analysis.sh b/ci/muscle3/static-code-analysis.sh new file mode 100755 index 0000000..1783333 --- /dev/null +++ b/ci/muscle3/static-code-analysis.sh @@ -0,0 +1,10 @@ +#!/bin/sh --login +# Task: Setup Environment and Run Pylint code check. Publish report artifacts with python modules and imas env modules. +set -e + +source ci/muscle3/setup-test-env.sh "$@" || exit 1 +echo "--------------Module load Pylint--------------" +module load Pylint/3.2.5-GCCcore-13.2.0 +python -m pylint -E ./iwrap_plugins > pylint.log + + diff --git a/ci/muscle3/test-runner.sh b/ci/muscle3/test-runner.sh new file mode 100755 index 0000000..fa251d0 --- /dev/null +++ b/ci/muscle3/test-runner.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +source ci/muscle3/setup-test-env.sh $* || exit 1 + +cd $TEST_DIR + +echo "- - - - - TEST CASE: " $TEST_DIR " - - - - - - - - - - - - " +make test + diff --git a/docs/_config.yml b/docs/_config.yml index 3c02dd1..e1366c8 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -33,8 +33,6 @@ sphinx: html_theme_options: show_navbar_depth: 3 #left sidebar show_toc_level: 3 #rigth sidebar - repo_url: "https://git.iter.org/projects/IMEX/repos/iwrap" - repo_name: "iWrap" suppress_warnings: - "etoc.toctree" diff --git a/docs/_toc.yml b/docs/_toc.yml index 9d5d78e..3c909ce 100644 --- a/docs/_toc.yml +++ b/docs/_toc.yml @@ -7,6 +7,8 @@ root: iWrap_introduction.md parts: - caption: Welcome to Documentation chapters: + - file: documentation/quickstart.rst + title: Quick Start Guide - file: documentation/iWrap_intro.rst title: Users Manual sections: @@ -32,6 +34,7 @@ parts: - file: documentation/developers_manual/05_plugins_compatibility.rst - file: documentation/developers_manual/06_code_desc_compatibility.rst - file: documentation/developers_manual/07_code_parameters_handling.rst + - file: documentation/developers_manual/08_ids_handling.rst - file: documentation/developers_manual/09_system_tests.rst title: System tests sections: @@ -43,6 +46,11 @@ parts: sections: - file: documentation/installation_guide/iwrap_requirements.rst - file: documentation/installation_guide/iwrap_configuration.rst + + - file: documentation/muscle3.rst + title: MUSCLE3 Actors + sections: + - file: documentation/muscle3_resources/code_wrapping.rst - caption: Welcome to Tutorial chapters: diff --git a/docs/documentation/actor_generation_cmdln.rst b/docs/documentation/actor_generation_cmdln.rst index 92d30b9..20436e1 100644 --- a/docs/documentation/actor_generation_cmdln.rst +++ b/docs/documentation/actor_generation_cmdln.rst @@ -35,7 +35,7 @@ Once YAML file is prepared it can be used for generation of an actor using iWrap lists details of given actor type generator -v, --version show program's version number and exit - For more information, visit . + For more information, visit . If YAML file contains both code description and actor description parts, no additional switches are required. diff --git a/docs/documentation/installation_guide/iwrap_installation.rst b/docs/documentation/installation_guide/iwrap_installation.rst index b9622d7..ccff134 100644 --- a/docs/documentation/installation_guide/iwrap_installation.rst +++ b/docs/documentation/installation_guide/iwrap_installation.rst @@ -147,3 +147,305 @@ Load the module into the environment: iwrap-gui +iWrap Python Installation +####################################################################################################################### + +MUSCLE3 support is now included in the main iWrap package and can be installed using pip with the ``muscle3`` extra. + + +Requirements +####################################################################################################################### + +Software required to use MUSCLE3 actor generators: + +- `Python` +- `iWrap` +- `MUSCLE3` +- `IMAS-Fortran` (if using Fortran actors) +- `IMAS-Cpp` (if using C++ actors) +- `IMAS-Java` (if using Java actors) +- `XMLLib` (for XML parsing) + +Software required to build documentation: + +- `make` +- `Python` +- Python packages: + + - `Sphinx` + - `sphinx-bootstrap-theme` + - `sphinx-rtd-theme` + + +`C and Fortran libmuscle installation `_ + +Overview +================ + +Install iWrap without MUSCLE3 support: + +.. code-block:: bash + + pip install iwrap + +This installs the basic iWrap functionality with the standard Python actor generator. + +Installation with MUSCLE3 Support +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To enable MUSCLE3 actor generators, install with the MUSCLE3 extra: + +.. code-block:: bash + + pip install iwrap[muscle3] + +Or alternatively: + +.. code-block:: bash + + pip install iwrap + pip install -r requirements_muscle3.txt + +This will install: + +- iWrap core package +- MUSCLE3-Python actor generator +- MUSCLE3 dependencies (muscle3, ymmsl) + +If you need MUSCLE3-Cpp or MUSCLE3-Fortran actors, install libmuscle separately: + +`C and Fortran libmuscle installation `_ + + +Development Installation +~~~~~~~~~~~~~~~~~~~~~~~~ + +For development, install in editable mode: + +.. code-block:: bash + + git clone https://github.com/iterorganization/iWrap.git + cd iwrap + pip install -e .[muscle3] + +Full Installation (All Features) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For development or to install all optional features: + +.. code-block:: bash + + pip install iwrap[all] + +This includes MUSCLE3 support and all other optional dependencies. + +Installation from Source +####################################################################################################################### + +Using Makefile +========================================================================================= + +The iWrap repository includes a Makefile for building and installing: + +.. code-block:: console + + make all # Build iWrap + make install # Install iWrap + make install_module # Install environment module + +To check all available targets and configuration options: + +.. code-block:: console + + shell> make help + +Variables that can be configured: + +- ``PYTHON_CMD`` - Python interpreter to use (default: python) +- ``INSTALL_PREFIX`` - Installation directory prefix +- ``INSTALL_MOD`` - Module file installation directory +- ``VERSION`` - iWrap version (auto-detected from git) + +Verification +####################################################################################################################### + +Verification of MUSCLE3 Generators Installation +========================================================================================= + +Once iWrap is installed with MUSCLE3 support, verify that the MUSCLE3 actor generators are available: + +.. code-block:: console + + shell> iwrap --list-actor-types + + Id : Name : Description + ---------------------------------------------------------------------- + python : Simple Python actor : Simple Python actor + MUSCLE3-Python : MUSCLE3 (Python) : Wrapping Python code into MUSCLE3 micro model + +If the MUSCLE3 actor types are listed, the installation was successful. + +Verify Python Imports +========================================================================================= + +You can also verify that the MUSCLE3 generators can be imported: + +.. code-block:: python + + from iwrap.generators.actor_generators.muscle3_python import M3PythonActor + from iwrap.generators.actor_generators.muscle3_common import m3_utils + +If these imports succeed without errors, the MUSCLE3 generators are correctly installed. + +Manual Environment Setup +======================== + +If you're not using the provided scripts, ensure the following are available: + +1. **MUSCLE3 Libraries**: Available via pkg-config + + .. code-block:: bash + + pkg-config --modversion muscle3 + + * libmuscle (CPP + Fortran) — `MUSCLE3 `_ (if using MUSCLE3 actors) + +2. **IMAS Access Layer**: Properly configured + + .. code-block:: bash + + module load IMAS-Matlab + module load IMAS-Fortran + module load IMAS-Cpp + module load IMAS-Java + + See: `IMAS-Matlab `_, + `IMAS-Fortran `_, + `IMAS-Cpp `_, + `IMAS-Java `_ + +3. **Python Path**: iWrap should be in your PYTHONPATH (automatically handled by pip install) + +4. **Extras**: Properly configured + + .. code-block:: bash + + module load XMLLib + +Verifying Installation +~~~~~~~~~~~~~~~~~~~~~~ + +To verify which actor generators are available: + +.. code-block:: bash + + iwrap --list-actor-types + +Expected output with MUSCLE3 installed: + +.. code-block:: text + + Id : Name : Description + ---------------------------------------------------------------------- + python : Simple Python actor : Simple Python actor + MUSCLE3-Python : MUSCLE3 (Python) : Wrapping Python code into MUSCLE3 micro model + +You can also verify that the MUSCLE3 generators can be imported: + +.. code-block:: python + + from iwrap.generators.actor_generators.muscle3_python import M3PythonActor + from iwrap.generators.actor_generators.muscle3_common import m3_utils + +If these imports succeed without errors, the MUSCLE3 plugins are correctly installed. + +Troubleshooting +####################################################################################################################### + +MUSCLE3 Actor Types Not Listed +=============================== + +If ``iwrap --list-actor-types`` doesn't show MUSCLE3 actor types: + +1. Verify MUSCLE3 extra was installed: + + .. code-block:: bash + + pip show iwrap | grep muscle3 + +2. Reinstall with MUSCLE3 support: + + .. code-block:: bash + + pip install --force-reinstall iwrap[muscle3] + +Import Errors +============= + +If you get import errors when trying to use MUSCLE3 generators: + +1. Verify iWrap is installed: + + .. code-block:: bash + + python -c "import iwrap; print(iwrap.__version__)" + +2. Verify MUSCLE3 dependencies are installed: + + .. code-block:: bash + + python -c "import muscle3; print(muscle3.__version__)" + +3. Check your Python environment is correct: + + .. code-block:: bash + + which python + which iwrap + +MUSCLE3 Library Not Found +========================= + +If you get errors about MUSCLE3 libraries not being found during actor compilation: + +1. Verify MUSCLE3 is available via pkg-config: + + .. code-block:: bash + + pkg-config --modversion muscle3 + pkg-config --cflags muscle3 + pkg-config --libs muscle3 + +2. Load the MUSCLE3 module (if using environment modules): + + .. code-block:: bash + + module load muscle3 + +3. Check the MUSCLE3 installation documentation for your platform. + + +Migration from Separate Plugin Package +####################################################################################################################### + +If you were previously using the separate ``iwrap-plugins-muscle3`` package: + +Import Path Changes +========================================================================================= + +Update your code to use the new import paths: + +**Old (separate plugin):** + +.. code-block:: python + + from iwrap_plugins.iwrap_actor_generator.muscle3_python import M3PythonActor + +**New (integrated):** + +.. code-block:: python + + from iwrap.generators.actor_generators.muscle3_python import M3PythonActor + + + diff --git a/docs/documentation/muscle3.rst b/docs/documentation/muscle3.rst new file mode 100644 index 0000000..96e6da0 --- /dev/null +++ b/docs/documentation/muscle3.rst @@ -0,0 +1,41 @@ +####################################################################################################################### +MUSCLE3 Actors +####################################################################################################################### + +.. toctree:: + :maxdepth: 10 + :caption: MUSCLE3 Actors + +.. note:: + + MUSCLE3 + MUSCLE3 is a coupling library for building multiscale simulations out of single-scale models. + It is the third incarnation of the MUSCLE Multiscale Coupling Library and Environment, + developed by the e-MUSC project of the University of Amsterdam and the Netherlands eScience Center. + For any details related to MUSCLE3, please take a look at documentation + available `here `_ . + + + MUSCLE3 Actor + A standalone program, generated by iWrap, containing native code calls wrapped into MUSCLE3 function instance + + +In most cases, MUSCLE3 actors are implemented manually, through calls to the MUSCLE3 API in an ad-hoc manner. +However, for simpler cases it is possible to generate actors automatically using iWrap component generator. + +The iWrap is extended to generate MUSCLE3 actors from any code that follows iWrap standardized code specification, +without changes of the code API. The MUSCLE3 wrapper is one of ‘actor generators’ provided by the iWrap. + +The iWrap is able to wrap native code provided in Python, C++ and Fortran, producing either a standalone binary +executable (C++ and Fortran) or a wrapping script (Python). + + +.. note:: Further reading + + - `MUSCLE3 documentation `_ + - `MUSCLE3 repository `_ + + + + + diff --git a/docs/documentation/muscle3_resources/actor-ports.png b/docs/documentation/muscle3_resources/actor-ports.png new file mode 100644 index 0000000..48e51ac Binary files /dev/null and b/docs/documentation/muscle3_resources/actor-ports.png differ diff --git a/docs/documentation/muscle3_resources/code_wrapping.rst b/docs/documentation/muscle3_resources/code_wrapping.rst new file mode 100644 index 0000000..8f01fb8 --- /dev/null +++ b/docs/documentation/muscle3_resources/code_wrapping.rst @@ -0,0 +1,457 @@ +.. sectnum:: + +.. toctree:: + :numbered: + :maxdepth: 10 + +MUSCLE3 actor generators requirements +####################################################################################################################### +Following software must be available to wrap the code into MUSCLE3 models: + +- *iWrap* - MUSCLE3 actors generators are built-in and included in the main iWrap package (v0.8.0+) +- *MUSCLE3 libraries* - indispensable for building actors - should be available via ``pkg-config`` mechanism + +.. note:: + Usually operation system can be configured using ``modules`` toolkit, so in most cases loading ``iwrap`` + module is enough to configure user working environment. MUSCLE3 libraries must also be available. + + **Warning**: Module name(s) may vary depending on platform! + +The code to be wrapped +####################################################################################################################### +Physics codes, to be used for generation of an actor can be of arbitrary content but must follow the standardized interface (API) +introduced with iWrap: a mandatory main function and optional ones for the initialization, +the finalization, for saving/restoring internal state and for getting the current timestamp. + +.. warning:: + The INIT and FINALIZE methods of the provided code cannot have IDS arguments, to be wrapped to MUSCLE3 model. + +.. note:: + To keep compatibility of the native code with other kinds of generators, the code itself should be ‘MUSCLE3 agnostic’: + no MUSCLE3 methods should be called from within a wrapped code. + +Actor generation +####################################################################################################################### + +To generate MUSCLE3 actor, `muscle3-` needs to be chosen out of all available actor types. +it can be done using either iwrap command line, YAML describing an actor or iWrap GUI. + +.. code-block:: console + + shell> iwrap --actor-type muscle3- -f code_description.yaml + +.. code-block:: YAML + :caption: Actor description + :emphasize-lines: 3 + + actor_description: + ... + actor_type: muscle3- # Actor type set to MUSCLE3 actor + ... + + code_description: + ... + +.. tip:: + To help developers write MUSCLE3-compliant code, the iWrap MUSCLE3 actor generators allow you + to generate so-called **actor's skeleton**. + The **actor's skeleton** is a source code for a given actor without embedding actual calls to native code. + All places where subroutines from the code could be introduced are clearly marked with comments + in the generated source code. + To generate this kind of the source code, simply leave out (or comment out the `programming_language` entry + of code description + +MUSCLE3 actor +####################################################################################################################### + +Created actor contains a native code placed into a standalone executable `.exe` that acts +as a MUSCLE3 function. + +A generated code intermediates between a wrapped code and MUSCLE3 library. It's main functionality includes: + +- Management of inter-model communication: + + - deserialization of IDSes received from MUSCLE3 and passing them to wrapped code + - serialization of IDSes returned by wrapped code and sending them using MUSCLE3 +- Handling actor checkpointing and restarting + +Physical time handling +========================================================================================= + +MUSCLE3 supports sending and receiving timestamped messages, which can help support consistent physical time handling +throughout the coupled simulation. Providing timestamps is optional, however, for checkpoint support it is required. +IDSes are sent in separate messages, using dedicated channels: + +- If code provides the GET_TIMESTAMP method, (single) timestamp obtained from this method will be used + for all output IDS messages (this is the time of the model, time in individual IDS may be slightly smaller, + but never larger). + +- If GET_TIMESTAMP method is not available, while sending particular IDS: + + - if IDS/time is allocated - IDS/time[last_index] will be used as message timestamp + (note: heterogeneous can also have that filled as the "common" time for this IDS) + - if IDS/time is not allocated - timestamp will be set to invalid value + + + +iWrap MUSCLE3 actor control flow +========================================================================================= +Basic MUSCLE3 control flow could be briefly described as follows: once the MUSCLE3 +and the wrapped code are initialized, at every iteration of the MUSCLE3 reuse loop, +the actor will receive data from other models, call the ‘main’ method of the wrapped model +and send data computed by mode. Computations finish with the clean up operations. + +#. Initialisation: + + a. MUSCLE3 system initialisation + #. Model initialisation - a model INIT method is called (if provided) + +#. A main ‘MUSCLE3 controlled’ loop: + + a. The actor receives input IDSes from interoperating MUSCLE3 models + #. The MAIN method of the wrapped code is called + #. Sending output IDSes to interoperating MUSCLE3 models + +#. Finalisation + + a. Model finalization: a model finish method is called (if provided) + #. MUSCLE3 system cleanup + +.. code-block:: fortran + :caption: MUSCLE3 actor structure (simplified Fortran-like pseudo code) + + > > > ENVIRONMENT INITIALIZATION < < < + CALL init_muscle(instance) + CALL wrapped_code_init([code_params]) ! called if init provided + + ! > > > MUSCLE3 LOOP. < < < + do while (LIBMUSCLE_Instance_reuse_instance(instance)) + + ! > > > RECEIVING INPUT IDSes < < < + CALL receive_input_idses(instance, ids1_in, ids2_in, ...) + + ! > > > WRAPPED CODE < < < + CALL wrapped_code(ids1_in, ids2_in, ..., ids1_out, ids2_out, ...[, code_params]) + + ! > > > SENDING OUTPUT IDSes < < < + ! if GET_TIMESTAMP method provided + timestamp = get_timestamp_from_wrapped() + ! else obtained from IDS time vector, while sending particular IDS + + CALL send_output_idses(instance, ids1_out, ids2_out, ..., timestamp) + + end do ! MUSCLE3 loop + + ! > > > FINALIZATION < < < + CALL wrapped_code_finalize() ! called if finalize provided + CALL finalize_muscle(instance) + + +Wrapping of MPI codes +========================================================================================= + +Since the actor doesn’t know how the model distributes its work over the processes, +it is the wrapped code duty to properly communicate the inputs to the other MPI processes, +run computations, collect the results, and return them in the root process. + +* The MAIN function (see iWrap model basic API) is called on all MPI processes and all processes + received valid IDSes. In contrary, the result of the main function is ignored in all non-root processes. + +* The INIT and FINALIZE functions are called on all MPI processes. + +* The GET_STATE and SET_STATE functions are called on all MPI processes. However, + only the root process will receive a non-empty input for SET_STATE. Similarly, + the result of the GET_STATE function is ignored in all non-root processes. + +Note that, even though each process in the template below calls `receive_idses`, +only the root process actually receives data. The other processes receive a dummy message, +thereby acting as a barrier without relying on a spinloop. For a detailed explanation of the MUSCLE3 API with MPI, +we refer to the `MUSCLE3 documentation `_ +To overcome this issue a wrapping code broadcasts every IDS from root to all non-root processes. + +.. warning:: + MUSCLE3 provides MPI support only for C++ and Fortran, so the iWrap generator handles MPI only in these languages. + +.. code-block:: fortran + :caption: Generated MUSCLE3 MPI actor structure (simplified Fortran-like pseudo code) + + > > > ENVIRONMENT INITIALIZATION < < < + CALL mpi_initialization(mpi_rank) + CALL init_muscle(instance) + CALL wrapped_code_init([code_params]) ! called if init provided + + ! > > > MUSCLE3 LOOP. < < < + do while (LIBMUSCLE_Instance_reuse_instance(instance)) + + ! > > > RECEIVING INPUT IDSes < < < + CALL receive_idses(instance, ids1_in, ids2_in, ...) + + if (mpi_rank == 0) then + ! Broadcast all IDSes to non-root processes + endif + + ! > > > WRAPPED CODE < < < + CALL wrapped_code(ids1_in, ids2_in, ..., ids1_out, ids2_out, ..., [, code_params]) + + if (mpi_rank == 0) then + ! > > > SENDING OUTPUT IDSes < < < + ! if GET_TIMESTAMP method provided + timestamp = get_timestamp_from_wrapped() + ! ELSE obtained from IDS time vector, while sending particular IDS + + CALL send_ids(instance, ids1_out, ids2_out, ..., timestamp) + endif + + end do ! MUSCLE3 loop + + ! > > > FINALIZATION < < < + CALL wrapped_code_finalize() ! called if finalize provided + CALL finalize_muscle(instance) + CALL mpi_finalization() + + +The restart feature +========================================================================================= +The wrapped code is run ‘atomically’, so no interaction between an actor and native method is possible +(the actor cannot force the model method to save a checkpoint at an arbitrary time, while it is executed). +Additionally, the wrapped methods are ‘MUSCLE3 agnostic’ - no calls of MUSCLE3 checkpointing methods +are available within the wrapped code. Nevertheless, to support stateful, compute demanding codes, +iWrap generated actors offers the possibility of a smooth restart of the model without losing +results obtained before error/crash occurred, based on MUSCLE3 checkpointing mechanism. + +To enable this feature, the wrapped code MUST provide both a method `GET_STATE` returning information +describing the code internal state, and a method `SET_STATE` restoring the state. + + +.. code-block:: fortran + :caption: Generated MUSCLE3 actor structure with restart enabled (simplified Fortran-like pseudo code) + + logical :: initial_run = .TRUE. + + > > > ENVIRONMENT INITIALIZATION < < < + CALL mpi_initialization(mpi_rank) + CALL init_muscle(instance) + + ! > > > MUSCLE3 LOOP. < < < + do while (LIBMUSCLE_Instance_reuse_instance(instance)) + + ! - - - WRAPPED CODE INITIALIZATION - - - # # # # + if ( LIBMUSCLE_Instance_resuming(instance)) then + ! > > > RESTART : the code is restored from the saved snapshot < < < + if (process_rank == MPI_ROOT_RANK) then + CALL restore_wrapped_code_state(instance) + endif + else ! m3_instance is not resuming + ! > > > INITIALIZATION by calling INIT method of the code < < < + if (initial_run) then + CALL wrapped_code_init([code_params]) ! called if init provided + endif + endif ! LIBMUSCLE_Instance_resuming + initial_run = .FALSE. + + if( LIBMUSCLE_Instance_should_init(instance)) then + ! > > > RECEIVING INPUT IDSes < < < + CALL receive_idses(instance, ids1_in, ids2_in, ...) + endif ! LIBMUSCLE_Instance_should_init + + ! > > > WRAPPED CODE < < < + CALL wrapped_code(ids1_in, ids2_in, ..., ids1_out, ids2_out, ..., [, code_params]) + + if (mpi_rank == MPI_ROOT_RANK) then + ! > > > SENDING OUTPUT IDSes < < < + ! if GET_TIMESTAMP method provided + timestamp = get_timestamp_from_wrapped() + ! ELSE obtained from IDS time vector, while sending particular IDS + + CALL send_ids(instance, ids1_out, ids2_out, ..., timestamp) + endif + + ! > > > SAVE SNAPSHOT < < < + if (LIBMUSCLE_Instance_should_save_final_snapshot(instance)) then + if (process_rank == MPI_ROOT_RANK ) then + CALL save_wrapped_code_state(instance) + endif + endif + + end do ! < < < MUSCLE3 LOOP < < < + + ! > > > FINALIZATION < < < + CALL wrapped_code_finalize() ! called if finalize provided + CALL finalize_muscle(instance) + CALL mpi_finalization() + +MUSCLE3 models coupling and running +####################################################################################################################### +Models, either wrapped by iWrap or prepared by user in any other way, have to be coupled manually +by workflow/scenario designer following MUSCLE3 standard guidelines and rules. + + +MUSCLE3 models coupling +========================================================================================= +MUSCLE3 uses the Multiscale Modelling and Simulation Language (MMSL) to describe +the structure of a multiscale model. MMSL can be expressed as a YAML file (yMMSL). +The MMSL lets one describe which components (submodels, scale bridges, data converters, UQ components, etc.) +a multiscale model consist of, how many instances of each we need, and how they are wired together. + +Following asumptions and limitations should be taken into consideration while designing a MUSCLE3 scenario: + + - Communication is managed (only) by a wrapper + - Actor communicates with other model(s) by sending/receiving IDSes + - Every message contains only one IDS + - IDSes are received via port of `F_INIT` type + - IDSes are sent via port of `O_F` type + - Every port handles only one in/out argument (IDS) + - Port number and names correspond to argument of wrapped code + - MPI: data are sent and received only by the root MPI process + +Changing code parameters of an actor +========================================================================================= +Setting ``.parameters_file`` property in yMMLS file (see example below) +forces an actor to read parameters from the pointed file and not from the default one. + +.. warning:: + - Only absolute path to code parameters file can be specified + - No system variables can be a part of specified path + - The code parameters read from the file must conform a code parameter schema provided by a code developer at the actor generation stage. + +.. code-block:: YAML + :caption: An example of a MUSCLE3 yMMLS workflow description + :emphasize-lines: 7,8 + + model: + name: helloworld + components: + macro: macro + micro: micro + ... + settings: + micro.parameters_file: /path/to/parameters.xml + +Provenance info +========================================================================================= + +YAML description +--------------------- +In order to provide additional information about actor configuration, actor/code description YAML file is being copied into actor's directory during generation process. + +Code parameters +--------------------- +Setting ``copy_xml_parameters`` property in yMMLS file (see example below) +forces an actor to save code parameters XML file in workdir during execution. If not set, parameters won't be saved. + +.. code-block:: YAML + :caption: An example of a MUSCLE3 yMMLS workflow description + :emphasize-lines: 7,8 + + model: + name: helloworld + components: + macro: macro + micro: micro + ... + settings: + copy_xml_parameters: true + + +Running MUSCLE3 scenario +========================================================================================= +An yMMSL description of the computing scenario needs to be run using `muscle_manager` +- the central run-time component of MUSCLE3. It is started together with the component +intances, and provides a central coordination point that the instances use to find each other. +The manager also collects log messages from the individual instances to aid in debugging, +and some profiling information to aid in scheduling and performance optimisation. + +.. note:: Further reading: + + - `yMMSL overview `_ + - `MUSCLE3 tutorial `_ + +Example +####################################################################################################################### +This example shows how to prepare and run an MUSCLE3 computing scenario, consisting of two codes +(aka. MUSCLE3 models), communicating with each other by sending/receiving IDSes: + +- prepared manually *MUSCLE3 macro model* +- a native code wrapped automatically by iWrap into *MUSCLE3 function* + +.. image:: macro-actor.png + + +Wrapping code into MUSCLE3 function +========================================================================================= + +A code to be wrapped is a very simple Fortran subroutine, obtaining `core_profiles` IDS as input +and returning `distribution_sources` IDS. To be compatible with 'iWrap standardized API' argument list +contains also `status_code` and `status_message`. + +.. literalinclude:: example/wrapped_code.f90 + :language: fortran + :caption: wrapped_code.f90 + +The wrapped code has to be compiled and packed into a static library. + +.. code-block:: console + :caption: Building wrapped code + + gfortran -c -o wrapped_code.o wrapped_code.f90 `pkg-config --cflags imas-gfortran` + @ar -rcs libwrapped_code.a wrapped_code.o + +Once a static library is built, the code description has to to be prepared, +to provide iWrap with all information required to generate an actor. + +.. literalinclude:: example/code_description.yaml + :language: YAML + :caption: code_description.yaml + +To generate an actor, one has to call `iwrap` command providing an actor type (`muscle3`), +an arbitrary actor name and YAML file containing the code description. + +.. code-block:: console + :caption: Actor generation + + iwrap --actor-type muscle3 --actor-name m3_actor -f ./code_description.yaml + +If generation was successful, an executable `m3_actor.exe` containing user provided subroutine +wrapped into MUSCLE3 function, is created in `/m3_actor/bin/` directory. + +.. image:: actor-ports.png +.. note:: + Please notice, that ports of generated actor have exactly the same names + as they have been defined in code description YAML + +.. literalinclude:: example/standalone.f90 + :language: fortran + :caption: Autogenerated code + +Macro model +========================================================================================= +A `macro model` code has to be prepared manually. This code "triggers" a generated actor +sending to it `core_profiles` IDS and receiving back computed `distribution_sources` IDS + +.. literalinclude:: example/macro.f90 + :language: fortran + :caption: macro.f90 + +.. code-block:: console + :caption: Building macro model + + gfortran -o macro.exe macro.f90 `pkg-config --cflags --libs imas-gfortran ymmsl libmuscle_fortran` + + +Scenario description +========================================================================================= + + +.. literalinclude:: example/example.ymmsl + :language: YAML + :caption: example.ymmsl + + +Launching an example +========================================================================================= + +.. code-block:: console + :caption: Running an example + + muscle_manager --start-all example.ymmsl + diff --git a/docs/documentation/muscle3_resources/example/Makefile b/docs/documentation/muscle3_resources/example/Makefile new file mode 100644 index 0000000..5aa450b --- /dev/null +++ b/docs/documentation/muscle3_resources/example/Makefile @@ -0,0 +1,19 @@ +.PHONY: native + +native: + gfortran -g -c -o wrapped_code.o wrapped_code.f90 `pkg-config --cflags imas-gfortran` + @ar -rcs libwrapped_code.a wrapped_code.o + +actor: + iwrap --actor-type muscle3 --actor-name m3_actor -f ./code_description.yaml + +macro: + gfortran -g -o macro.exe macro.f90 `pkg-config --cflags --libs imas-gfortran ymmsl libmuscle_fortran` + + +run: + EXAMPLE_DIR=`pwd` muscle_manager --start-all example.ymmsl + +clean: + rm -rf *.o *.exe *.mod *.a run_* + diff --git a/docs/documentation/muscle3_resources/example/code_description.yaml b/docs/documentation/muscle3_resources/example/code_description.yaml new file mode 100644 index 0000000..e746873 --- /dev/null +++ b/docs/documentation/muscle3_resources/example/code_description.yaml @@ -0,0 +1,22 @@ +code_description: + implementation: + subroutines: + init: wrapped_code_init + main: wrapped_code_main + finalize: wrapped_code_finalise + programming_language: Fortran + data_type: legacy + code_path: ./libwrapped_code.a + include_path: ./mod_wrapped_code.mod + arguments: + - name: core_profiles_in + type: core_profiles + intent: IN + - name: distribution_sources_out + type: distribution_sources + intent: OUT + settings: + compiler_cmd: gfortran + + + diff --git a/docs/documentation/muscle3_resources/example/example.ymmsl b/docs/documentation/muscle3_resources/example/example.ymmsl new file mode 100644 index 0000000..aa9d914 --- /dev/null +++ b/docs/documentation/muscle3_resources/example/example.ymmsl @@ -0,0 +1,30 @@ +ymmsl_version: v0.1 + +model: + name: example + components: + macro: macro + micro: micro + conduits: + macro.core_profiles_out: micro.core_profiles_in + micro.distribution_sources_out: macro.distribution_sources_in +settings: + t_max: 2.0 + dt: 1.0 + muscle_remote_log_level: DEBUG + +implementations: + micro: + executable: ${IWRAP_ACTORS_DIR}/m3_actor/bin/m3_actor.exe + + macro: + executable: ${EXAMPLE_DIR}/macro.exe + +resources: + macro: + threads: 1 + + micro: + threads: 1 + + diff --git a/docs/documentation/muscle3_resources/example/macro.f90 b/docs/documentation/muscle3_resources/example/macro.f90 new file mode 100644 index 0000000..96e4f7d --- /dev/null +++ b/docs/documentation/muscle3_resources/example/macro.f90 @@ -0,0 +1,83 @@ +program macro + use ids_routines + use ymmsl + use libmuscle + implicit none + + type (ids_core_profiles) :: core_profiles_out + type (ids_distribution_sources) :: distribution_sources_in + + real (selected_real_kind(15)) :: t_cur, t_next, t_max, dt + character(len=1), dimension(:), allocatable :: serialized_ids + + type(LIBMUSCLE_PortsDescription) :: ports + type(LIBMUSCLE_Instance) :: instance + + type(LIBMUSCLE_Message) :: rmsg + type(LIBMUSCLE_DataConstRef) :: ritem + + type(LIBMUSCLE_Message) :: smsg + type(LIBMUSCLE_Data) :: sitem + + + ! > > > - - - INITIALISATION - - - < < < + ports = LIBMUSCLE_PortsDescription_create() + call LIBMUSCLE_PortsDescription_add(ports, YMMSL_OPERATOR_O_I, 'core_profiles_out') + call LIBMUSCLE_PortsDescription_add(ports, YMMSL_OPERATOR_S, 'distribution_sources_in') + instance = LIBMUSCLE_Instance_create(ports) + call LIBMUSCLE_PortsDescription_free(ports) + + ! > > > - - - MAIN MUSCLE3 LOOP - - - < < < + do while (LIBMUSCLE_Instance_reuse_instance(instance)) + ! F_INIT + t_max = LIBMUSCLE_Instance_get_setting_as_real8(instance, 't_max') + dt = LIBMUSCLE_Instance_get_setting_as_real8(instance, 'dt') + + t_cur = 0.0 + + ! > > > - - - INTERNAL LOOP - - - < < < + do while (t_cur + dt < t_max) + ! O_I + t_next = t_cur + dt + + + ! > > > - - - MODEL COMPUTATIONS - - - < < < + + core_profiles_out%ids_properties%homogeneous_time = IDS_TIME_MODE_HOMOGENEOUS + allocate(core_profiles_out%time(1)) + core_profiles_out%time(1) = 1.0 + + ! . . . + ! > > > - - - - - - - - - - - - - - - < < < + + ! > > > - - - SENDING DATA - - - < < < + call ids_serialize(core_profiles_out, serialized_ids) + call ids_deallocate(core_profiles_out) + + sitem = LIBMUSCLE_Data_create_byte_array(serialized_ids) + smsg = LIBMUSCLE_Message_create(t_cur, sitem) + + call LIBMUSCLE_Instance_send(instance, 'core_profiles_out', smsg) + call LIBMUSCLE_Message_free(smsg) + call LIBMUSCLE_Data_free(sitem) + deallocate(serialized_ids) + + + ! > > > - - - RECEIVING DATA - - - < < < + rmsg = LIBMUSCLE_Instance_receive(instance, 'distribution_sources_in') + ritem = LIBMUSCLE_Message_get_data(rmsg); + + allocate(serialized_ids(LIBMUSCLE_DataConstRef_size(ritem))) + call LIBMUSCLE_DataConstRef_as_byte_array(ritem, serialized_ids) + call LIBMUSCLE_DataConstRef_free(ritem) + call LIBMUSCLE_Message_free(rmsg) + + call ids_deserialize(serialized_ids, distribution_sources_in) + deallocate(serialized_ids) + call ids_deallocate(distribution_sources_in) + + t_cur = t_cur + dt + end do + end do + +end program macro \ No newline at end of file diff --git a/docs/documentation/muscle3_resources/example/standalone.f90 b/docs/documentation/muscle3_resources/example/standalone.f90 new file mode 100644 index 0000000..825f063 --- /dev/null +++ b/docs/documentation/muscle3_resources/example/standalone.f90 @@ -0,0 +1,62 @@ +program standalone + + use iwrap_tools + use muscle3_tools + use mod_wrapped_code + + implicit none + + !--------------------------------------------------------- + character(len=*), parameter :: ACTOR_NAME = "m3_actor" + type(LIBMUSCLE_Instance) :: instance + integer :: process_rank + + + !---- Status info ---- + integer :: status_code = 0 + character(len=:), pointer :: status_message + + !---- Code parameters ---- + character(len=:), allocatable :: xml_string + type(ids_parameters_input) :: imas_code_params + + !---- IN/OUT IDSes ---- + ! core_profiles_in + type (ids_core_profiles) :: core_profiles_in + ! distribution_sources_out + type (ids_distribution_sources) :: distribution_sources_out + + ! > > > - - - INITIALISATION - - - < < < + CALL init_muscle(instance) + + ! > > > - - - WRAPPED CODE CALL - INIT SBRT - - - < < < + CALL wrapped_code_init( status_code, status_message) + CALL check_status(instance, status_code, status_message, ACTOR_NAME) + + ! > > > MUSCLE3 LOOP. < < < + do while (LIBMUSCLE_Instance_reuse_instance(instance)) + + ! > > > RECEIVING INPUT IDSes < < < + CALL recveive_input_idses(core_profiles_in, instance ) + + ! > > > - - - WRAPPED CODE CALL - MAIN SBRT - - - < < < + CALL wrapped_code_main( core_profiles_in, distribution_sources_out, status_code, status_message) + + !-----------Check status info --------------------- + CALL check_status(instance, status_code, status_message, ACTOR_NAME) + + ! > > > SENDING OUTPUT IDSes < < < + CALL send_output_idses(distribution_sources_out, instance) + + ! > > > CLEAN UP < < < + CALL ids_deallocate(core_profiles_in) + CALL ids_deallocate(distribution_sources_out) + + end do ! < < < MUSCLE3 LOOP < < < + + ! > > > - - - - - - - - - - - - - WRAPPED CODE CALL - FINALISE SBRT - - - - - - - - - - - - - - - - - - < < < + CALL wrapped_code_finalise(status_code, status_message) + !-----------Check status info --------------------- + CALL check_status(instance, status_code, status_message, ACTOR_NAME) + +end program \ No newline at end of file diff --git a/docs/documentation/muscle3_resources/example/wrapped_code.f90 b/docs/documentation/muscle3_resources/example/wrapped_code.f90 new file mode 100644 index 0000000..dc2f540 --- /dev/null +++ b/docs/documentation/muscle3_resources/example/wrapped_code.f90 @@ -0,0 +1,71 @@ +module mod_wrapped_code +contains + + ! + ! INITIALISATION SUBROUTINE + ! + subroutine wrapped_code_init(status_code, status_message) + use ids_schemas, only: ids_parameters_input + implicit none + integer, intent(out) :: status_code + character(len=:), pointer, intent(out) :: status_message + + + ! Setting status to SUCCESS + status_code = 0 + allocate(character(50):: status_message) + status_message = 'OK' + + write(*,*) '=======================================' + write(*,*) 'Wrapped code: INITIALISATION called' + write(*,*) '=======================================' + + end subroutine wrapped_code_init + + + ! + ! MAIN SUBROUTINE + ! + subroutine wrapped_code_main(coreprofilesin, distsourceout, error_flag, error_message) + use ids_schemas + implicit none + + type (ids_core_profiles) :: coreprofilesin + type (ids_distribution_sources) :: distsourceout + integer, intent(out) :: error_flag + character(len=:), pointer, intent(out) :: error_message + + write(*,*) '=======================================' + write(*,*) 'Wrapped code: MAIN called' + write(*,*) '=======================================' + + !COMPUTATIONS + allocate(distsourceout%time(1)) + distsourceout%time(1) = coreprofilesin%time(1) + 1 + distsourceout%ids_properties%homogeneous_time = 1 + + return + end subroutine wrapped_code_main + + ! + ! FINALISATION SUBROUTINE + ! + subroutine wrapped_code_finalise(status_code, status_message) + implicit none + integer, intent(out) :: status_code + character(len=:), pointer, intent(out) :: status_message + + ! Setting status to SUCCESS + status_code = 0 + allocate(character(50):: status_message) + status_message = 'OK' + + write(*,*) '=======================================' + write(*,*) 'Wrapped code: FINALISATION called' + write(*,*) '=======================================' + + end subroutine wrapped_code_finalise + +end module mod_wrapped_code + + diff --git a/docs/documentation/muscle3_resources/macro-actor.png b/docs/documentation/muscle3_resources/macro-actor.png new file mode 100644 index 0000000..cd4508e Binary files /dev/null and b/docs/documentation/muscle3_resources/macro-actor.png differ diff --git a/docs/documentation/quickstart.rst b/docs/documentation/quickstart.rst new file mode 100644 index 0000000..966b0ed --- /dev/null +++ b/docs/documentation/quickstart.rst @@ -0,0 +1,256 @@ +======================================== +Quick Start Guide +======================================== + +This guide will help you get started with iWrap quickly, covering both standard Python actors and MUSCLE3 actors. + +Installation +============ + +For detailed installation instructions, see :doc:`installation_guide/iwrap_installation`. + +Step 1: Prepare Your Code +-------------------------- + +Create a simple Python physics code ``my_code.py``: + +.. code-block:: python + + # my_code.py + def init(): + """Initialize the code""" + print("Initializing...") + + def run(equilibrium_in, core_profiles_out): + """Main computation""" + print("Running step...") + # Your physics calculations here + return equilibrium_in, core_profiles_out + + def finalize(): + """Cleanup""" + print("Finalizing...") + +Step 2: Create Code Description +-------------------------------- + +Create ``code_description.yaml``: + +.. code-block:: yaml + + actor_description: + actor_name: my_first_actor + actor_type: python + + code_description: + implementation: + programming_language: python + code_path: ./my_code.py + + subroutines: + init: + name: init + arguments: [] + + step: + name: run + arguments: + - name: equilibrium + type: input + - name: core_profiles + type: output + + finalize: + name: finalize + arguments: [] + +Step 3: Generate Actor +----------------------- + +.. code-block:: bash + + iwrap -a my_first_actor -t python -f code_description.yaml + +Step 4: Use the Actor +---------------------- + +Create a workflow ``workflow.py``: + +.. code-block:: python + + import imas + from my_first_actor import MyFirstActor + + # Create IDS objects + _factory = imas.IDSFactory() + equilibrium = _factory.equilibrium() + core_profiles = _factory.core_profiles() + + # Initialize and run actor + actor = MyFirstActor() + actor.init() + actor.run(equilibrium, core_profiles) + actor.finalize() + +Run it: + +.. code-block:: bash + + python workflow.py + +Your First MUSCLE3 Actor +======================================== + +Prerequisites +------------- + +Ensure MUSCLE3 is installed: +For detailed installation instructions, see :doc:`installation_guide/iwrap_installation`. + +Step 1: Prepare Your Code +-------------------------- + +Create a MUSCLE3-compatible code ``my_muscle3_code.py``: + +.. code-block:: python + + # my_muscle3_code.py + def init(): + """Initialize - no IDS arguments allowed""" + print("MUSCLE3 actor initializing...") + + def run(equilibrium_in, core_profiles_out): + """Main computation with IDS""" + print("MUSCLE3 step running...") + # Your physics here + return equilibrium_in, core_profiles_out + + def finalize(): + """Cleanup - no IDS arguments allowed""" + print("MUSCLE3 actor finalizing...") + +Step 2: Create Code Description +-------------------------------- + +Create ``muscle3_code_description.yaml``: + +.. code-block:: yaml + + actor_description: + actor_name: my_muscle3_actor + actor_type: MUSCLE3-Python + + code_description: + implementation: + programming_language: python + code_path: ./my_muscle3_code.py + + subroutines: + init: + name: init + arguments: [] # IMPORTANT: No IDS for MUSCLE3 init + + step: + name: run + arguments: + - name: equilibrium + type: input + - name: core_profiles + type: output + + finalize: + name: finalize + arguments: [] # IMPORTANT: No IDS for MUSCLE3 finalize + +Step 3: Generate MUSCLE3 Actor +------------------------------- + +.. code-block:: bash + + iwrap -a my_muscle3_actor -t MUSCLE3-Python -f muscle3_code_description.yaml + +Step 4: Create MUSCLE3 Workflow +-------------------------------- + +Create ``workflow.ymmsl``: + +.. code-block:: yaml + + ymmsl_version: v0.1 + + model: + name: my_first_muscle3_simulation + components: + macro: + implementation: macro_model + ports: + o_i: core_profiles + s_o: equilibrium + + micro: + implementation: my_muscle3_actor + ports: + f_init: equilibrium + o_f: core_profiles + + settings: + micro.time_step: 0.1 + macro.iterations: 10 + +Step 5: Run MUSCLE3 Workflow +----------------------------- + +.. code-block:: bash + + muscle_manager workflow.ymmsl + +Using the GUI +======================================== + +iWrap provides a graphical interface for easy actor generation. + +Launch GUI +---------- + +.. code-block:: bash + + iwrap-gui + +Or load an existing description: + +.. code-block:: bash + + iwrap-gui -f code_description.yaml + +Common Tasks +======================================== + +Listing Available Actor Types +------------------------------ + +.. code-block:: bash + + iwrap --list-actor-types + +Getting Help +------------ + +.. code-block:: bash + + iwrap -h + iwrap-gui -h + +Checking Version +---------------- + +.. code-block:: bash + + iwrap --version + +Viewing Actor Generator Details +-------------------------------- + +.. code-block:: bash + + iwrap --list-actor-details python + iwrap --list-actor-details MUSCLE3-Python diff --git a/docs/iWrap_introduction.md b/docs/iWrap_introduction.md index a99b0f2..9944f86 100644 --- a/docs/iWrap_introduction.md +++ b/docs/iWrap_introduction.md @@ -74,11 +74,17 @@ Source: https://en.wikipedia.org/wiki/Actor_model iWrap is built around a plug-in based modular design, making it very flexible and versatile. It enables you to generate various types of actors and switch between different data representation and access methods. -```{admonition} MUSCLE3 Integration +```{admonition} Available Actor Types :class: important iWrap is engineered with modularity in mind, enabling it to generate a variety of actor types to suit your specific needs. -Currently, it supports the creation of `Python` actors for straightforward Python integrations and `Muscle3` actors for more complex, high-performance computing scenarios. + +**Built-in Actor Generators:** +- **Python Actor**: Standard Python actors for straightforward Python integrations +- **MUSCLE3 Actors**: High-performance coupling framework actors (requires `pip install iwrap[muscle3]`) + - MUSCLE3-Python: Python code with MUSCLE3 coupling + - MUSCLE3-CPP: C++ code with MUSCLE3 coupling + - MUSCLE3-Fortran: Fortran code with MUSCLE3 coupling ``` diff --git a/docs/images/muscle3/actor-ports.png b/docs/images/muscle3/actor-ports.png new file mode 100644 index 0000000..48e51ac Binary files /dev/null and b/docs/images/muscle3/actor-ports.png differ diff --git a/docs/images/muscle3/macro-actor.png b/docs/images/muscle3/macro-actor.png new file mode 100644 index 0000000..cd4508e Binary files /dev/null and b/docs/images/muscle3/macro-actor.png differ diff --git a/docs/requirements.txt b/docs/requirements.txt index 26c6c33..a973efd 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,18 +1,18 @@ -urllib3 -jupyterlab -jupyter-book -docutils -jupyterlab_myst -lxml -markdown_it_py -mdit_py_plugins -myst-nb -myst-parser -sphinx -sphinx_inline_tabs -tomli -typing-extensions -pylint -pyparsing +urllib3==2.5.0 +jupyterlab==4.4.5 +jupyter-book==1.0.4.post1 +docutils==0.21.2 +jupyterlab_myst==2.4.2 +lxml==6.0.0 +markdown_it_py==3.0.0 +mdit_py_plugins==0.4.2 +myst-nb==1.3.0 +myst-parser==3.0.1 +sphinx==7.4.7 +sphinx_inline_tabs==2023.4.21 +tomli==2.2.1 +typing-extensions==4.14.1 +pylint==3.3.7 +pyparsing==3.2.3 numpy==1.26.4 -sphinx_immaterial \ No newline at end of file +sphinx_immaterial==0.13.5 \ No newline at end of file diff --git a/envs/common/01_set_iwrap_env.sh b/envs/common/01_set_iwrap_env.sh index 16a4946..62d12ba 100644 --- a/envs/common/01_set_iwrap_env.sh +++ b/envs/common/01_set_iwrap_env.sh @@ -2,10 +2,6 @@ export PATH=${IWRAP_HOME}/bin:${PATH} -if [[ ! -n $AL_VERSION ]]; then - export AL_VERSION=$UAL_VERSION -fi - export PYTHONPATH=${IWRAP_HOME}:${PYTHONPATH} export TESTS_DIR="${IWRAP_HOME}/tests" diff --git a/iwrap/__init__.py b/iwrap/__init__.py index 97cccb7..e549516 100644 --- a/iwrap/__init__.py +++ b/iwrap/__init__.py @@ -2,5 +2,14 @@ IWRAP_DIR = os.path.dirname(os.path.realpath(__file__)) -from . import _version -__version__ = _version.get_versions()['version'] +try: + from importlib.metadata import version, PackageNotFoundError +except ImportError: + # Python < 3.8 + from importlib_metadata import version, PackageNotFoundError + +try: + __version__ = version("iwrap") +except PackageNotFoundError: + # Package is not installed + __version__ = "unknown" diff --git a/iwrap/_version.py b/iwrap/_version.py deleted file mode 100644 index ed0a4d5..0000000 --- a/iwrap/_version.py +++ /dev/null @@ -1,665 +0,0 @@ - -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. -# Generated by versioneer-0.28 -# https://github.com/python-versioneer/python-versioneer - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys -from typing import Callable, Dict -import functools - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "$Format:%d$" - git_describe_output = "$Format:%(describe)$" - git_full = "$Format:%H$" - git_date = "$Format:%ci$" - mo_tag_in_git_refnames = re.search(r'(\d+\.)(\d+\.)(\d+)', git_refnames) - if not mo_tag_in_git_refnames: - mo_tag_in_git_describe = re.search(r'(\d+\.)(\d+\.)(\d+)', git_describe_output) - if mo_tag_in_git_describe: - mo_describe = '(, tag: '+ git_describe_output +',,'+ git_refnames + ')' - git_refnames = mo_describe - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "pep440" - cfg.tag_prefix = "" - cfg.parentdir_prefix = "" - cfg.versionfile_source = "iwrap/_version.py" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY: Dict[str, str] = {} -HANDLERS: Dict[str, Dict[str, Callable]] = {} - - -def register_vcs_handler(vcs, method): # decorator - """Create decorator to mark a method as the handler of a VCS.""" - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - process = None - - popen_kwargs = {} - if sys.platform == "win32": - # This hides the console window if pythonw.exe is used - startupinfo = subprocess.STARTUPINFO() - startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW - popen_kwargs["startupinfo"] = startupinfo - - for command in commands: - dispcmd = str([command] + args) - try: - # remember shell=False, so use git.cmd on windows, not just git - process = subprocess.Popen([command] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None), **popen_kwargs) - break - except OSError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None, None - stdout = process.communicate()[0].strip().decode() - if process.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, process.returncode - return stdout, process.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for _ in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print("Tried directories %s but none started with prefix %s" % - (str(rootdirs), parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - with open(versionfile_abs, "r") as fobj: - for line in fobj: - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - except OSError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if "refnames" not in keywords: - raise NotThisMethod("Short version file found") - date = keywords.get("date") - if date is not None: - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = {r.strip() for r in refnames.strip("()").split(",")} - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = {r for r in refs if re.search(r'\d', r)} - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] - # Filter out refs that exactly match prefix or that don't start - # with a number once the prefix is stripped (mostly a concern - # when prefix is '') - if not re.match(r'\d', r): - continue - if verbose: - print("picking %s" % r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - # GIT_DIR can interfere with correct operation of Versioneer. - # It may be intended to be passed to the Versioneer-versioned project, - # but that should not change where we get our version from. - env = os.environ.copy() - env.pop("GIT_DIR", None) - runner = functools.partial(runner, env=env) - - _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=not verbose) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = runner(GITS, [ - "describe", "--tags", "--dirty", "--always", "--long", - "--match", f"{tag_prefix}[[:digit:]]*" - ], cwd=root) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], - cwd=root) - # --abbrev-ref was added in git-1.6.3 - if rc != 0 or branch_name is None: - raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") - branch_name = branch_name.strip() - - if branch_name == "HEAD": - # If we aren't exactly on a branch, pick a branch which represents - # the current commit. If all else fails, we are on a branchless - # commit. - branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) - # --contains was added in git-1.5.4 - if rc != 0 or branches is None: - raise NotThisMethod("'git branch --contains' returned error") - branches = branches.split("\n") - - # Remove the first line if we're running detached - if "(" in branches[0]: - branches.pop(0) - - # Strip off the leading "* " from the list of branches. - branches = [branch[2:] for branch in branches] - if "master" in branches: - branch_name = "master" - elif not branches: - branch_name = None - else: - # Pick the first branch that is returned. Good or bad. - branch_name = branches[0] - - pieces["branch"] = branch_name - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) - if not mo: - # unparsable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%s'" - % describe_out) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" - % (full_tag, tag_prefix)) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) - pieces["distance"] = len(out.split()) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_branch(pieces): - """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . - - The ".dev0" means not master branch. Note that .dev0 sorts backwards - (a feature branch will appear "older" than the master branch). - - Exceptions: - 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0" - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def pep440_split_post(ver): - """Split pep440 version string at the post-release segment. - - Returns the release segments before the post-release and the - post-release version number (or -1 if no post-release segment is present). - """ - vc = str.split(ver, ".post") - return vc[0], int(vc[1] or 0) if len(vc) == 2 else None - - -def render_pep440_pre(pieces): - """TAG[.postN.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post0.devDISTANCE - """ - if pieces["closest-tag"]: - if pieces["distance"]: - # update the post release segment - tag_version, post_version = pep440_split_post(pieces["closest-tag"]) - rendered = tag_version - if post_version is not None: - rendered += ".post%d.dev%d" % (post_version + 1, pieces["distance"]) - else: - rendered += ".post0.dev%d" % (pieces["distance"]) - else: - # no commits, use the tag as the version - rendered = pieces["closest-tag"] - else: - # exception #1 - rendered = "0.post0.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_post_branch(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . - - The ".dev0" means not master branch. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-branch": - rendered = render_pep440_branch(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-post-branch": - rendered = render_pep440_post_branch(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, - verbose) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for _ in cfg.versionfile_source.split('/'): - root = os.path.dirname(root) - except NameError: - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None} - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", "date": None} diff --git a/iwrap/generators/actor_generators/muscle3_common/__init__.py b/iwrap/generators/actor_generators/muscle3_common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/iwrap/generators/actor_generators/muscle3_common/dumper.py b/iwrap/generators/actor_generators/muscle3_common/dumper.py new file mode 100644 index 0000000..fe2f18b --- /dev/null +++ b/iwrap/generators/actor_generators/muscle3_common/dumper.py @@ -0,0 +1,27 @@ +import yaml + + +class CustomDumper(yaml.SafeDumper): + def ignore_aliases(self, data): + return True + + #dump empty sequence as: ' ' (nothing) + def represent_sequence(self, tag, sequence, flow_style=None): + if not sequence: # Check if the sequence is empty + return super().represent_scalar(u'tag:yaml.org,2002:null', u'') + return super().represent_sequence(tag, sequence, flow_style) + + # dump empty dict as: ' ' (nothing) + def represent_mapping(self, tag, mapping, flow_style=None): + if not mapping: # Check if the sequence is empty + return super().represent_scalar(u'tag:yaml.org,2002:null', u'') + return super().represent_mapping(tag, mapping, flow_style) + + def represent_none(self, data): + return super().represent_scalar('tag:yaml.org,2002:null', u'') + + # dump empty dict as: ' ' (nothing) + def represent_scalar(self, tag, value, style=None): + if not value or value.lower() == 'null': # Check if value is None or 'null' + return super().represent_scalar(u'tag:yaml.org,2002:null', u'') + return super().represent_scalar(tag, value, style) diff --git a/iwrap/generators/actor_generators/muscle3_common/m3_utils.py b/iwrap/generators/actor_generators/muscle3_common/m3_utils.py new file mode 100644 index 0000000..cef8795 --- /dev/null +++ b/iwrap/generators/actor_generators/muscle3_common/m3_utils.py @@ -0,0 +1,99 @@ +import os +import shutil +from typing import List + +import yaml +from .dumper import CustomDumper + + +def template_filter_func(templates_list: List[str]) -> bool: + if "__pycache__" in templates_list: + return False + + if "macros" in templates_list: + return False + + return True + +def copy_native_lib(install_dir: str, project_settings:dict, destination_dir:str): + + native_lib_path = project_settings['code_description'].get('implementation', {}).get('code_path') + if not native_lib_path: + return + + destination_dir = os.path.join( install_dir, destination_dir) + if not os.path.isdir( destination_dir ): + os.makedirs( destination_dir ) + + shutil.copy( native_lib_path, destination_dir ) + +def copy_include(install_dir: str, project_settings:dict): + + include_path = project_settings['code_description'].get('implementation', {}).get('include_path') + if not include_path: + return + + destination_dir = os.path.join( install_dir, 'include' ) + if not os.path.isdir( destination_dir ): + os.makedirs( destination_dir ) + + shutil.copy( include_path, destination_dir ) + +def copy_extra_libs(install_dir: str, project_settings: dict): + + libraries = project_settings['code_description'].get('settings', {}).get('extra_libraries', {}).get('path_defined') + + if not libraries: + return + + destination_dir = os.path.join( install_dir, 'extra-libs' ) + if not os.path.isdir( destination_dir ): + os.makedirs( destination_dir ) + + for library_path in libraries: + shutil.copy( library_path, destination_dir ) + + +def copy_code_params_files(install_dir: str, project_settings:dict): + code_parameters = project_settings['code_description'].get('implementation', {}).get('code_parameters') + if not code_parameters: + return + + parameters_file = code_parameters.get('parameters') + schema_file = code_parameters.get('schema') + + if not parameters_file: + return + + if parameters_file and not schema_file: + raise Exception('Error! Code parameters schema file (XSD) is missing!') + + destination_dir = os.path.join(install_dir, 'input') + if not os.path.isdir( destination_dir ): + os.makedirs( destination_dir ) + + shutil.copy( parameters_file, destination_dir ) + shutil.copy( schema_file, destination_dir ) + +def copy_build_info(install_dir: str, project_settings:dict): + output_dict = dict((key, project_settings[key]) for key in project_settings if key in ['actor_description','code_description','build_info']) + with open(f'{install_dir}/{project_settings["actor_description"]["actor_name"]}.yaml', "w") as file: + yaml.dump(output_dict, file, Dumper=CustomDumper) + + +def validate(project_settings: dict = None): + + init_arguments = project_settings['code_description'] \ + .get('implementation', {}) \ + .get('subroutines', {}) \ + .get('init', {})\ + .get('arguments') + + finalize_arguments = project_settings['code_description'] \ + .get('implementation', {}) \ + .get('subroutines', {}) \ + .get('finalize', {})\ + .get('arguments') + + if init_arguments or finalize_arguments: + raise ValueError( 'MUSCLE3 actor generator cannot handle INIT/FINALIZE methods with IDS arguments' ) \ No newline at end of file diff --git a/iwrap/generators/actor_generators/muscle3_cpp/__init__.py b/iwrap/generators/actor_generators/muscle3_cpp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/iwrap/generators/actor_generators/muscle3_cpp/m3_cpp_actor.py b/iwrap/generators/actor_generators/muscle3_cpp/m3_cpp_actor.py new file mode 100644 index 0000000..d977ef9 --- /dev/null +++ b/iwrap/generators/actor_generators/muscle3_cpp/m3_cpp_actor.py @@ -0,0 +1,120 @@ +import logging +import os +import shutil +import subprocess +import tempfile +from pathlib import Path +from typing import Set, List + + +from iwrap.generation_engine.utils.jinja2_template_processing import process_template_dir +from iwrap.generators.actor_generators import ActorGenerator +from iwrap.settings.project import ProjectSettings + +import jinja2 +import sys + +from iwrap.settings.platform.platform_settings import PlatformSettings + + +from iwrap.generators.actor_generators.muscle3_common import m3_utils + + +class CppActorGenerator(ActorGenerator): + # Class logger + __logger = logging.getLogger(__name__ + "." + __qualname__) + + COMPLIANT_API = '2.1' + + @property + def type(self) -> str: + return 'MUSCLE3-CPP' + + @property + def name(self) -> str: + return 'MUSCLE3 (C++)' + + @property + def description(self) -> str: + return 'Wrapping C++ code into MUSCLE3 micro model' + + @property + def actor_language(self) -> List[str]: + return 'cpp' + + @property + def actor_data_types(self) -> List[str]: + return ['legacy'] + + @property + def code_data_types(self) -> List[str]: + return ['legacy'] + + @property + def code_languages(self) -> Set[str]: + return {'cpp'} + + def __init__(self): + + self.__info_output_stream = None + self.temp_dir: tempfile.TemporaryDirectory = None + self.jinja_env: jinja2.Environment = None + self.install_dir: str = None + self.skeleton_generation: bool = False + + def configure(self, info_output_stream=sys.stdout): + self.__info_output_stream = info_output_stream + + def validate(self, project_settings: dict = None): + m3_utils.validate( project_settings ) + + def initialize(self, project_settings: dict = None): + install_dir = None + + if project_settings: + install_dir = project_settings['actor_description'].get( 'install_dir' ) + programming_language = project_settings['code_description'].get('implementation', {}).get('programming_language') + if not programming_language: + self.skeleton_generation = True + + if not install_dir: + install_dir = PlatformSettings().directories.actor_install_dir + self.install_dir: str = str(Path(install_dir, ProjectSettings.get_settings().actor_description.actor_name)) + + def generate(self, project_settings: dict): + self.cleanup( project_settings ) + os.makedirs( self.install_dir, exist_ok=True ) + + current_path = os.path.dirname(os.path.realpath(__file__)) + process_template_dir( None, current_path + '/resources', self.install_dir, project_settings, + filter_func=m3_utils.template_filter_func, + output_stream= self.__info_output_stream, ) + + m3_utils.copy_native_lib( self.install_dir, project_settings, "lib" ) + m3_utils.copy_include( self.install_dir, project_settings ) + m3_utils.copy_extra_libs( self.install_dir, project_settings ) + m3_utils.copy_code_params_files( self.install_dir, project_settings ) + m3_utils.copy_build_info( self.install_dir, project_settings ) + + def build(self, project_settings: dict): + + if self.skeleton_generation: + return + + proc = subprocess.Popen( [], executable = "make", cwd=self.install_dir, + encoding='utf-8', text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) + + for line in proc.stdout: + print( line, file=self.__info_output_stream, end='' ) + + return_code = proc.wait() + if return_code: + raise subprocess.CalledProcessError( return_code, 'make' ) + + def install(self, project_settings: dict): + ... + + def cleanup(self, project_settings: dict): + shutil.rmtree(self.install_dir, ignore_errors=True) + + diff --git a/iwrap/generators/actor_generators/muscle3_cpp/resources/Makefile.jinja2 b/iwrap/generators/actor_generators/muscle3_cpp/resources/Makefile.jinja2 new file mode 100644 index 0000000..53c9769 --- /dev/null +++ b/iwrap/generators/actor_generators/muscle3_cpp/resources/Makefile.jinja2 @@ -0,0 +1,76 @@ +# actor name +ACTOR_NAME={{actor_description.actor_name}} + + +{% if code_description.settings.mpi_compiler_cmd %} +CXX={{ code_description.settings.mpi_compiler_cmd }} + +{% elif code_description.settings.compiler_cmd %} +CXX={{ code_description.settings.compiler_cmd }} +{% endif %} + + +WRAPPER_DIR=$(dir $(realpath $(firstword $(MAKEFILE_LIST)))) + +COMPILER_FLAGS={{ code_description.settings.compiler_flags or '' }} + +CXXFLAGS= -g -fpic -pthread $(COMPILER_FLAGS) + +CODE_LIB_DEF= +{%- if code_description.implementation.code_path -%} +./lib/{{code_description.implementation.code_path | basename}} +{% endif %} + +{% if code_description.settings.mpi_compiler_cmd %} +MUSCLE3_PKG=libmuscle_mpi +{% else %} +MUSCLE3_PKG=libmuscle +{% endif %} + +# main IMAS libs to be used +{% if build_info.al_version.startswith('4.') %} +IMAS_LIB_NAME=imas-cpp +{% else %} +IMAS_LIB_NAME=al-cpp +{% endif %} +CORE_LIBS_FLAGS= $(shell pkg-config --cflags $(IMAS_LIB_NAME) ymmsl $(MUSCLE3_PKG)) +CORE_LIBS=$(shell pkg-config --libs $(IMAS_LIB_NAME) ymmsl $(MUSCLE3_PKG)) + +# required libs published using pkg-config mechanism +{% if code_description.settings.extra_libraries and code_description.settings.extra_libraries.pkg_config_defined %} +LIBS_PKGCONFIG_DEFINED_NAMES={{ (code_description.settings.extra_libraries.pkg_config_defined or []) | join(' ')}} +LIBS_PKGCONFIG_DEFINED_FLAGS=$(shell pkg-config --cflags $(LIBS_PKGCONFIG_DEFINED_NAMES)) +LIBS_PKGCONFIG_DEFINED=$(shell pkg-config --libs $(LIBS_PKGCONFIG_DEFINED_NAMES)) +{% else %} +LIBS_PKGCONFIG_DEFINED_NAMES= +LIBS_PKGCONFIG_DEFINED_FLAGS= +LIBS_PKGCONFIG_DEFINED= +{% endif %} + + +# required libs provided by user as a path +LIBS_PATH_DEFINED= +{%- if code_description.settings %} +{%- for lib_path in code_description.settings.extra_libraries.path_defined or [] %} + ./extra-libs/{{ lib_path | basename }} +{%- endfor %} +{% endif %} + +INCLUDES=-I./include $(CORE_LIBS_FLAGS) $(LIBS_PKGCONFIG_DEFINED_FLAGS) +LIBS=$(CORE_LIBS) $(LIBS_PKGCONFIG_DEFINED) $(LIBS_PATH_DEFINED) $(CODE_LIB_DEF) + + +all: ./bin/$(ACTOR_NAME).exe + +./bin/$(ACTOR_NAME).exe: src/standalone.cpp build/muscle3_tools.o + @echo " * $^ -> $@" + @mkdir -p ./bin + $(CXX) $(CXXFLAGS) $(INCLUDES) -o $@ -Wl,--start-group $^ $(LIBS) -Wl,--end-group -Wl,-rpath,$(WRAPPER_DIR)/extra-libs/ + +build/%.o: src/%.cpp + @echo " * $^ -> $@" + @mkdir -p build + $(CXX) $(CXXFLAGS) -c -w $^ -o $@ $(INCLUDES) + +clean: + -rm -f *.o build/* ./bin/* diff --git a/iwrap/generators/actor_generators/muscle3_cpp/resources/macros/flags.jinja2 b/iwrap/generators/actor_generators/muscle3_cpp/resources/macros/flags.jinja2 new file mode 100644 index 0000000..feb9158 --- /dev/null +++ b/iwrap/generators/actor_generators/muscle3_cpp/resources/macros/flags.jinja2 @@ -0,0 +1,5 @@ +{%- set checkpointable = code_description.implementation.subroutines.get_state and code_description.implementation.subroutines.set_state %} +{%- set code_parameters_provided = code_description.implementation.code_parameters.parameters %} +{%- set mpi_code = code_description.settings.mpi_compiler_cmd -%} + +{%- set skeleton_only = not code_description.implementation.programming_language -%} diff --git a/iwrap/generators/actor_generators/muscle3_cpp/resources/macros/legacy_ids.jinja2 b/iwrap/generators/actor_generators/muscle3_cpp/resources/macros/legacy_ids.jinja2 new file mode 100644 index 0000000..fea7695 --- /dev/null +++ b/iwrap/generators/actor_generators/muscle3_cpp/resources/macros/legacy_ids.jinja2 @@ -0,0 +1,26 @@ +{% macro imports(al_version) -%} + +{% if al_version.startswith('4.') %} +#include "UALClasses.h" +{% else %} +#include "ALClasses.h" +{% endif %} +{%- endmacro -%} + +{% macro declare(ids_name, ids_var_name) -%} + IdsNs::IDS::{{ ids_name }} {{ ids_var_name }}; +{%- endmacro -%} + +{%- macro argument(ids_name, ids_var_name) -%} + IdsNs::IDS::{{ ids_name }}& {{ ids_var_name }} +{%- endmacro -%} + +{%- macro provenance(ids_var_name, sbrt_name) -%} + {{ ids_var_name }}.code.name = "{{sbrt_name}}"; + {{ ids_var_name }}.code.version = ""; +{%- endmacro -%} + +{% macro deallocate(ids_var_name) -%} + // Should not be called until IMAS-3937 will be resolved + // {{ ids_var_name }}.clear(); +{%- endmacro -%} \ No newline at end of file diff --git a/iwrap/generators/actor_generators/muscle3_cpp/resources/src/muscle3_tools.cpp.jinja2 b/iwrap/generators/actor_generators/muscle3_cpp/resources/src/muscle3_tools.cpp.jinja2 new file mode 100644 index 0000000..e2cf310 --- /dev/null +++ b/iwrap/generators/actor_generators/muscle3_cpp/resources/src/muscle3_tools.cpp.jinja2 @@ -0,0 +1,356 @@ +{% import './macros/%s_ids.jinja2' % code_description.implementation.data_type as ids_macro %} + +{%- import './macros/flags.jinja2' as flags with context-%} + +#include +#include +#include +#include + +//for code_parameters copying +#include +#include +#include +#include +#include + +{{ ids_macro.imports(build_info.al_version) }} + +#include "muscle3_tools.h" + +using libmuscle::Data; +using libmuscle::DataConstRef; +using libmuscle::Message; +using ymmsl::Operator; + +{% if flags.mpi_code %} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// INIT MPI +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +int init_mpi() +{ + int mpi_rank; + int was_mpi_initialized; + + MPI_Initialized(&was_mpi_initialized); + if (!was_mpi_initialized) + MPI_Init(NULL, NULL); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + + return mpi_rank; +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// FINISH MPI +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +void finish_mpi() +{ + int was_mpi_finalized; + //---- MPI Finalization ---- + MPI_Finalized(&was_mpi_finalized); + if (!was_mpi_finalized) + MPI_Finalize(); +} + + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// BROADCAST IDS +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +void broadcast_ids(char** serialized_ids, int *serialized_ids_size) +{ + // Share information among all processors + MPI_Bcast(serialized_ids_size, 1, MPI_INTEGER, MPI_ROOT_RANK, MPI_COMM_WORLD); + + if( *serialized_ids == NULL && *serialized_ids_size > 0) // Not a MPI root + { + *serialized_ids = (char*) malloc(*serialized_ids_size); + } + + MPI_Bcast( *serialized_ids, *serialized_ids_size, MPI_CHARACTER, MPI_ROOT_RANK, MPI_COMM_WORLD); +} +{% endif %} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// INIT MUSCLE3 +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +libmuscle::Instance* init_muscle3() +{ + libmuscle::Instance* m3_instance; + std::vector input_ports; + std::vector output_ports; + {% if flags.checkpointable %} + libmuscle::InstanceFlags flags = libmuscle::InstanceFlags::USES_CHECKPOINT_API; + {% else %} + libmuscle::InstanceFlags flags = libmuscle::InstanceFlags::NONE; + {% endif %} + + /* * * * - - - INPUT ('F_INIT') ports - - - * * * */ +{% for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'IN' %} + // {{ argument.name }} + input_ports.push_back("{{ argument.name }}"); + +{% endfor %} + + /* * * * - - - OUTPUT ('O_F') ports - - - * * * */ +{% for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'OUT' %} + // {{ argument.name }} + output_ports.push_back("{{ argument.name }}"); + + {% endfor %} + + + {% if flags.mpi_code %} + m3_instance = new libmuscle::Instance(0, NULL, + {% raw %}{{Operator::F_INIT, input_ports}, {Operator::O_F, output_ports}}, {% endraw %} + flags, + MPI_COMM_WORLD, MPI_ROOT_RANK); + {% else %} + m3_instance = new libmuscle::Instance(0, NULL, + {% raw %} {{Operator::F_INIT, input_ports}, {Operator::O_F, output_ports}}, {% endraw %} + flags); + {% endif %} + + return m3_instance; +} + + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// FINISH MUSCLE3 +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +void finish_muscle3(libmuscle::Instance* m3_instance) +{ + + delete(m3_instance); +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// RECEIVE IDS +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +void receive_ids(libmuscle::Instance* m3_instance, std::string argument_name, IdsNs::Ids* ids) +{ + char* serialized_ids = NULL; + int serialized_ids_size = 0; + + auto msg = m3_instance->receive(argument_name); + libmuscle::DataConstRef data = msg.data(); + + if(! data.is_nil() ) // MPI root or a single process + { + serialized_ids_size = data.size(); + serialized_ids = (char*)data.as_byte_array(); + } + + {% if flags.mpi_code %} + broadcast_ids(&serialized_ids, &serialized_ids_size); + {% endif %} + + std::string ids_string(serialized_ids, serialized_ids_size); + + ids->deserialize(ids_string); + + if( data.is_nil() ) // should be called for MPI run (not a root process) + free(serialized_ids); +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// GET INPUT IDSES +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +void receive_input_idses( +{% for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'IN' %} + {{ ids_macro.argument(argument.type, argument.name) }}, +{% endfor %} + libmuscle::Instance* m3_instance) +{ + + {% for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'IN' %} + //--------- RECEIVE IDS : {{ argument.name }} ------------------------ + receive_ids(m3_instance, "{{ argument.name }}", &{{ argument.name }}); + +{% endfor %} +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// SEND IDS +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +void send_ids(libmuscle::Instance* m3_instance, std::string argument_name, IdsNs::Ids* ids, double timestamp) +{ + std::string ids_str; + + ids_str = ids->serialize(DEFAULT_SERIALIZER_PROTOCOL); + auto ids_bytes = libmuscle::Data::byte_array(ids_str.data(), ids_str.size()); + libmuscle::Message msg( timestamp, ids_bytes); + + m3_instance->send(argument_name, msg); +} + + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// SEND OUTPUT IDSES +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +void send_output_idses(double timestamp, +{%- for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'OUT' -%} + {{ ids_macro.argument(argument.type, argument.name) }}, +{% endfor %} + libmuscle::Instance* m3_instance) +{ + int time_size = -1.0; + + + + {% for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'OUT' %} + //--------- SEND IDS : {{ argument.name }} ------------------------ + {% if not code_description.implementation.subroutines.get_timestamp %} + timestamp = -1.0; // resetting value + time_size = {{ argument.name }}.time.size(); + if (time_size > 0) + { + timestamp = {{ argument.name }}.time(time_size - 1); + } + {% endif %} + send_ids(m3_instance, "{{ argument.name }}", &{{ argument.name }}, timestamp); + +{% endfor %} +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// READ CODE PARAMETERS +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +void read_code_parameters(libmuscle::Instance* m3_instance, char* exec_path, IdsNs::codeparam_t &imas_code_params) +{ + std::string code_parameters_file_path; + std::string exec_path_str(exec_path); + bool copy_xml_parameters = false; + std::string default_file_name = "{{ (code_description.implementation.code_parameters.parameters or "") | basename }}"; + + try + { + // checking if property 'parameters_file' can be read + code_parameters_file_path = m3_instance->get_setting_as("parameters_file"); + } + catch (std::out_of_range& exc) + { + //Value for setting parameters_file was not set. Using default file. + string directory; + const size_t last_slash_idx = exec_path_str.rfind('/'); + directory = exec_path_str.substr(0, last_slash_idx); + code_parameters_file_path = directory + "/../input/" + default_file_name; + } + + read_code_parameters_file(code_parameters_file_path, imas_code_params); + + try + { + // checking if property 'copy_code_parameters' can be read + copy_xml_parameters = m3_instance->get_setting_as("copy_xml_parameters"); + + if(copy_xml_parameters) + { + // copy code parameters into workdir + // create file descriptors + int fd_in = open(code_parameters_file_path.c_str(),O_RDONLY); + int fd_out = open(default_file_name.c_str(),O_RDWR); + struct stat stat_buf ; + + fstat(fd_in,&stat_buf); + ssize_t size = sendfile(fd_out,fd_in,0,stat_buf.st_size); + + close(fd_in); + close(fd_out); + } + } + catch (std::out_of_range& exc) + { + //just do not copy parameters file + } +} + +void read_code_parameters_file(std::string file_path, IdsNs::codeparam_t &imas_code_params) +{ + std::string code_parameters; + char* cstr; + + code_parameters = read_file( file_path); + + cstr = (char*) malloc(code_parameters.size() + 1); + strcpy(cstr, code_parameters.c_str()); + + *(imas_code_params.parameters) = cstr; + imas_code_params.default_param = NULL; + imas_code_params.schema = NULL; +} + +std::string read_file(std::string file_path) +{ + std::ifstream file; + std::stringstream buffer; + std::string file_content; + + file = std::ifstream (file_path); + buffer << file.rdbuf(); + file_content = buffer.str(); + return file_content; +} + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// XXX XXX +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +void handle_status_info(int status_code, std::string status_message, const std::string actor_name) +{ + std::string message = ""; + if(status_message.empty()) + { + message = status_message; + } + + if(status_code !=0) { + printf("---Diagnostic information returned from *** %s ***:---\n", actor_name); + printf("-------Status code : %d\n", status_code); + printf("-------Status message : %s\n", message); + printf("---------------------------------------------------------\n"); + } +} + + +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +// RESTORING CODE STATE +// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +{% if flags.checkpointable %} +std::string receive_code_state(libmuscle::Instance* m3_instance) +{ + std::string state_str; + + //---- Status info ---- + int status_code = 0; + std::string status_message; + + libmuscle::Message m3_message = m3_instance->load_snapshot(); + libmuscle::DataConstRef m3_data = m3_message.data(); + + if (!m3_data.is_nil()) { + state_str = m3_data.as(); + + return state_str; + + } +} // receive_code_state + + +void save_code_state(libmuscle::Instance* m3_instance, std::string state_str) +{ + double timestamp = -1.0; + + //---- Status info ---- + int status_code = 0; + std::string status_message; + + libmuscle::DataConstRef m3_data = libmuscle::DataConstRef(state_str); + libmuscle::Message m3_message(timestamp, m3_data); + m3_instance->save_final_snapshot(m3_message); + + } // save_code_state +{% else %} +// Subroutines not generated - no GET_STATE/SET_STATE methods provided +{% endif %} \ No newline at end of file diff --git a/iwrap/generators/actor_generators/muscle3_cpp/resources/src/muscle3_tools.h.jinja2 b/iwrap/generators/actor_generators/muscle3_cpp/resources/src/muscle3_tools.h.jinja2 new file mode 100644 index 0000000..9ac7f49 --- /dev/null +++ b/iwrap/generators/actor_generators/muscle3_cpp/resources/src/muscle3_tools.h.jinja2 @@ -0,0 +1,55 @@ +{% import './macros/%s_ids.jinja2' % code_description.implementation.data_type as ids_macro %} +{%- import './macros/flags.jinja2' as flags with context-%} + + +#ifndef _IWRAP_TOOLS +#define _IWRAP_TOOLS + +#include +{{ ids_macro.imports(build_info.al_version) }} + + +const int MPI_ROOT_RANK = 0; + +{% if flags.mpi_code %} +int init_mpi(); +void finish_mpi(); +{% endif %} + +libmuscle::Instance* init_muscle3(); +void finish_muscle3(libmuscle::Instance* m3_instance); + +void send_ids(libmuscle::Instance* m3_instance, std::string argument_name, IdsNs::Ids* ids, double timestamp); +void send_output_idses(double timestamp, +{% for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'OUT' %} + {{ ids_macro.argument(argument.type, argument.name) }}, +{% endfor %} + libmuscle::Instance* m3_instance); + +void receive_ids(libmuscle::Instance* m3_instance, std::string argument_name, IdsNs::Ids* ids); +void receive_input_idses( +{% for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'IN' %} + {{ ids_macro.argument(argument.type, argument.name) }}, +{% endfor %} + libmuscle::Instance* m3_instance); + +void read_code_parameters(libmuscle::Instance* m3_instance, char* exec_path, IdsNs::codeparam_t &imas_code_params); + +void read_code_parameters_file(std::string file_path, IdsNs::codeparam_t &imas_code_params); + +std::string read_file(std::string file_path); + +void handle_status_info(int status_code, std::string status_message, const std::string actor_name); + +void release_status_info(char* status_message); + +// RESTORING CODE STATE +{% if flags.checkpointable %} +std::string receive_code_state(libmuscle::Instance* m3_instance); +void save_code_state(libmuscle::Instance* m3_instance, std::string state_str); +{% else %} +// Subroutines not generated - no GET_STATE/SET_STATE methods provided +{% endif %} + + +#endif // _IWRAP_TOOLS \ No newline at end of file diff --git a/iwrap/generators/actor_generators/muscle3_cpp/resources/src/standalone.cpp.jinja2 b/iwrap/generators/actor_generators/muscle3_cpp/resources/src/standalone.cpp.jinja2 new file mode 100644 index 0000000..3228bea --- /dev/null +++ b/iwrap/generators/actor_generators/muscle3_cpp/resources/src/standalone.cpp.jinja2 @@ -0,0 +1,217 @@ +{% import './macros/%s_ids.jinja2' % code_description.implementation.data_type as ids_macro -%} + +{%- import './macros/flags.jinja2' as flags with context -%} +{%- if flags.skeleton_only %} + {% set comment_out = '//' -%} +{% endif %} + +{%- macro skeleton_call(method_name) -%} + // >>> Subroutine {{ method_name }} should be called here <<< +{%- endmacro -%} + +#include +{% if flags.mpi_code %} + +#include +{% endif %} +#include "muscle3_tools.h" +{{ ids_macro.imports(build_info.al_version) }} +{%if code_description.implementation.include_path %} +#include "{{code_description.implementation.include_path | basename }}" +{% else %} +// Put '#include ".h"' here +{% endif %} + + +int main(int argc, char **argv) +{ + libmuscle::Instance* m3_instance; + double timestamp = -1.0; + + int mpi_rank = 0; + //---- Status info ---- + int status_code = 0; + std::string status_message; + //---- Code parameters ---- + IdsNs::codeparam_t imas_code_params; +{% if flags.checkpointable %} + std::string state_str; +{% endif %} + +{% if code_description.implementation.subroutines.init.name or flags.skeleton_only %} + bool initial_run = true; +{% endif %} + + //---- IN/OUT IDSes ---- +{% for argument in code_description.implementation.subroutines.main.arguments %} + {{ ids_macro.declare(argument.type, argument.name) }} +{% endfor %} + + /* * * * - - - INITIALISATION - - - * * * */ +{% if flags.mpi_code %} + mpi_rank = init_mpi(); +{% endif %} + m3_instance = init_muscle3(); + + {% if flags.code_parameters_provided %} + //---- Code parameters ---- + read_code_parameters(m3_instance, argv[0], "{{code_description.implementation.code_parameters.parameters | basename }}", imas_code_params); + {% endif %} + + /* * * * - - - MUSCLE3 LOOP - - - * * * */ + while (m3_instance->reuse_instance()) { + + // - - - WRAPPED CODE INITIALIZATION - - - +{% if flags.checkpointable %} + // > > > RESTART : the code is restored from the saved snapshot < < < + if (m3_instance->resuming()) { + if (mpi_rank == MPI_ROOT_RANK){ + state_str = receive_code_state(m3_instance); + // Calling SET_STATE + {% if flags.skeleton_only %} + // + // >>> Subroutine "SET_STATE" should be called here <<< + // + {% endif %} + {{ comment_out }} {{code_description.implementation.subroutines.set_state}}(state_str, status_code, status_message); + + } + } +{% endif %} + +{%- if code_description.implementation.subroutines.init.name %} + {% if flags.checkpointable %} + else // m3_instance is not resuming + { // > > > INITIALIZATION by calling INIT method of the code < < < + {% endif %} + + /* * * * - - - WRAPPED CODE CALL - INIT SBRT - - - * * * */ + if (initial_run) { + {% if flags.skeleton_only %} + // + // >>> Subroutine "INIT" should be called here <<< + // + {% endif %} + {{ comment_out }} {{code_description.implementation.subroutines.init.name}} ( + {% if flags.code_parameters_provided %} + {{ comment_out }} imas_code_params, + {% endif %} + {{ comment_out }} status_code, status_message); + } + {% if flags.checkpointable %} + } // m3_instance->resuming() + {% endif %} + initial_run = false; + {% endif %} + +{% if flags.checkpointable %} + if(m3_instance->should_init()) { +{% endif %} + + /* * * * - - - RECEIVING INPUT IDSes - - - * * * */ + receive_input_idses( + {% for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'IN' %} + {{ argument.name }}, + {% endfor %} + m3_instance ); + +{% if flags.checkpointable %} + } // m3_instance->should_init() +{% endif %} + + /* * * * - - - WRAPPED CODE CALL - MAIN SBRT - - - * * * */ + {% if flags.skeleton_only %} + // + // >>> Subroutine "MAIN" should be called here <<< + // + {% endif %} + {{ comment_out }} {{code_description.implementation.subroutines.main.name}}( + {% for argument in code_description.implementation.subroutines.main.arguments %} + {{ comment_out }} {{ argument.name }}, + {% endfor %} + {% if flags.code_parameters_provided %} + {{ comment_out }} imas_code_params, + {% endif %} + {{ comment_out }} status_code, status_message ); + + handle_status_info(status_code, status_message, "{{actor_description.actor_name}}"); + + + {% if flags.mpi_code %} + if (mpi_rank == MPI_ROOT_RANK) + { + // --- called only for RANK 0 process +{% endif %} + + /* * * * - - - SENDING OUTPUT IDSes - - - * * * */ + {% if code_description.implementation.subroutines.get_timestamp %} + // > > > GET TIMESTAMP < < < + {% if flags.skeleton_only %} + // + // >>> Subroutine "GET_TIMESTAMP" should be called here <<< + // + {% endif %} + {{ comment_out }} {{code_description.implementation.subroutines.get_timestamp}}(timestamp, status_code, status_message); + handle_status_info(status_code, status_message, "{{actor_description.actor_name}}"); + + {% endif %} + send_output_idses(timestamp, + {% for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'OUT' %} + {{ argument.name }}, + {% endfor %} + m3_instance); + +{% if flags.mpi_code %} + } //The end of section called only for RANK 0 process +{% endif %} + + + +{% if flags.checkpointable %} + // > > > SAVE SNAPSHOT < < < + if (m3_instance->should_save_final_snapshot()) { + if (mpi_rank == MPI_ROOT_RANK ) { + // Calling GET_STATE + {% if flags.skeleton_only %} + // + // >>> Subroutine "GET_STATE" should be called here <<< + // + {% endif %} + {{ comment_out }} {{code_description.implementation.subroutines.get_state}}(state_str, status_code, status_message); + + save_code_state(m3_instance, state_str); + } + } +{% endif %} + + /* * * * - - - CLEAN UP - - - * * * */ + {% for argument in code_description.implementation.subroutines.main.arguments %} + {{ ids_macro.deallocate( argument.name) }} + {% endfor %} + + } // > > > > THE END OF MUSCLE3 LOOP + + /* * * * - - - FINALIZATION - - - * * * */ + {% if code_description.implementation.subroutines.finalize.name %} + + /* * * * - - - WRAPPED CODE CALL - FINISH SBRT - - - * * * */ + {% if flags.skeleton_only %} + // + // >>> Subroutine "FINALIZE" should be called here <<< + // + {% endif %} + {{ comment_out }} {{code_description.implementation.subroutines.finalize.name}} (status_code, status_message); + + handle_status_info(status_code, status_message, "{{actor_description.actor_name}}"); + {% endif %} + + {% if flags.code_parameters_provided %} + // ------Deallocating code parameters ---------------------------- + {% endif %} + + {% if flags.mpi_code %} + finish_mpi(); + {% endif %} + + finish_muscle3(m3_instance); +} diff --git a/iwrap/generators/actor_generators/muscle3_fortran/__init__.py b/iwrap/generators/actor_generators/muscle3_fortran/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/iwrap/generators/actor_generators/muscle3_fortran/m3_fortran_actor.py b/iwrap/generators/actor_generators/muscle3_fortran/m3_fortran_actor.py new file mode 100644 index 0000000..8895537 --- /dev/null +++ b/iwrap/generators/actor_generators/muscle3_fortran/m3_fortran_actor.py @@ -0,0 +1,119 @@ +import logging +import os +import shutil +import subprocess +import tempfile +from pathlib import Path +from typing import Set, List + +from iwrap.generation_engine.utils.jinja2_template_processing import process_template_dir +from iwrap.generators.actor_generators import ActorGenerator +from iwrap.settings.project import ProjectSettings + + +import jinja2 +import sys + +from iwrap.settings.platform.platform_settings import PlatformSettings + +from iwrap.generators.actor_generators.muscle3_common import m3_utils + + +class FortranPAFActorGenerator(ActorGenerator): + # Class logger + __logger = logging.getLogger(__name__ + "." + __qualname__) + + COMPLIANT_API = '2.1' + + @property + def type(self) -> str: + return 'MUSCLE3-Fortran' + + @property + def name(self) -> str: + return 'MUSCLE3 (Fortran)' + + @property + def description(self) -> str: + return 'Wrapping Fortran code into MUSCLE3 micro model' + + @property + def actor_language(self) -> List[str]: + return 'fortran' + + @property + def actor_data_types(self) -> List[str]: + return ['legacy'] + + @property + def code_data_types(self) -> List[str]: + return ['legacy'] + + @property + def code_languages(self) -> Set[str]: + return {'fortran'} + + def __init__(self): + + self.__info_output_stream = None + self.temp_dir: tempfile.TemporaryDirectory = None + self.jinja_env: jinja2.Environment = None + self.install_dir: str = None + self.skeleton_generation = False + + def configure(self, info_output_stream=sys.stdout): + self.__info_output_stream = info_output_stream + + def validate(self, project_settings: dict = None): + m3_utils.validate( project_settings ) + + def initialize(self, project_settings: dict = None): + install_dir = None + + if project_settings: + install_dir = project_settings['actor_description'].get( 'install_dir' ) + programming_language = project_settings['code_description'].get('implementation', {}).get('programming_language') + if not programming_language: + self.skeleton_generation = True + + if not install_dir: + install_dir = PlatformSettings().directories.actor_install_dir + self.install_dir: str = str(Path(install_dir, ProjectSettings.get_settings().actor_description.actor_name)) + + def generate(self, project_settings: dict): + self.cleanup(project_settings) + os.makedirs( self.install_dir, exist_ok=True ) + + current_path = os.path.dirname(os.path.realpath(__file__)) + process_template_dir( None, current_path + '/resources', self.install_dir, project_settings, + filter_func=m3_utils.template_filter_func, + output_stream= self.__info_output_stream, ) + + m3_utils.copy_native_lib( self.install_dir, project_settings, "lib" ) + m3_utils.copy_include( self.install_dir, project_settings ) + m3_utils.copy_extra_libs( self.install_dir, project_settings ) + m3_utils.copy_code_params_files( self.install_dir, project_settings ) + m3_utils.copy_build_info( self.install_dir, project_settings ) + + def build(self, project_settings: dict): + + if self.skeleton_generation: + return + + proc = subprocess.Popen( [], executable = "make", cwd=self.install_dir, + encoding='utf-8', text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) + + for line in proc.stdout: + print( line, file=self.__info_output_stream, end='' ) + + return_code = proc.wait() + if return_code: + raise subprocess.CalledProcessError( return_code, 'make' ) + + def install(self, project_settings: dict): + ... + + def cleanup(self, project_settings: dict): + shutil.rmtree(self.install_dir, ignore_errors=True) + + diff --git a/iwrap/generators/actor_generators/muscle3_fortran/resources/Makefile.jinja2 b/iwrap/generators/actor_generators/muscle3_fortran/resources/Makefile.jinja2 new file mode 100644 index 0000000..ce39209 --- /dev/null +++ b/iwrap/generators/actor_generators/muscle3_fortran/resources/Makefile.jinja2 @@ -0,0 +1,94 @@ +# actor name +ACTOR_NAME={{actor_description.actor_name}} + +{% if code_description.settings.compiler_cmd %} +# Fortran compiler name +FC={{ code_description.settings.compiler_cmd }} +{% endif %} + +{% if code_description.settings.mpi_compiler_cmd %} +# MPI Fortran compiler - can be empty for non-MPI code +MPIFC={{ code_description.settings.mpi_compiler_cmd or '' }} +{% endif %} + +WRAPPER_DIR=$(dir $(realpath $(firstword $(MAKEFILE_LIST)))) + +# Open MP Flag if the code requires it +COMPILER_FLAGS={{ code_description.settings.compiler_flags or '' }} + +MOD_DIR=./build/ + +#TO DO - any clever way to do this? +ifeq ($(FC),$(filter $(FC),ifort ifx)) +# INTEL + FLAGS=-g -fPIC -module $(MOD_DIR) +else +# GFORTRAN + FLAGS=-g -fPIC -J$(MOD_DIR) +endif + +FCLAGS=$(FLAGS) $(COMPILER_FLAGS) + +# the name of static library file containing wrapped code +CODE_LIB_DEF= +{%- if code_description.implementation.code_path -%} +./lib/{{code_description.implementation.code_path | basename}} +{% endif %} + +{% if code_description.settings.mpi_compiler_cmd %} +MUSCLE3_PKG=libmuscle_mpi_fortran +{% else %} +MUSCLE3_PKG=libmuscle_fortran +{% endif %} + +# main IMAS libs to be used +{% if build_info.al_version.startswith('4.') %} +IMAS_LIB_NAME=imas-$(FC) +{% else %} +IMAS_LIB_NAME=al-fortran +{% endif %} +CORE_LIBS_FLAGS:= $(shell pkg-config --cflags "$(IMAS_LIB_NAME) ymmsl $(MUSCLE3_PKG)") +CORE_LIBS:=$(shell pkg-config --libs "$(IMAS_LIB_NAME)" ymmsl $(MUSCLE3_PKG)) + +# required libs published using pkg-config mechanism +{% if code_description.settings and code_description.settings.extra_libraries.pkg_config_defined %} +LIBS_PKGCONFIG_DEFINED_NAMES={{ (code_description.settings.extra_libraries.pkg_config_defined or []) | join(' ')}} +LIBS_PKGCONFIG_DEFINED_FLAGS=$(shell pkg-config --cflags $(LIBS_PKGCONFIG_DEFINED_NAMES)) +LIBS_PKGCONFIG_DEFINED=$(shell pkg-config --libs $(LIBS_PKGCONFIG_DEFINED_NAMES)) +{% else %} +LIBS_PKGCONFIG_DEFINED_NAMES= +LIBS_PKGCONFIG_DEFINED_FLAGS= +LIBS_PKGCONFIG_DEFINED= +{% endif %} + +# required libs provided by user as a path +LIBS_PATH_DEFINED= +{%- if code_description.settings %} +{%- for lib_path in code_description.settings.extra_libraries.path_defined or [] %} + ./extra-libs/{{ lib_path | basename }} +{%- endfor %} +{% endif %} + +INCLUDES=-I./include $(CORE_LIBS_FLAGS) $(LIBS_PKGCONFIG_DEFINED_FLAGS) +LIBS=$(CORE_LIBS) $(LIBS_PKGCONFIG_DEFINED) $(LIBS_PATH_DEFINED) $(CODE_LIB_DEF) + +{% if code_description.settings.mpi_compiler_cmd %} +# MPI compilation wrapper will be used +FC=$(MPIFC) + +{% endif %} +all: ./bin/$(ACTOR_NAME).exe + +./bin/$(ACTOR_NAME).exe: src/standalone.f90 build/fortrantools.o build/muscle3_tools.o + @echo " * $^ -> $@" + @mkdir -p ./bin + $(FC) $(FCLAGS) -o $@ -Wl,--start-group $^ $(INCLUDES) -I$(MOD_DIR) $(LIBS) -Wl,--end-group -Wl,-rpath,$(WRAPPER_DIR)/extra-libs/ + +build/%.o: src/%.f90 + @echo $(PWD) + @echo " * $< -> $@" + @mkdir -p ./build + $(FC) $(FCLAGS) -c -w $^ -o $@ $(INCLUDES) + +clean: + -rm -f *.o build/* ../bin/* diff --git a/iwrap/generators/actor_generators/muscle3_fortran/resources/macros/flags.jinja2 b/iwrap/generators/actor_generators/muscle3_fortran/resources/macros/flags.jinja2 new file mode 100644 index 0000000..7c727ce --- /dev/null +++ b/iwrap/generators/actor_generators/muscle3_fortran/resources/macros/flags.jinja2 @@ -0,0 +1,5 @@ +{%- set checkpointable = code_description.implementation.subroutines.get_state and code_description.implementation.subroutines.set_state %} +{%- set code_parameters_provided = code_description.implementation.code_parameters.parameters %} +{%- set mpi_code = code_description.settings.mpi_compiler_cmd -%} + +{%- set skeleton_only = not code_description.implementation.programming_language -%} \ No newline at end of file diff --git a/iwrap/generators/actor_generators/muscle3_fortran/resources/macros/legacy_ids.jinja2 b/iwrap/generators/actor_generators/muscle3_fortran/resources/macros/legacy_ids.jinja2 new file mode 100644 index 0000000..76f1fcc --- /dev/null +++ b/iwrap/generators/actor_generators/muscle3_fortran/resources/macros/legacy_ids.jinja2 @@ -0,0 +1,26 @@ +{% macro imports() -%} + use ids_schemas + use ids_routines +{%- endmacro -%} + +{% macro declare(ids_name, ids_var_name) -%} + ! {{ ids_var_name }} + type (ids_{{ ids_name }}) :: {{ ids_var_name }} +{%- endmacro -%} + +{%- macro provenance(ids_var_name, sbrt_name) -%} + if (associated({{ ids_var_name }}%code%name)) then + deallocate({{ ids_var_name }}%code%name) + endif + if (associated({{ ids_var_name }}%code%version)) then + deallocate({{ ids_var_name }}%code%version) + endif + allocate({{ ids_var_name }}%code%name(1)) + allocate({{ ids_var_name }}%code%version(1)) + {{ ids_var_name }}%code%name(1) = "{{sbrt_name}}" + {{ ids_var_name }}%code%version(1) = "" +{%- endmacro -%} + +{% macro deallocate(ids_var_name) -%} + call ids_deallocate({{ ids_var_name }}) +{%- endmacro -%} \ No newline at end of file diff --git a/iwrap/generators/actor_generators/muscle3_fortran/resources/src/fortrantools.f90 b/iwrap/generators/actor_generators/muscle3_fortran/resources/src/fortrantools.f90 new file mode 100644 index 0000000..95572dc --- /dev/null +++ b/iwrap/generators/actor_generators/muscle3_fortran/resources/src/fortrantools.f90 @@ -0,0 +1,145 @@ + +{%- import './macros/flags.jinja2' as flags with context-%} + +module iwrap_tools + {% if flags.mpi_code %} + use libmuscle_mpi + {% else %} + use libmuscle + {% endif %} + + implicit none + +contains + + SUBROUTINE check_status(instance, status_code, status_message, actor_name, actor_method) + + type(LIBMUSCLE_Instance), INTENT(IN) :: instance + integer :: status_code + character(*) :: actor_name + character(*) :: actor_method + character(:), pointer :: status_message + character(:), allocatable :: status_info + + if(associated(status_message)) then + status_info = status_message + deallocate(status_message) + else + status_info = "" + end if + + ! NO ERROR + if(status_code == 0) return + + ! WARNING + if(status_code > 0) then + print *, "---WARNING returned from *** ", actor_name, "/", actor_method, " ***:---" + print *, "-------Output flag : ", status_code + print *, "-------Status info: ", status_info + print *, "---------------------------------------------------------" + + end if + + ! ERROR + if(status_code < 0) then + print *, "---WARNING returned from *** ", actor_name, "/", actor_method, " ***:---" + print *, "-------Output flag : ", status_code + print *, "-------Status info: ", status_info + print *, "---------------------------------------------------------" + CALL LIBMUSCLE_Instance_error_shutdown(instance, "*"//actor_name//"/"//actor_method//"* error: "//status_info) + CALL exit(status_code) + end if + + END SUBROUTINE check_status + + + FUNCTION get_exec_dir() RESULT(exec_path) + CHARACTER(len=2055) :: exec_path + integer :: idx + + CALL get_command_argument(0, exec_path) + + idx = index(trim(exec_path), '/', .True.) + exec_path = exec_path(:idx) + + END FUNCTION get_exec_dir + + + FUNCTION read_code_parameters_file(file_name, imas_code_params) RESULT(status) + use ids_schemas, ONLY: ids_parameters_input + character(len=:),allocatable, intent(in) :: file_name + type(ids_parameters_input), intent(OUT) :: imas_code_params + character(len=:), allocatable :: code_parameters_str + integer :: status + integer :: iloopmax, string_size, err_code + + status = read_file(file_name, code_parameters_str) + if (status /= 0) return + + string_size = LEN_TRIM(code_parameters_str) + + iloopmax=string_size/132 + if (mod(string_size,132)/=0) then + iloopmax = iloopmax + 1 + endif + allocate(imas_code_params%parameters_value(iloopmax)) + + imas_code_params%parameters_value = transfer(code_parameters_str(1:string_size), imas_code_params%parameters_value) + + if(mod(string_size,132)/=0) then + imas_code_params%parameters_value(iloopmax)(mod(string_size,132)+1:132) = ' ' + endif + + END FUNCTION read_code_parameters_file + + !--------------------------------------------------- + FUNCTION read_file(filename, str) RESULT(status) + implicit none + + character(len=*),intent(in) :: filename + character(len=:),allocatable, intent(out) :: str + + !local variables: + integer :: iunit,istat,filesize, status + character(len=1) :: c + + status = 0 + + open(newunit=iunit,file=filename,status='OLD',& + form='UNFORMATTED',access='STREAM',iostat=istat) + + if (istat /=0) then + write(*,*) 'Error opening file: ', filename + status = -1 + return + end if + + !how many characters are in the file: + inquire(file=filename, size=filesize) + if (filesize < 1) then + write(*,*) 'Error getting file size: ', filename + status = -1 + return + end if + + !read the file all at once: + allocate( character(len=filesize) :: str ) + read(iunit,pos=1,iostat=istat) str + + if (istat /=0 ) then + write(*,*) 'Error reading file: ', filename + status = -1 + return + end if + + !make sure it was all read by trying to read more: + read(iunit,pos=filesize+1,iostat=istat) c + if (.not. IS_IOSTAT_END(istat)) & + write(*,*) 'Error: file was not completely read.' + + close(iunit, iostat=istat) + end function read_file + +end module iwrap_tools + + diff --git a/iwrap/generators/actor_generators/muscle3_fortran/resources/src/muscle3_tools.f90.jinja2 b/iwrap/generators/actor_generators/muscle3_fortran/resources/src/muscle3_tools.f90.jinja2 new file mode 100644 index 0000000..a347bed --- /dev/null +++ b/iwrap/generators/actor_generators/muscle3_fortran/resources/src/muscle3_tools.f90.jinja2 @@ -0,0 +1,359 @@ +{% import './macros/%s_ids.jinja2' % code_description.implementation.data_type as ids_macro %} + +{%- import './macros/flags.jinja2' as flags with context-%} + +MODULE muscle3_tools + use ymmsl + {% if flags.mpi_code %} + use mpi + use libmuscle_mpi + {% else %} + use libmuscle + {% endif %} + use ids_schemas + use ids_routines + use iwrap_tools + implicit none + + integer, parameter :: MPI_ROOT_RANK = 0 +contains + + +{% if flags.mpi_code %} + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + ! INIT MPI + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + SUBROUTINE init_mpi(process_rank) + + integer, intent(OUT) :: process_rank + integer :: status + logical :: was_initialized + + call MPI_Init(status) + call MPI_COMM_RANK(MPI_COMM_WORLD, process_rank, status) + + END SUBROUTINE init_mpi + + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + ! FINISH MPI + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + SUBROUTINE finish_mpi() + + integer :: status + logical :: was_mpi_finalized + + call MPI_finalized(was_mpi_finalized, status) + if (.not. was_mpi_finalized) call MPI_Finalize(status) + + END SUBROUTINE finish_mpi + + + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + ! BROADCAST IDS + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + SUBROUTINE broadcast_ids(serialized_ids) + + character(len=1), dimension(:), allocatable :: serialized_ids + integer :: ierror + integer :: serialized_ids_size = 0 + + ! Share information among all processors + if(allocated(serialized_ids)) serialized_ids_size = SIZE(serialized_ids) + + CALL mpi_bcast(serialized_ids_size, 1, MPI_INTEGER, MPI_ROOT_RANK, MPI_COMM_WORLD, ierror) + if(.not. allocated(serialized_ids) .and. serialized_ids_size > 0) allocate(serialized_ids(serialized_ids_size)) + CALL mpi_bcast(serialized_ids, serialized_ids_size, MPI_CHARACTER, MPI_ROOT_RANK, MPI_COMM_WORLD, ierror) + + END SUBROUTINE broadcast_ids + +{% endif %} + + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + ! INIT MUSCLE3 + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + SUBROUTINE init_muscle(instance) + + type(LIBMUSCLE_Instance), INTENT(OUT) :: instance + type(LIBMUSCLE_PortsDescription) :: ports + + logical, parameter :: CHECKPOINT_API_ENABLED = {% if flags.checkpointable %}.TRUE.{% else %}.FALSE.{% endif %} + + + ports = LIBMUSCLE_PortsDescription_create() + + ! Creation of INPUT ('F_INIT') ports + {% for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'IN' %} + ! {{ argument.name }} + CALL LIBMUSCLE_PortsDescription_add(ports, YMMSL_OPERATOR_F_INIT, '{{ argument.name }}') + {% endfor %} + + ! Creation of OUTPUT ('O_F') ports + {% for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'OUT' %} + ! {{ argument.name }} + CALL LIBMUSCLE_PortsDescription_add(ports, YMMSL_OPERATOR_O_F, '{{ argument.name }}') + {% endfor %} + + {% if flags.mpi_code %} + instance = LIBMUSCLE_Instance_create(ports, LIBMUSCLE_InstanceFlags(USES_CHECKPOINT_API=CHECKPOINT_API_ENABLED), & + MPI_COMM_WORLD, MPI_ROOT_RANK) + {% else %} + instance = LIBMUSCLE_Instance_create(ports, LIBMUSCLE_InstanceFlags( USES_CHECKPOINT_API=CHECKPOINT_API_ENABLED)) + {% endif %} + + call LIBMUSCLE_PortsDescription_free(ports) + + END SUBROUTINE init_muscle + + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + ! SEND IDS + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + SUBROUTINE send_ids(instance, serialized_ids, ids_name, time_stamp) + + type(LIBMUSCLE_Instance) :: instance + character(len=1), dimension(:), allocatable :: serialized_ids + character(len=*) :: ids_name + real (kind=LIBMUSCLE_int8) :: time_stamp + real (kind=LIBMUSCLE_int8) :: next_time_stamp + + type(LIBMUSCLE_Message) :: m3_message + type(LIBMUSCLE_Data) :: m3_data + + m3_data = LIBMUSCLE_Data_create_byte_array(serialized_ids) + m3_message = LIBMUSCLE_Message_create(time_stamp, m3_data) + + call LIBMUSCLE_Instance_send(instance, ids_name, m3_message) + call LIBMUSCLE_Message_free(m3_message) + + call LIBMUSCLE_Data_free(m3_data) + + END SUBROUTINE send_ids + + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + ! RECEIVE IDS + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + SUBROUTINE receive_ids(instance, ids_name, serialized_ids) + + type(LIBMUSCLE_Instance), INTENT(IN) :: instance + character(len=*), INTENT(IN) :: ids_name + character(len=1), dimension(:), allocatable, INTENT(OUT) :: serialized_ids + + type(LIBMUSCLE_Message) :: m3_message + type(LIBMUSCLE_DataConstRef) :: m3_data + + m3_message = LIBMUSCLE_Instance_receive(instance, ids_name) + m3_data = LIBMUSCLE_Message_get_data(m3_message) + + if ( .not. LIBMUSCLE_DataConstRef_is_nil(m3_data)) then + allocate(serialized_ids(LIBMUSCLE_DataConstRef_size(m3_data))) + call LIBMUSCLE_DataConstRef_as_byte_array(m3_data, serialized_ids) + call LIBMUSCLE_DataConstRef_free(m3_data) + end if + call LIBMUSCLE_Message_free(m3_message) + + + END SUBROUTINE receive_ids + + + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + ! RECEIVE INPUT IDSES + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + SUBROUTINE receive_input_idses(& + {% for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'IN' %} + {{ argument.name }}, & + {% endfor %} + instance) + + type(LIBMUSCLE_Instance), INTENT(IN) :: instance + character(len=1), dimension(:), allocatable :: serialized_ids + + {% if flags.mpi_code %} + ! workaround for fixed random seed and MPI (to be removed if IMAS-4524 solved) + integer, allocatable :: wa_seed(:) + integer :: wa_n + {% endif %} + + {% for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'IN' %} + {{ ids_macro.declare(argument.type, argument.name) }} + {% endfor %} + + {% for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'IN' %} + !--------- RECEIVE IDS : {{ argument.name }} ------------------------ + CALL receive_ids(instance, '{{ argument.name }}', serialized_ids) + + {% if flags.mpi_code %} + CALL broadcast_ids(serialized_ids) ! MUSCLE3 sends data to root process only + + ! Workaround: get current random state (to be removed if IMAS-4524 solved) + call random_seed(size = wa_n) + allocate(wa_seed(wa_n)) + call random_seed(get = wa_seed) + ! replace with a unique random seed (hopefully unique across different MPI processes) + call random_seed() + {% endif %} + + CALL ids_deserialize(serialized_ids, {{ argument.name }}) + deallocate(serialized_ids) + + {% if flags.mpi_code %} + ! restore random seed (to be removed if IMAS-4524 solved) + call random_seed(put = wa_seed) + deallocate(wa_seed) + {% endif %} + + {% endfor %} + + END SUBROUTINE receive_input_idses + + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + ! RETURN OUTPUT IDSES + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + SUBROUTINE send_output_idses(instance, & + {% for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'OUT' %} + {{ argument.name }}, & + {% endfor %} + in_timestamp) + + type(LIBMUSCLE_Instance), INTENT(IN) :: instance + real (kind=LIBMUSCLE_real8), INTENT(IN) :: in_timestamp + real (kind=LIBMUSCLE_real8) :: timestamp ! = -1.0_LIBMUSCLE_real8 + integer :: time_size + !---- Status info ---- + integer :: status_code = 0 + character(len=:), pointer :: status_message + + character(len=1), dimension(:), allocatable :: serialized_ids + {% for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'OUT' %} + {{ ids_macro.declare(argument.type, argument.name) }} + {% endfor %} + + {% for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'OUT' %} + !--------- SEND IDS : {{ argument.name }} ------------------------ + CALL ids_serialize({{ argument.name }}, serialized_ids) + if ( in_timestamp .GE. 0) then + timestamp = in_timestamp + else + timestamp = -1.0_LIBMUSCLE_real8 ! resetting a value + if ( ASSOCIATED({{ argument.name }}%time) .AND. SIZE({{ argument.name }}%time) > 0) then + time_size = SIZE({{ argument.name }}%time) + timestamp = {{ argument.name }}%time(time_size) + endif + endif + CALL send_ids(instance, serialized_ids, '{{ argument.name }}', timestamp ) + deallocate(serialized_ids) + + {% endfor %} + + END SUBROUTINE send_output_idses + + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + ! RESTORING CODE STATE + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +{% if flags.checkpointable %} + SUBROUTINE receive_wrapped_code_state(instance, state_str) + type(LIBMUSCLE_Instance), intent(IN) :: instance + character(len=:), allocatable, intent(OUT) :: state_str + type(LIBMUSCLE_Message) :: m3_message + type(LIBMUSCLE_DataConstRef) :: m3_data + integer :: string_size + !---- Status info ---- + integer :: status_code = 0 + character(len=:), pointer :: status_message + + + m3_message = LIBMUSCLE_Instance_load_snapshot(instance) + m3_data = LIBMUSCLE_Message_get_data(m3_message) + + if ( .not. LIBMUSCLE_DataConstRef_is_nil(m3_data)) then + state_str = LIBMUSCLE_DataConstRef_as_character(m3_data ) + call LIBMUSCLE_DataConstRef_free(m3_data) + end if + call LIBMUSCLE_Message_free(m3_message) + END SUBROUTINE receive_wrapped_code_state + + SUBROUTINE save_wrapped_code_state(instance, state_str) + + type(LIBMUSCLE_Instance), intent(IN) :: instance + character(len=:), allocatable, intent(IN) :: state_str + type(LIBMUSCLE_Message) :: m3_message + type(LIBMUSCLE_Data) :: m3_data + integer :: string_size + real (kind=LIBMUSCLE_real8) :: timestamp ! = -1.0_LIBMUSCLE_real8 + !---- Status info ---- + integer :: status_code = 0 + character(len=:), pointer :: status_message + + m3_data = LIBMUSCLE_Data_create(state_str) + m3_message = LIBMUSCLE_Message_create(timestamp, m3_data) + CALL LIBMUSCLE_Instance_save_final_snapshot(instance, m3_message) + + call LIBMUSCLE_Data_free(m3_data) + call LIBMUSCLE_Message_free(m3_message) + END SUBROUTINE save_wrapped_code_state +{% else %} + ! Subroutines not generated - no GET_STATE/SET_STATE methods provided +{% endif %} + + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + ! CHECK CONNECTION + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + SUBROUTINE check_connection(instance, port_name) + + type(LIBMUSCLE_Instance) :: instance + character(len=*) :: port_name + + if ( LIBMUSCLE_Instance_is_connected(instance, port_name)) return + + ! any error to report? + write(*,*) "ERROR: Port " // port_name //" not connected!" + call LIBMUSCLE_Instance_error_shutdown(instance, "ERROR: Port " // port_name //" not connected!") + call exit(1) + + END SUBROUTINE check_connection + + FUNCTION read_code_parameters(instance, imas_code_params) RESULT(status) + use ids_schemas, ONLY: ids_parameters_input + type(LIBMUSCLE_Instance), INTENT(IN) :: instance + character(len=:), allocatable :: default_file_name + character(len=:), allocatable :: exec_dir, file_path, err_msg + type(ids_parameters_input), intent(OUT) :: imas_code_params + integer :: status + logical :: copy_xml_parameters + copy_xml_parameters = .false. + + default_file_name = "{{ (code_description.implementation.code_parameters.parameters or "") | basename }}" + file_path = LIBMUSCLE_Instance_get_setting_as_character(instance, "parameters_file", status, err_msg) + + ! checking if property was read or not + if (status .ne. LIBMUSCLE_success) then + ! checking if an error occured while reading property + if (status .ne. LIBMUSCLE_out_of_range) then + write(*,*) " ERROR: Reading 'parameters_file' property failed! ", err_msg + return + end if + + ! reading default code parameters file + exec_dir = get_exec_dir() + exec_dir = trim(exec_dir) + file_path = exec_dir//"/../input/"//default_file_name + end if + + status = read_code_parameters_file(file_path, imas_code_params) + if (status /= 0) return + + copy_xml_parameters = LIBMUSCLE_Instance_get_setting_as_logical(instance, "copy_xml_parameters", status, err_msg) + + if (status .eq. LIBMUSCLE_success .and. & + copy_xml_parameters .eqv. .true. .and. & + associated(imas_code_params%parameters_value)) then + OPEN(UNIT=1, FILE=default_file_name, STATUS="REPLACE") + write(1, *) imas_code_params%parameters_value + CLOSE(UNIT=1) + else + !copy_xml_parameters setting is not mandatory so don't propagate error + status = 0 + return + end if + + END FUNCTION read_code_parameters + +END MODULE muscle3_tools \ No newline at end of file diff --git a/iwrap/generators/actor_generators/muscle3_fortran/resources/src/standalone.f90.jinja2 b/iwrap/generators/actor_generators/muscle3_fortran/resources/src/standalone.f90.jinja2 new file mode 100644 index 0000000..43250e8 --- /dev/null +++ b/iwrap/generators/actor_generators/muscle3_fortran/resources/src/standalone.f90.jinja2 @@ -0,0 +1,243 @@ +{% import './macros/%s_ids.jinja2' % code_description.implementation.data_type as ids_macro %} + +{%- import './macros/flags.jinja2' as flags with context-%} + +{%- if flags.skeleton_only %} + {% set comment_out = '!' -%} +{% endif %} + +! = = = = = = = ACTOR: {{actor_description.actor_name}} = = = = = = +program standalone + +{% if flags.mpi_code %} + use mpi +{% endif %} + use iwrap_tools + use muscle3_tools + {% if code_description.implementation.include_path %} + use {{code_description.implementation.include_path | stemname}} + {% else %} + ! Put "use " here + {% endif %} + implicit none + + !--------------------------------------------------------- + character(len=*), parameter :: ACTOR_NAME = "{{actor_description.actor_name}}" + type(LIBMUSCLE_Instance) :: instance + integer :: process_rank = MPI_ROOT_RANK + + !---- code state ---- + character(len=:), allocatable :: state_str + + !---- timestamp ---- + real (kind=LIBMUSCLE_real8) :: timestamp = -1.0_LIBMUSCLE_real8 + + !---- Status info ---- + integer :: status_code = 0 + character(len=:), pointer :: status_message + + !---- Code parameters ---- + character(len=:), allocatable :: xml_string + type(ids_parameters_input) :: imas_code_params + + !---- IN/OUT IDSes ---- +{% for argument in code_description.implementation.subroutines.main.arguments %} + {{ ids_macro.declare(argument.type, argument.name) }} +{% endfor %} + +{% if code_description.implementation.subroutines.init.name %} + logical :: initial_run + initial_run = .TRUE. +{% endif %} + + + ! > > > - - - INITIALISATION - - - < < < +{% if flags.mpi_code %} + CALL init_mpi(process_rank) +{% endif %} + + CALL init_muscle(instance) + +{% if flags.code_parameters_provided %} + ! ------------------ code parameters ---------------------------- + status_code = read_code_parameters(instance, imas_code_params) + if (status_code /= 0) CALL exit (status_code) + +{% endif %} + + ! > > > MUSCLE3 LOOP. < < < + do while (LIBMUSCLE_Instance_reuse_instance(instance)) + + ! - - - WRAPPED CODE INITIALIZATION - - - # # # # +{% if flags.checkpointable %} + if ( LIBMUSCLE_Instance_resuming(instance)) then + ! > > > RESTART : the code is restored from the saved snapshot < < < + if (process_rank == MPI_ROOT_RANK) then + CALL receive_wrapped_code_state(instance, state_str) + ! Calling SET_STATE + {% if flags.skeleton_only %} + ! + ! >>> Subroutine "SET_STATE" should be called here <<< + ! + {% endif %} + {{ comment_out }} CALL {{code_description.implementation.subroutines.set_state}}(state_str, status_code, status_message) + + endif +{% endif %} + +{% if code_description.implementation.subroutines.init.name %} + {% if flags.checkpointable %} + else ! m3_instance is not resuming + ! > > > INITIALIZATION by calling INIT method of the code < < < + {% endif %} + + if (initial_run) then + !---- RESET status info ---- + nullify(status_message) + status_code = 0 + ! Calls the wrapped INIT subroutine + {% if flags.skeleton_only %} + ! + ! >>> Subroutine "INIT" should be called here <<< + ! + {% endif %} + {{ comment_out }} CALL {{code_description.implementation.subroutines.init.name}}( + {%- if flags.code_parameters_provided -%} + imas_code_params, + {%- endif -%} + status_code, status_message) + CALL check_status(instance, status_code, status_message, ACTOR_NAME, "INIT") + initial_run = .FALSE. + endif + {% endif %} + + {% if flags.checkpointable %} + endif ! LIBMUSCLE_Instance_resuming + + if( LIBMUSCLE_Instance_should_init(instance)) then +{% endif %} + + ! > > > RECEIVING INPUT IDSes < < < + CALL receive_input_idses(& + {% for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'IN' %} + {{ argument.name }}, & + {% endfor %} + instance ) + +{% if flags.checkpointable %} + endif ! LIBMUSCLE_Instance_should_init +{% endif %} + + !---- RESET status info ---- + nullify(status_message) + status_code = 0 + + ! > > > - - - WRAPPED CODE CALL - MAIN SBRT - - - < < < + {% if flags.skeleton_only %} + ! + ! >>> Subroutine "MAIN" should be called here <<< + ! + {% endif %} + {{ comment_out }} CALL {{code_description.implementation.subroutines.main.name}}( & + {% for argument in code_description.implementation.subroutines.main.arguments %} + {{ comment_out }} {{ argument.name }}, & + {% endfor %} + {% if flags.code_parameters_provided %} + {{ comment_out }} imas_code_params, & + {% endif %} + {{ comment_out }} status_code, status_message) + + + !-----------Check status info --------------------- + CALL check_status(instance, status_code, status_message, ACTOR_NAME, "MAIN") + +{% if flags.mpi_code %} + if (process_rank == MPI_ROOT_RANK) then +{% endif %} + + ! > > > RETURNING OUTPUT IDSes < < < + {% if code_description.implementation.subroutines.get_timestamp %} + ! > > > GET TIMESTAMP < < < + nullify(status_message) + status_code = 0 + {% if flags.skeleton_only %} + ! + ! >>> Subroutine "GET_TIMESTAMP" should be called here <<< + ! + {% endif %} + {{ comment_out }} CALL {{code_description.implementation.subroutines.get_timestamp}}(timestamp, status_code, status_message) + + CALL check_status(instance, status_code, status_message, "{{actor_description.actor_name}}", "GET_TIMESTAMP") + + {% endif %} + CALL send_output_idses(instance, & + {% for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'OUT' %} + {{ argument.name }}, & + {% endfor %} + timestamp) + +{% if flags.mpi_code %} + endif ! --- The end of section called only for RANK 0 process +{% endif %} + + +{% if flags.checkpointable %} + ! > > > SAVE SNAPSHOT < < < + if (LIBMUSCLE_Instance_should_save_final_snapshot(instance)) then + if (process_rank == MPI_ROOT_RANK ) then ! Always .TRUE. for non-MPI code + ! Calling GET_STATE + nullify(status_message) + status_code = 0 + {% if flags.skeleton_only %} + ! + ! >>> Subroutine "GET_STATE" should be called here <<< + ! + {% endif %} + + {{ comment_out }} CALL {{code_description.implementation.subroutines.get_state}}(state_str, status_code, status_message) + + CALL check_status(instance, status_code, status_message, ACTOR_NAME, "GET_STATE") + CALL save_wrapped_code_state(instance, state_str) + endif + endif + +{% endif %} + + ! > > > CLEAN UP < < < + {% for argument in code_description.implementation.subroutines.main.arguments %} + {{ ids_macro.deallocate( argument.name) }} + {% endfor %} + + end do ! < < < MUSCLE3 LOOP < < < + + {% if code_description.implementation.subroutines.finalize.name %} + ! > > > - - - - - - - - - - - - - WRAPPED CODE CALL - FINALISE SBRT - - - - - - - - - - - - - - - - - - < < < + !---- RESET status info ---- + nullify(status_message) + status_code = 0 + {% if flags.skeleton_only %} + ! + ! >>> Subroutine "FINALIZE" should be called here <<< + ! + {% endif %} + {{ comment_out }} CALL {{code_description.implementation.subroutines.finalize.name}}(status_code, status_message) + + !-----------Check status info --------------------- + CALL check_status(instance, status_code, status_message, ACTOR_NAME, "FINALIZE") + +{% endif %} + + {% if flags.code_parameters_provided %} + ! ------Deallocating code parameters ---------------------------- + if (associated(imas_code_params%parameters_value)) deallocate(imas_code_params%parameters_value) + {% endif %} + + !---- MUSCLE3 Finalization ---- + call LIBMUSCLE_Instance_free(instance) + + {% if flags.mpi_code %} + !---- MPI Finalization ---- + call finish_mpi() + + {% endif %} +end program diff --git a/iwrap/generators/actor_generators/muscle3_python/__init__.py b/iwrap/generators/actor_generators/muscle3_python/__init__.py new file mode 100644 index 0000000..802dd39 --- /dev/null +++ b/iwrap/generators/actor_generators/muscle3_python/__init__.py @@ -0,0 +1,3 @@ +from iwrap.generators.actor_generators.muscle3_python.m3_python_actor import PythonActorGenerator + +__all__ = ['PythonActorGenerator'] \ No newline at end of file diff --git a/iwrap/generators/actor_generators/muscle3_python/m3_python_actor.py b/iwrap/generators/actor_generators/muscle3_python/m3_python_actor.py new file mode 100644 index 0000000..d90daec --- /dev/null +++ b/iwrap/generators/actor_generators/muscle3_python/m3_python_actor.py @@ -0,0 +1,103 @@ +import logging +import os +import shutil +import tempfile +from pathlib import Path +from typing import Set, List + + +from iwrap.generation_engine.utils.jinja2_template_processing import process_template_dir +from iwrap.generators.actor_generators import ActorGenerator +from iwrap.settings.project import ProjectSettings + +import jinja2 +import sys + +from iwrap.settings.platform.platform_settings import PlatformSettings + +from iwrap.generators.actor_generators.muscle3_common import m3_utils + + +class PythonActorGenerator(ActorGenerator): + # Class logger + __logger = logging.getLogger(__name__ + "." + __qualname__) + + COMPLIANT_API = '2.1' + + @property + def type(self) -> str: + return 'MUSCLE3-Python' + + @property + def name(self) -> str: + return 'MUSCLE3 (Python)' + + @property + def description(self) -> str: + return 'Wrapping Python code into MUSCLE3 micro model' + + @property + def actor_language(self) -> List[str]: + return 'python' + + @property + def actor_data_types(self) -> List[str]: + return ['legacy'] + + @property + def code_data_types(self) -> List[str]: + return ['legacy'] + + @property + def code_languages(self) -> Set[str]: + return {'python'} + + def __init__(self): + + self.__info_output_stream = None + self.temp_dir: tempfile.TemporaryDirectory = None + self.jinja_env: jinja2.Environment = None + self.install_dir: str = None + self.wrapper_dir = 'wrapper' + + def configure(self, info_output_stream=sys.stdout): + self.__info_output_stream = info_output_stream + + def validate(self, project_settings: dict = None): + m3_utils.validate( project_settings ) + + def initialize(self, project_settings: dict = None): + install_dir = None + + if project_settings: + install_dir = project_settings['actor_description'].get( 'install_dir' ) + + if not install_dir: + install_dir = PlatformSettings().directories.actor_install_dir + self.install_dir: str = str(Path(install_dir, ProjectSettings.get_settings().actor_description.actor_name)) + + def generate(self, project_settings: dict): + self.cleanup(project_settings) + os.makedirs( self.install_dir, exist_ok=True ) + current_path = os.path.dirname(os.path.realpath(__file__)) + process_template_dir( None, current_path + '/resources', self.install_dir, project_settings, + filter_func=m3_utils.template_filter_func, + output_stream= self.__info_output_stream, ) + + m3_utils.copy_native_lib( self.install_dir, project_settings, "wrapped_code" ) + m3_utils.copy_code_params_files( self.install_dir, project_settings ) + m3_utils.copy_build_info( self.install_dir, project_settings ) + + + def build(self, project_settings: dict): + ... + + def install(self, project_settings: dict): + ... + + def cleanup(self, project_settings: dict): + shutil.rmtree(self.install_dir, ignore_errors=True) + + + + diff --git a/iwrap/generators/actor_generators/muscle3_python/resources/__init__.py.jinja2 b/iwrap/generators/actor_generators/muscle3_python/resources/__init__.py.jinja2 new file mode 100644 index 0000000..e69de29 diff --git a/iwrap/generators/actor_generators/muscle3_python/resources/macros/flags.jinja2 b/iwrap/generators/actor_generators/muscle3_python/resources/macros/flags.jinja2 new file mode 100644 index 0000000..7c727ce --- /dev/null +++ b/iwrap/generators/actor_generators/muscle3_python/resources/macros/flags.jinja2 @@ -0,0 +1,5 @@ +{%- set checkpointable = code_description.implementation.subroutines.get_state and code_description.implementation.subroutines.set_state %} +{%- set code_parameters_provided = code_description.implementation.code_parameters.parameters %} +{%- set mpi_code = code_description.settings.mpi_compiler_cmd -%} + +{%- set skeleton_only = not code_description.implementation.programming_language -%} \ No newline at end of file diff --git a/iwrap/generators/actor_generators/muscle3_python/resources/macros/legacy_ids.jinja2 b/iwrap/generators/actor_generators/muscle3_python/resources/macros/legacy_ids.jinja2 new file mode 100644 index 0000000..9a780e7 --- /dev/null +++ b/iwrap/generators/actor_generators/muscle3_python/resources/macros/legacy_ids.jinja2 @@ -0,0 +1,13 @@ +{% macro imports() -%} + import imas +{%- endmacro -%} + +{% macro declare(ids_name, ids_var_name) -%} + _factory = imas.IDSFactory() + {{ ids_var_name }} = _factory.{{ ids_name }}() +{%- endmacro -%} + +{%- macro provenance(ids_var_name, sbrt_name) -%} + {{ ids_var_name }}.code.name = "{{sbrt_name}}" + {{ ids_var_name }}.code.version = "" +{%- endmacro -%} diff --git a/iwrap/generators/actor_generators/muscle3_python/resources/muscle3_tools.py.jinja2 b/iwrap/generators/actor_generators/muscle3_python/resources/muscle3_tools.py.jinja2 new file mode 100644 index 0000000..3a4d210 --- /dev/null +++ b/iwrap/generators/actor_generators/muscle3_python/resources/muscle3_tools.py.jinja2 @@ -0,0 +1,157 @@ +{%- import './macros/flags.jinja2' as flags with context -%} + +import os +import ymmsl +from shutil import copy +import libmuscle +from libmuscle import Instance, USES_CHECKPOINT_API + +import imas + + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# INIT MUSCLE3 +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +def init_muscle3(): + + input_ports = [] + output_ports = [] + + # # # # - - - INPUT ('F_INIT') ports - - - # # # # +{% for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'IN' %} + # {{ argument.name }} + input_ports.append("{{ argument.name }}") + +{% endfor %} + # # # # - - - OUTPUT ('O_F') ports - - - # # # # +{% for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'OUT' %} + # {{ argument.name }} + output_ports.append("{{ argument.name }}") + + {% endfor %} + ports = {ymmsl.Operator.F_INIT : input_ports, ymmsl.Operator.O_F : output_ports} +{% if flags.checkpointable %} + m3_instance = libmuscle.Instance(ports, libmuscle.USES_CHECKPOINT_API) +{% else %} + m3_instance = libmuscle.Instance(ports) +{% endif %} + + return m3_instance + + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# FINISH MUSCLE3 +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +def finish_muscle3(m3_instance): + ... + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# RECEIVE IDS +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +def receive_ids(m3_instance: libmuscle.Instance, port_name, ids): + m3_message = m3_instance.receive(port_name) + + ids_bytes = m3_message.data + ids.deserialize(ids_bytes) + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# SEND IDS +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +def send_ids(m3_instance: libmuscle.Instance, port_name, ids, timestamp): + + ids_bytes = ids.serialize(imas.ids_defs.DEFAULT_SERIALIZER_PROTOCOL) + m3_message = libmuscle.Message( timestamp, None, ids_bytes) + + m3_instance.send(port_name, m3_message) + + + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# RECEIVE INPUT IDSES +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +def receive_input_idses(m3_instance: libmuscle.Instance, +{% for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'IN' %} + {{ argument.name }}, +{%- endfor -%} + ): + +{% for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'IN' %} + # {{ argument.name }} + receive_ids(m3_instance, "{{ argument.name }}", {{ argument.name }}) + +{% endfor %} + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# SEND OUTPUT IDSES +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +def send_output_idses(m3_instance: libmuscle.Instance, timestamp, +{% for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'OUT' %} + {{ argument.name }}, +{% endfor -%} + ): + + {% for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'OUT' %} + #--------- SEND IDS : {{ argument.name }} ------------------------ + {% if not code_description.implementation.subroutines.get_timestamp %} + timestamp = -1.0 # resetting value + time_size = len({{ argument.name }}.time) + if time_size > 0: + time_stamp = {{ argument.name }}.time[time_size - 1] + + {% endif %} + send_ids(m3_instance, "{{ argument.name }}", {{ argument.name }}, timestamp) + +{% endfor %} + +{% if code_description.implementation.code_parameters.parameters %} +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# XXX XXX +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +def read_code_parameters(m3_instance: libmuscle.Instance): + + try: + # checking if property 'parameters_file' can be read + code_parameters_file_path = m3_instance.get_setting("parameters_file", "str") + except KeyError: + + # Value for setting parameters_file was not set. Using default file. + directory = os.path.realpath(os.path.dirname(__file__)) + code_parameters_file_path = directory + "/input/{{code_description.implementation.code_parameters.parameters | basename }}" + + with open(code_parameters_file_path, 'r') as file: + code_parameters = file.read() + + try: + # checking if property 'copy_xml_parameters' can be read + copy_xml_parameters = m3_instance.get_setting("copy_xml_parameters", "bool") + if copy_xml_parameters: + #shutil.copy() + copy( code_parameters_file_path, "./" ) + except KeyError: + ... + #just don't copy parameters + + return code_parameters + +{% endif %} +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# RESTORING CODE STATE +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +{% if flags.checkpointable %} +def receive_code_state(m3_instance: libmuscle.Instance): + m3_message = m3_instance.load_snapshot() + if m3_message is None: + return None + + return m3_message.data + +def save_code_state(m3_instance: libmuscle.Instance, state_str): + + timestamp = -1.0 + m3_message = libmuscle.Message( timestamp, None, state_str) + m3_instance.save_final_snapshot(m3_message) + +{% else %} +# Methods not generated - no GET_STATE/SET_STATE methods provided +{% endif %} \ No newline at end of file diff --git a/iwrap/generators/actor_generators/muscle3_python/resources/standalone.py.jinja2 b/iwrap/generators/actor_generators/muscle3_python/resources/standalone.py.jinja2 new file mode 100644 index 0000000..20f1371 --- /dev/null +++ b/iwrap/generators/actor_generators/muscle3_python/resources/standalone.py.jinja2 @@ -0,0 +1,162 @@ +{% import './macros/%s_ids.jinja2' % code_description.implementation.data_type as ids_macro %} + +{%- import './macros/flags.jinja2' as flags with context -%} + +{% set ARG_INDENT = "\n"~20*" " %} + +{%- if flags.checkpointable %} + {% set chkp_indent = 4 %} +{% else %} + {% set chkp_indent = 0 %} +{% endif %} + +{%- if code_description.implementation.code_path %} + {% set python_module = code_description.implementation.code_path | stemname %} +{% else %} + {% set python_module = "" %} +{% endif %} + +{%- if flags.skeleton_only %} + {% set comment_out = '# ' -%} +{% endif %} + +{{ ids_macro.imports() }} + +from {{actor_description.actor_name}} import muscle3_tools +{{ comment_out }}from {{actor_description.actor_name}}.wrapped_code import {{ python_module }} + +if __name__ == "__main__": + + timestamp = -1.0 +{% if code_description.implementation.subroutines.init.name %} + initial_run = True +{% endif %} + + #---- IN/OUT IDSes ---- +{% for argument in code_description.implementation.subroutines.main.arguments %} + {{ ids_macro.declare(argument.type, argument.name) }} +{% endfor %} + + # # # # - - - M3 INITIALISATION - - - # # # # + m3_instance = muscle3_tools.init_muscle3() + + {% if flags.code_parameters_provided %} + #---- Code parameters ---- + imas_code_params = muscle3_tools.read_code_parameters(m3_instance) + + {% endif %} + + # # # # - - - MUSCLE3 LOOP - - - # # # # + while m3_instance.reuse_instance(): + + # # # # - - - WRAPPED CODE INITIALIZATION - - - # # # # +{% if flags.checkpointable %} + if m3_instance.resuming(): + # > > > RESTART : the code is restored from the saved snapshot < < < + state_str = muscle3_tools.receive_code_state(m3_instance) + # Calling SET_STATE + {% if flags.skeleton_only %} + # + # >>> Subroutine "SET_STATE" should be called here <<< + # + {% endif %} + {{ comment_out }}{{ python_module }}.{{code_description.implementation.subroutines.set_state}}(state_str) +{% endif %} + {% if code_description.implementation.subroutines.init.name %} + {% if flags.checkpointable %} + else: # m3_instance is not resuming + # > > > INITIALIZATION by calling INIT method of the code < < < + {% endif %} + {% filter indent(width=chkp_indent, first=True) %} + if initial_run: + {% if flags.skeleton_only %} + ... + # + # >>> Subroutine "INIT" should be called here <<< + # + {% endif %} + {{ comment_out }}{{ python_module }}.{{code_description.implementation.subroutines.init.name}}( + {%- if flags.code_parameters_provided -%} + imas_code_params + {%- endif -%} + ) + + {% endfilter %} + initial_run = False + {% endif %} + +{% if flags.checkpointable %} + if m3_instance.should_init(): +{% endif %} + {% filter indent(width=chkp_indent) %} + # # # # - - - RECEIVING INPUT IDSes - - - # # # # + muscle3_tools.receive_input_idses( + {{- ARG_INDENT }} m3_instance, + {%- for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'IN' -%} + {{ ARG_INDENT }} {{ argument.name }}, + {%- endfor -%} ) + + {% endfilter %} + + # # # # - - - WRAPPED CODE CALL - MAIN SBRT - - - # # # # + {% if flags.skeleton_only %} + # + # >>> Subroutine "MAIN" should be called here <<< + # + {% endif %} + {{ comment_out }}{{ python_module }}.{{code_description.implementation.subroutines.main.name}}( + {%- for argument in code_description.implementation.subroutines.main.arguments -%} + {{ ARG_INDENT }}{{ comment_out }} {{ argument.name }}, + {%- endfor -%} + {% if flags.code_parameters_provided -%} + {{ ARG_INDENT }}{{ comment_out }} imas_code_params + {% endif -%} ) + + + # # # # - - - SENDING OUTPUT IDSes - - - # # # # + {% if code_description.implementation.subroutines.get_timestamp %} + # > > > GET TIMESTAMP < < < + {% if flags.skeleton_only %} + # + # >>> Subroutine "GET_TIMESTAMP" should be called here <<< + # + {% endif %} + {{ comment_out }}timestamp = {{ python_module }}.{{code_description.implementation.subroutines.get_timestamp}}() + + + {% endif %} + muscle3_tools.send_output_idses( + {{- ARG_INDENT }} m3_instance, timestamp, + {%- for argument in code_description.implementation.subroutines.main.arguments if argument.intent == 'OUT' -%} + {{ ARG_INDENT }} {{ argument.name }}, + {%- endfor -%} ) + +{% if flags.checkpointable %} + # > > > SAVE SNAPSHOT < < < + if m3_instance.should_save_final_snapshot(): + # Calling GET_STATE + {% if flags.skeleton_only %} + # + # >>> Subroutine "GET_STATE" should be called here <<< + # + {% endif %} + {{ comment_out }}state_str = {{ python_module }}.{{code_description.implementation.subroutines.get_state}}() + + muscle3_tools.save_code_state(m3_instance, state_str) +{% endif %} + + #> > > > THE END OF MUSCLE3 LOOP + + # # # # - - - FINALIZATION - - - # # # # + {% if code_description.implementation.subroutines.finalize.name %} + + # # # # - - - WRAPPED CODE CALL - FINISH SBRT - - - # # # # + {% if flags.skeleton_only %} + # + # >>> Subroutine "FINALIZATION" should be called here <<< + # + {% endif %} + {{ comment_out }}{{ python_module }}.{{code_description.implementation.subroutines.finalize.name}} () + {% endif %} + + muscle3_tools.finish_muscle3(m3_instance) diff --git a/iwrap/generators/actor_generators/python_actor/resources/common/code_parameters.py b/iwrap/generators/actor_generators/python_actor/resources/common/code_parameters.py index c61332c..4898e8c 100644 --- a/iwrap/generators/actor_generators/python_actor/resources/common/code_parameters.py +++ b/iwrap/generators/actor_generators/python_actor/resources/common/code_parameters.py @@ -1,5 +1,5 @@ import logging -from .code_parameters_handlers.handler_factory import HandlerFactory +from iwrap.settings.code_parameters_handlers.handler_factory import HandlerFactory class CodeParameters: # Class logger @@ -36,6 +36,32 @@ def set_parameter(self, path_to_node:str, value) -> None: self.__handler.set_parameter(path_to_node, value) def __init__(self, default_parameters_path:str, schema_path:str, parameters_format="xml"): + from pathlib import Path as PathLib + + # Resolve paths relative to actor's input directory + # Actor structure: actor_name/common/code_parameters.py and actor_name/input/ + # Use resolve() to get the absolute real path, handling symlinks + actor_dir = PathLib(__file__).resolve().parent.parent # Go up from common/ to actor root + input_dir = actor_dir / 'input' + + self.__logger.debug(f"Actor directory: {actor_dir}") + self.__logger.debug(f"Input directory: {input_dir}") + self.__logger.debug(f"Input params path: {default_parameters_path}") + self.__logger.debug(f"Input schema path: {schema_path}") + + # Convert relative paths (filenames) to absolute paths in actor's input directory + if default_parameters_path: + params_path = PathLib(default_parameters_path) + if not params_path.is_absolute(): + default_parameters_path = str(input_dir / params_path) + self.__logger.debug(f"Resolved params path: {default_parameters_path}") + + if schema_path: + schema_path_obj = PathLib(schema_path) + if not schema_path_obj.is_absolute(): + schema_path = str(input_dir / schema_path_obj) + self.__logger.debug(f"Resolved schema path: {schema_path}") + self._default_parameters_path = default_parameters_path self._schema_path = schema_path diff --git a/iwrap/generators/actor_generators/python_actor/resources/common/code_parameters_handlers b/iwrap/generators/actor_generators/python_actor/resources/common/code_parameters_handlers deleted file mode 120000 index 183ba09..0000000 --- a/iwrap/generators/actor_generators/python_actor/resources/common/code_parameters_handlers +++ /dev/null @@ -1 +0,0 @@ -../../../../../settings/code_parameters_handlers/ \ No newline at end of file diff --git a/iwrap/gui/settings/code_parameters_pane.py b/iwrap/gui/settings/code_parameters_pane.py index ebda2ff..a4bad2e 100644 --- a/iwrap/gui/settings/code_parameters_pane.py +++ b/iwrap/gui/settings/code_parameters_pane.py @@ -8,8 +8,8 @@ from iwrap.settings.project import ProjectSettings from iwrap.gui.settings.tooltip import ToolTip -from iwrap.generators.actor_generators.python_actor.resources.common.code_parameters_handlers.handler_factory import HandlerFactory -from iwrap.generators.actor_generators.python_actor.resources.common.code_parameters_handlers.parameters_handler_interface import ParametersHandlerInterface +from iwrap.settings.code_parameters_handlers.handler_factory import HandlerFactory +from iwrap.settings.code_parameters_handlers.parameters_handler_interface import ParametersHandlerInterface class CodeParametersPane(ttk.Frame, IWrapPane): # Class logger diff --git a/iwrap/iwrap_main.py b/iwrap/iwrap_main.py index af3d02c..33b08ed 100644 --- a/iwrap/iwrap_main.py +++ b/iwrap/iwrap_main.py @@ -17,7 +17,7 @@ def get_parser(is_commandline_mode: bool) -> argparse.ArgumentParser: # Management of input arguments parser = argparse.ArgumentParser( description='iWrap - a modular IMAS actor generator, used for creating ' 'standardized actors from a code interfaced with IDSs.', - epilog='For more information, visit .', + epilog='For more information, visit .', # formatter_class=argparse.ArgumentDefaultsHelpFormatter ) diff --git a/iwrap/settings/code_parameters_handlers/generic_handler.py b/iwrap/settings/code_parameters_handlers/generic_handler.py index b267182..f273f57 100644 --- a/iwrap/settings/code_parameters_handlers/generic_handler.py +++ b/iwrap/settings/code_parameters_handlers/generic_handler.py @@ -10,7 +10,6 @@ class GenericHandler(ParametersHandlerInterface, ABC): __logger = logging.getLogger(__name__ + "." + __qualname__) def __init__(self): - self._default_params_dir = Path(Path(__file__).parent, '../../input/') self._new_path_set: bool = True self._parameters_path: str = None self._default_parameters_path: str = None @@ -105,16 +104,25 @@ def initialize(self, parameters_path: str, schema_path: str): self._parameters_path = parameters_path self._schema_path = schema_path - # Read XSD (if not yet loaded) - if not self._schema_str: - schema_path = Path( self._default_params_dir, Path(self._schema_path)) - self._schema_str = self._read_file( schema_path ) - - if self._new_path_set: - if self._parameters_path: - parameters_path = Path( self._default_params_dir, Path(self._parameters_path)) - self._parameters_str = self._read_file(parameters_path) - self._new_path_set = False + # Read schema (if not yet loaded) + if not self._schema_str and schema_path: + schema_file = Path(schema_path).resolve() + self.__logger.debug(f"Reading schema from: {schema_file}") + + if not schema_file.exists(): + raise FileNotFoundError(f"Schema file not found: {schema_file}") + + self._schema_str = self._read_file(schema_file) + + if self._new_path_set and self._parameters_path: + params_file = Path(self._parameters_path).resolve() + self.__logger.debug(f"Reading parameters from: {params_file}") + + if not params_file.exists(): + raise FileNotFoundError(f"Parameters file not found: {params_file}") + + self._parameters_str = self._read_file(params_file) + self._new_path_set = False def restore_default_parameters_path(self): self._parameters_path = self._default_parameters_path diff --git a/iwrap/settings/compatibility/converter.py b/iwrap/settings/compatibility/converter.py index 63807d8..353678a 100644 --- a/iwrap/settings/compatibility/converter.py +++ b/iwrap/settings/compatibility/converter.py @@ -63,6 +63,10 @@ def __delete(self, splitted_path, condition, current_node=None): return False return True else: + # Check if the key exists before trying to access it + if splitted_path[0] not in current_node: + return False + if isinstance(current_node[splitted_path[0]], list): condition = condition.replace("$TARGET", "current_node[splitted_path[0]][x]") del_indexes = [] diff --git a/pyproject.toml b/pyproject.toml index 0444ef6..69b09f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,8 @@ [build-system] # Minimum requirements for the build system to execute. -requires = ["setuptools>=43", "wheel", "tomli", "versioneer[toml]"] +requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.2"] build-backend = "setuptools.build_meta" -[tool.versioneer] -VCS = "git" -style = "pep440" -versionfile_source = "iwrap/_version.py" -versionfile_build = "iwrap/_version.py" -tag_prefix = "" -parentdir_prefix = "" +[tool.setuptools_scm] +write_to = "iwrap/_version.py" diff --git a/requirements_muscle3.txt b/requirements_muscle3.txt new file mode 100644 index 0000000..a1f732f --- /dev/null +++ b/requirements_muscle3.txt @@ -0,0 +1,3 @@ +# Optional dependencies for MUSCLE3 actor generators +# Install with: pip install -r requirements_muscle3.txt +muscle3>=0.7.0 diff --git a/setup.cfg b/setup.cfg index 2cf0468..771cb0d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,8 +5,8 @@ author_email = olivier.hoenen@iter.org license = ITER license description = A modular component generator, implemented in Python, used for creating IMAS actors from physics models. long_description_content_type=text/x-rst -long_description = file: Readme.md -url = https://git.iter.org/projects/IMEX/repos/iwrap +long_description = file: README.md +url = https://github.com/iterorganization/iWrap.git [flake8] ignore = @@ -19,13 +19,4 @@ package_dir = iwrap max-line-length = 88 [aliases] -test = pytest - - -[versioneer] -VCS = git -style = pep440 -versionfile_source = iwrap/_version.py -versionfile_build = iwrap/_version.py -tag_prefix = -parentdir_prefix = \ No newline at end of file +test = pytest \ No newline at end of file diff --git a/setup.py b/setup.py index 5cd43b4..f8bc351 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,6 @@ import sys import subprocess from glob import glob -import versioneer # Import other stdlib packages from itertools import chain @@ -73,7 +72,7 @@ def list_docs_data_files(path_to_docs: str): pyproject_data = tomli.loads(pyproject_text) optional_reqs = {} -for req in ["core"]: +for req in ["core", "muscle3"]: optional_reqs[req] = DistTextFile(this_dir / f"requirements_{req}.txt").readlines() install_requires = optional_reqs.pop("core") # collect all optional dependencies in a "all" target @@ -89,17 +88,15 @@ def list_docs_data_files(path_to_docs: str): # For allowed version strings, see: # https://packaging.python.org/specifications/core-metadata/ for allow version strings - cmdclassdict = versioneer.get_cmdclass() - cmdclassdict['clean'] = CleanCommand - setup( - version=versioneer.get_version(), + use_scm_version={ + 'write_to': 'iwrap/_version.py', + }, packages=find_packages(exclude=('tests*', 'testing*', 'test_suite*')), setup_requires=pyproject_data["build-system"]["requires"], include_package_data=True, install_requires=install_requires, extras_require=optional_reqs, - cmdclass=cmdclassdict, entry_points={ 'console_scripts': [ 'iwrap-gui = bin.run:gui', diff --git a/test_muscle3_integration.py b/test_muscle3_integration.py new file mode 100644 index 0000000..2a9fa20 --- /dev/null +++ b/test_muscle3_integration.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +""" +Test script to verify MUSCLE3 integration into iWrap core +""" + +import sys +sys.path.insert(0, '/home/ITER/sawantp1/github/iWrap') + +from iwrap.generation_engine.engine import Engine + +print('🔧 Testing iWrap with MUSCLE3 Built-in Generators') +print('=' * 60) + +# Initialize engine +Engine().startup() +generators = Engine().registered_generators + +print(f'\n✓ Engine initialized successfully!') +print(f'✓ Found {len(generators)} actor generators:\n') + +for gen in generators: + print(f' Type: {gen.type:<20} | Name: {gen.name:<30}') + print(f' API Version: {gen.COMPLIANT_API}') + if hasattr(gen, 'code_languages'): + print(f' Code Languages: {gen.code_languages}') + print() + +print('=' * 60) +print('✅ MUSCLE3 Integration Successful!') +print('\nNext steps:') +print(' 1. Test actor generation') +print(' 2. Integrate tests') +print(' 3. Update documentation') +print(' 4. Configure CI/CD') diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..e123824 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,422 @@ +# iWrap Test Suite + +This directory contains the comprehensive test suite for iWrap, including unit tests, integration tests, and IMAS integration tests. + +## Test Structure + +``` +tests/ +├── conftest.py # Pytest configuration and shared fixtures +├── pytest.ini # Pytest markers and settings +│ +├── run-all-tests-master.sh # Master orchestrator for all tests +├── run-all-tests.sh # Comprehensive pytest-based tests +├── run-muscle3-tests.sh # MUSCLE3-specific tests only +├── run-tests.sh # Legacy IMAS integration tests +│ +├── unit/ # Fast, isolated component tests +│ ├── core/ # Core functionality tests +│ └── generators/ # Generator-specific tests +│ ├── test_muscle3_python.py +│ ├── test_muscle3_cpp.py +│ └── test_muscle3_fortran.py +│ +├── integration/ # Cross-component workflow tests +│ └── test_muscle3_integration.py +│ +├── muscle3/ # MUSCLE3-specific test cases +│ ├── actors/ +│ ├── wrapped_codes/ +│ ├── macro/ +│ ├── workflow/ +│ └── code_parameters/ +│ +└── test_cases/ # IMAS integration test cases + └── ... +``` + +## Running Tests + +### Quick Start + +Run all pytest-based tests (unit + integration): +```bash +./tests/run-all-tests-master.sh +``` + +### Test Runners + +#### Master Orchestrator (Recommended) +The master orchestrator provides fine-grained control over which test suites to run: + +```bash +# Run all tests including IMAS (slow) +./tests/run-all-tests-master.sh --all + +# Run only unit tests (fast) +./tests/run-all-tests-master.sh --unit-only + +# Run only integration tests +./tests/run-all-tests-master.sh --integration-only + +# Run only MUSCLE3 tests +./tests/run-all-tests-master.sh --muscle3-only + +# Include IMAS tests +./tests/run-all-tests-master.sh --imas + +# Skip MUSCLE3 tests +./tests/run-all-tests-master.sh --skip-muscle3 + +# Verbose output +./tests/run-all-tests-master.sh -v + +# Stop on first failure +./tests/run-all-tests-master.sh --fail-fast +``` + +#### Comprehensive Pytest Tests +Run all pytest-based tests with detailed reporting: +```bash +./tests/run-all-tests.sh +``` + +This script: +- Checks for MUSCLE3 availability +- Runs core unit tests +- Runs generator unit tests +- Runs MUSCLE3 tests (if available) +- Runs integration tests +- Provides color-coded summary + +#### MUSCLE3-Only Tests +Run only MUSCLE3 tests (requires MUSCLE3 installation): +```bash +./tests/run-muscle3-tests.sh +``` + +#### IMAS Integration Tests +Run legacy IMAS integration tests (slow, requires IMAS): +```bash +./tests/run-tests.sh +``` + +### Direct Pytest Usage + +You can also use pytest directly for more control: + +```bash +# Run all unit tests +pytest tests/unit/ -v + +# Run all integration tests +pytest tests/integration/ -v + +# Run only MUSCLE3 tests +pytest tests/ -v -m muscle3 + +# Run tests excluding MUSCLE3 +pytest tests/ -v -m "not muscle3" + +# Run specific test file +pytest tests/unit/generators/test_muscle3_python.py -v + +# Run with coverage +pytest tests/ --cov=iwrap --cov-report=html + +# Run tests in parallel (requires pytest-xdist) +pytest tests/ -n auto +``` + +## Test Markers + +Tests are organized using pytest markers: + +- `@pytest.mark.unit` - Fast, isolated unit tests +- `@pytest.mark.integration` - Cross-component integration tests +- `@pytest.mark.muscle3` - Tests requiring MUSCLE3 library +- `@pytest.mark.imas` - Tests requiring IMAS +- `@pytest.mark.slow` - Slow-running tests +- `@pytest.mark.core` - Core functionality (no optional deps) +- `@pytest.mark.generators` - Generator-specific tests + +### Running Tests by Marker + +```bash +# Run only unit tests +pytest -m unit + +# Run only integration tests +pytest -m integration + +# Run only MUSCLE3 tests +pytest -m muscle3 + +# Run core tests (no optional dependencies) +pytest -m core + +# Combine markers +pytest -m "unit and not muscle3" +pytest -m "integration or muscle3" +``` + +## MUSCLE3 Tests + +MUSCLE3 tests are **optional** and will be automatically skipped if MUSCLE3 is not installed. + +### Installing MUSCLE3 + +```bash +# Install iWrap with MUSCLE3 support +pip install -e .[muscle3] + +# Or install MUSCLE3 separately +pip install muscle3>=0.7.0 +``` + +### MUSCLE3 Test Categories + +1. **Unit Tests** (`tests/unit/generators/`) + - Generator instantiation + - API version verification + - Property validation + - Discovery mechanism + +2. **Integration Tests** (`tests/integration/`) + - Full workflow testing + - Generator selection + - Cross-generator compatibility + +3. **MUSCLE3-Specific Tests** (`tests/muscle3/`) + - Actor generation and compilation + - Wrapped code integration + - Macro execution + - Workflow validation + - Code parameter handling + +## Test Fixtures + +Common fixtures are provided in `conftest.py`: + +### Environment Fixtures +- `muscle3_available` - Check if MUSCLE3 is installed +- `skip_if_no_muscle3` - Skip test if MUSCLE3 unavailable +- `clean_environment` - Clean environment variables +- `mock_imas_environment` - Mock IMAS environment + +### Directory Fixtures +- `temp_dir` - Temporary directory for tests +- `temp_project_dir` - Temporary project structure +- `temp_actor_dir` - Temporary actor generation directory + +### Settings Fixtures +- `mock_settings` - Mock iWrap settings +- `mock_muscle3_settings` - Mock MUSCLE3 settings + +### Generator Fixtures +- `generator_registry` - All discovered generators +- `core_generators` - Only core generators +- `muscle3_generators` - Only MUSCLE3 generators + +### Data Fixtures +- `sample_actor_yaml` - Sample actor configuration +- `sample_muscle3_yaml` - Sample MUSCLE3 actor configuration +- `sample_python_file` - Sample Python file +- `sample_yaml_file` - Sample YAML file + +## Continuous Integration + +### GitHub Actions / GitLab CI + +Example CI configuration: + +```yaml +test: + matrix: + include: + - name: "Core Tests" + env: + MUSCLE3: "false" + - name: "Core + MUSCLE3 Tests" + env: + MUSCLE3: "true" + + script: + - pip install -e . + - if [ "$MUSCLE3" = "true" ]; then pip install -e .[muscle3]; fi + - ./tests/run-all-tests-master.sh +``` + +### Test Reports + +Test logs are saved to `/tmp/`: +- `/tmp/iwrap_core_tests.log` - Core unit test output +- `/tmp/iwrap_generator_tests.log` - Generator test output +- `/tmp/iwrap_muscle3_tests.log` - MUSCLE3 test output +- `/tmp/iwrap_integration_tests.log` - Integration test output + +### JUnit XML Reports + +Generate JUnit XML reports for CI integration: + +```bash +pytest tests/ --junitxml=test-results.xml +``` + +## Writing New Tests + +### Unit Test Template + +```python +import pytest +from iwrap.generators.actor_generators.your_generator import YourGenerator + +@pytest.mark.unit +@pytest.mark.generators +def test_generator_instantiation(): + """Test that generator can be instantiated.""" + generator = YourGenerator() + assert generator is not None + +@pytest.mark.unit +@pytest.mark.muscle3 # If requires MUSCLE3 +def test_muscle3_feature(skip_if_no_muscle3): + """Test MUSCLE3-specific feature.""" + # Test code here + pass +``` + +### Integration Test Template + +```python +import pytest +from iwrap.generation_engine import Engine + +@pytest.mark.integration +def test_full_workflow(temp_dir, mock_settings): + """Test complete actor generation workflow.""" + engine = Engine(mock_settings) + # Test workflow here + pass +``` + +### Using Fixtures + +```python +@pytest.mark.unit +def test_with_fixtures(temp_dir, sample_yaml_file, mock_settings): + """Test using multiple fixtures.""" + # Fixtures are automatically provided + assert temp_dir.exists() + assert sample_yaml_file.exists() + assert mock_settings.work_dir == str(temp_dir) +``` + +## Test Coverage + +Generate coverage report: + +```bash +# HTML report +pytest tests/ --cov=iwrap --cov-report=html +open htmlcov/index.html + +# Terminal report +pytest tests/ --cov=iwrap --cov-report=term-missing + +# XML report (for CI) +pytest tests/ --cov=iwrap --cov-report=xml +``` + +## Debugging Tests + +### Run Single Test +```bash +pytest tests/unit/generators/test_muscle3_python.py::test_generator_import -v +``` + +### Drop into PDB on Failure +```bash +pytest tests/ --pdb +``` + +### Print Output +```bash +pytest tests/ -s # Show print statements +``` + +### Verbose Output +```bash +pytest tests/ -vv # Very verbose +``` + +## Performance + +### Test Execution Time + +- **Unit Tests**: < 5 seconds +- **Integration Tests**: < 30 seconds +- **MUSCLE3 Tests**: < 1 minute +- **IMAS Tests**: 5-30 minutes (depending on configuration) + +### Parallel Execution + +Install pytest-xdist for parallel test execution: + +```bash +pip install pytest-xdist + +# Run tests in parallel +pytest tests/ -n auto # Use all CPU cores +pytest tests/ -n 4 # Use 4 workers +``` + +## Troubleshooting + +### MUSCLE3 Tests Always Skipped + +Ensure MUSCLE3 is installed: +```bash +python3 -c "import muscle3; print(muscle3.__version__)" +``` + +Install if missing: +```bash +pip install iwrap[muscle3] +``` + +### Import Errors + +Ensure PYTHONPATH is set: +```bash +export PYTHONPATH=$(pwd):$PYTHONPATH +pytest tests/ +``` + +### Tests Pass Locally but Fail in CI + +Check: +1. Python version compatibility +2. Dependency versions +3. Environment variables +4. File permissions +5. PYTHONPATH configuration + +## Contributing + +When adding new features: + +1. **Write tests first** (TDD approach) +2. **Add appropriate markers** (`@pytest.mark.unit`, etc.) +3. **Use existing fixtures** when possible +4. **Document new fixtures** in conftest.py +5. **Update this README** if adding new test categories +6. **Ensure tests pass** with and without MUSCLE3 + +## Questions? + +For questions about the test suite: +- Check existing test files for examples +- Review `conftest.py` for available fixtures +- Consult the [iWrap Documentation](../docs/) +- Open an issue on GitHub/GitLab diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..fb52ca6 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,396 @@ +""" +Pytest configuration and shared fixtures for iWrap test suite. + +This module provides common fixtures and configuration for all iWrap tests, +including support for conditional MUSCLE3 testing. +""" + +import os +import sys +import tempfile +import shutil +from pathlib import Path +import pytest + +# Add the project root to the Python path +PROJECT_ROOT = Path(__file__).parent.parent +sys.path.insert(0, str(PROJECT_ROOT)) + + +####################################################################################################################### +# MUSCLE3 Availability Check +####################################################################################################################### + +def pytest_configure(config): + """Configure pytest with custom markers.""" + config.addinivalue_line("markers", "muscle3: marks tests as requiring MUSCLE3 (skipped if not installed)") + config.addinivalue_line("markers", "unit: marks tests as unit tests") + config.addinivalue_line("markers", "integration: marks tests as integration tests") + config.addinivalue_line("markers", "imas: marks tests as requiring IMAS") + config.addinivalue_line("markers", "slow: marks tests as slow running") + config.addinivalue_line("markers", "core: marks tests as core functionality (no optional deps)") + config.addinivalue_line("markers", "generators: marks tests for generator modules") + + +@pytest.fixture(scope="session") +def muscle3_available(): + """Check if MUSCLE3 is available.""" + try: + import muscle3 + return True + except ImportError: + return False + + +@pytest.fixture(scope="session") +def skip_if_no_muscle3(muscle3_available): + """Skip test if MUSCLE3 is not available.""" + if not muscle3_available: + pytest.skip("MUSCLE3 is not installed") + + +####################################################################################################################### +# Temporary Directory Fixtures +####################################################################################################################### + +@pytest.fixture +def temp_dir(): + """Create a temporary directory for test files.""" + temp_path = tempfile.mkdtemp() + yield Path(temp_path) + shutil.rmtree(temp_path, ignore_errors=True) + + +@pytest.fixture +def temp_project_dir(temp_dir): + """Create a temporary project directory with standard structure.""" + project_dir = temp_dir / "test_project" + project_dir.mkdir(parents=True, exist_ok=True) + + # Create standard subdirectories + (project_dir / "src").mkdir() + (project_dir / "tests").mkdir() + (project_dir / "docs").mkdir() + + yield project_dir + + # Cleanup happens automatically via temp_dir cleanup + + +@pytest.fixture +def temp_actor_dir(temp_dir): + """Create a temporary directory for actor generation.""" + actor_dir = temp_dir / "actors" + actor_dir.mkdir(parents=True, exist_ok=True) + yield actor_dir + + +####################################################################################################################### +# Mock Settings Fixtures +####################################################################################################################### + +@pytest.fixture +def mock_settings(temp_dir): + """Create mock settings for testing.""" + from iwrap.settings.settings import Settings + + settings = Settings() + settings.work_dir = str(temp_dir) + settings.actor_name = "test_actor" + settings.actor_language = "python" + settings.actor_type = "python" + + return settings + + +@pytest.fixture +def mock_muscle3_settings(temp_dir, muscle3_available): + """Create mock MUSCLE3 settings for testing.""" + if not muscle3_available: + pytest.skip("MUSCLE3 is not installed") + + from iwrap.settings.settings import Settings + + settings = Settings() + settings.work_dir = str(temp_dir) + settings.actor_name = "test_muscle3_actor" + settings.actor_language = "python" + settings.actor_type = "MUSCLE3-Python" + + return settings + + +####################################################################################################################### +# Sample Data Fixtures +####################################################################################################################### + +@pytest.fixture +def sample_actor_yaml(): + """Provide sample actor YAML configuration.""" + return """ +actor: + name: test_actor + description: Test actor for unit testing + language: python + type: python + +code_parameters: + - name: test_param + type: float + default: 1.0 + description: Test parameter + +ports: + inputs: + - name: input_port + description: Test input port + outputs: + - name: output_port + description: Test output port +""" + + +@pytest.fixture +def sample_muscle3_yaml(): + """Provide sample MUSCLE3 actor YAML configuration.""" + return """ +actor: + name: test_muscle3_actor + description: Test MUSCLE3 actor + language: python + type: MUSCLE3-Python + +code_parameters: + - name: test_param + type: float + default: 1.0 + description: Test parameter + +ports: + inputs: + - name: input_port + description: Test input port + operator: F_INIT + outputs: + - name: output_port + description: Test output port + operator: O_F +""" + + +####################################################################################################################### +# Generator Registry Fixtures +####################################################################################################################### + +@pytest.fixture +def generator_registry(): + """Get the generator registry.""" + from iwrap.generators.actor_generators.base_actor_generator import ActorGenerator + + # Force discovery of all generators + from iwrap.generation_engine import _engine + + # Return all discovered generators + return {gen.get_type(): gen for gen in ActorGenerator.__subclasses__()} + + +@pytest.fixture +def core_generators(generator_registry): + """Get only core (non-MUSCLE3) generators.""" + return { + name: gen for name, gen in generator_registry.items() + if "MUSCLE3" not in name + } + + +@pytest.fixture +def muscle3_generators(generator_registry, muscle3_available): + """Get only MUSCLE3 generators.""" + if not muscle3_available: + pytest.skip("MUSCLE3 is not installed") + + return { + name: gen for name, gen in generator_registry.items() + if "MUSCLE3" in name + } + + +####################################################################################################################### +# File System Fixtures +####################################################################################################################### + +@pytest.fixture +def sample_python_file(temp_dir): + """Create a sample Python file for testing.""" + py_file = temp_dir / "test_script.py" + py_file.write_text(""" +def test_function(): + return "Hello, World!" + +if __name__ == "__main__": + print(test_function()) +""") + return py_file + + +@pytest.fixture +def sample_yaml_file(temp_dir, sample_actor_yaml): + """Create a sample YAML file for testing.""" + yaml_file = temp_dir / "test_actor.yaml" + yaml_file.write_text(sample_actor_yaml) + return yaml_file + + +@pytest.fixture +def sample_muscle3_yaml_file(temp_dir, sample_muscle3_yaml): + """Create a sample MUSCLE3 YAML file for testing.""" + yaml_file = temp_dir / "test_muscle3_actor.yaml" + yaml_file.write_text(sample_muscle3_yaml) + return yaml_file + + +####################################################################################################################### +# Environment Fixtures +####################################################################################################################### + +@pytest.fixture +def clean_environment(monkeypatch): + """Provide a clean environment without IMAS/iWrap specific variables.""" + env_vars_to_remove = [ + 'IMAS_PREFIX', + 'IMAS_HOME', + 'MUSCLE3_HOME', + 'IWRAP_HOME', + ] + + for var in env_vars_to_remove: + monkeypatch.delenv(var, raising=False) + + yield + + +@pytest.fixture +def mock_imas_environment(monkeypatch, temp_dir): + """Mock IMAS environment variables.""" + imas_prefix = temp_dir / "imas" + imas_prefix.mkdir(parents=True, exist_ok=True) + + monkeypatch.setenv('IMAS_PREFIX', str(imas_prefix)) + monkeypatch.setenv('IMAS_HOME', str(imas_prefix)) + + yield imas_prefix + + +####################################################################################################################### +# Utility Fixtures +####################################################################################################################### + +@pytest.fixture +def caplog_debug(caplog): + """Set log level to DEBUG for capturing all logs.""" + import logging + caplog.set_level(logging.DEBUG) + return caplog + + +@pytest.fixture +def isolated_filesystem(): + """Create an isolated filesystem for testing.""" + cwd = os.getcwd() + temp_path = tempfile.mkdtemp() + os.chdir(temp_path) + + yield Path(temp_path) + + os.chdir(cwd) + shutil.rmtree(temp_path, ignore_errors=True) + + +####################################################################################################################### +# Session Scoped Fixtures +####################################################################################################################### + +@pytest.fixture(scope="session") +def project_root(): + """Get the project root directory.""" + return PROJECT_ROOT + + +@pytest.fixture(scope="session") +def iwrap_version(): + """Get the iWrap version.""" + try: + from iwrap._version import __version__ + return __version__ + except ImportError: + return "unknown" + + +####################################################################################################################### +# Parametrize Helpers +####################################################################################################################### + +def pytest_generate_tests(metafunc): + """Custom test parametrization.""" + # Parametrize actor languages if requested + if "actor_language" in metafunc.fixturenames: + languages = ["python"] # Add more as needed: ["python", "cpp", "fortran"] + metafunc.parametrize("actor_language", languages) + + # Parametrize generator types + if "generator_type" in metafunc.fixturenames: + from iwrap.generators.actor_generators.base_actor_generator import ActorGenerator + + # Get all generator types + generator_types = [gen.get_type() for gen in ActorGenerator.__subclasses__()] + + # Filter based on markers + if metafunc.definition.get_closest_marker("muscle3"): + # Only MUSCLE3 generators + generator_types = [gt for gt in generator_types if "MUSCLE3" in gt] + elif metafunc.definition.get_closest_marker("core"): + # Only core generators + generator_types = [gt for gt in generator_types if "MUSCLE3" not in gt] + + metafunc.parametrize("generator_type", generator_types) + + +####################################################################################################################### +# Pytest Hooks +####################################################################################################################### + +def pytest_collection_modifyitems(config, items): + """Modify test collection to handle MUSCLE3 markers.""" + try: + import muscle3 + muscle3_available = True + except ImportError: + muscle3_available = False + + if not muscle3_available: + skip_muscle3 = pytest.mark.skip(reason="MUSCLE3 is not installed") + for item in items: + if "muscle3" in item.keywords: + item.add_marker(skip_muscle3) + + +def pytest_report_header(config): + """Add custom information to pytest report header.""" + try: + from iwrap._version import __version__ + version = __version__ + except ImportError: + version = "unknown" + + try: + import muscle3 + muscle3_version = muscle3.__version__ + muscle3_status = f"installed (v{muscle3_version})" + except ImportError: + muscle3_status = "NOT installed" + + return [ + f"iWrap version: {version}", + f"MUSCLE3 status: {muscle3_status}", + f"Project root: {PROJECT_ROOT}", + ] diff --git a/tests/integration/test_muscle3_integration.py b/tests/integration/test_muscle3_integration.py new file mode 100644 index 0000000..9624a33 --- /dev/null +++ b/tests/integration/test_muscle3_integration.py @@ -0,0 +1,88 @@ +""" +Integration tests for MUSCLE3 actor generators +Tests the full workflow of MUSCLE3 actor generation +""" +import pytest +import sys +import tempfile +import shutil +from pathlib import Path + +# Check if MUSCLE3 is available +muscle3_available = pytest.importorskip("muscle3", reason="MUSCLE3 not installed") + + +class TestMuscle3Integration: + """Integration tests for MUSCLE3 functionality""" + + @pytest.fixture(autouse=True) + def setup(self): + """Setup test environment""" + iwrap_root = Path(__file__).parent.parent + if str(iwrap_root) not in sys.path: + sys.path.insert(0, str(iwrap_root)) + + # Create temporary directory for test outputs + self.temp_dir = tempfile.mkdtemp(prefix='iwrap_muscle3_test_') + yield + # Cleanup + if Path(self.temp_dir).exists(): + shutil.rmtree(self.temp_dir) + + @pytest.mark.muscle3 + @pytest.mark.integration + @pytest.mark.slow + def test_all_muscle3_generators_registered(self): + """Test that all MUSCLE3 generators are properly registered""" + from iwrap.generation_engine.engine import Engine + + engine = Engine() + engine.startup() + + generators = {g.type: g for g in engine.registered_generators} + + # Check all MUSCLE3 generators are present + assert 'MUSCLE3-Python' in generators + assert 'MUSCLE3-CPP' in generators + assert 'MUSCLE3-Fortran' in generators + + # Check they have correct API version + for gen_type in ['MUSCLE3-Python', 'MUSCLE3-CPP', 'MUSCLE3-Fortran']: + assert generators[gen_type].COMPLIANT_API == '2.1' + + @pytest.mark.muscle3 + @pytest.mark.integration + def test_muscle3_generator_selection(self): + """Test selecting different MUSCLE3 generators""" + from iwrap.generation_engine.engine import Engine + + engine = Engine() + engine.startup() + + # Test getting each generator type + for gen_type in ['MUSCLE3-Python', 'MUSCLE3-CPP', 'MUSCLE3-Fortran']: + generator = Engine.get_generator(gen_type) + assert generator is not None + assert generator.type == gen_type + + @pytest.mark.muscle3 + @pytest.mark.integration + def test_core_without_muscle3_still_works(self): + """Test that core iWrap works even if MUSCLE3 generators fail""" + from iwrap.generation_engine.engine import Engine + + engine = Engine() + engine.startup() + + generators = engine.registered_generators + + # At least the core python generator should be available + generator_types = [g.type for g in generators] + assert 'python' in generator_types + + # And ideally MUSCLE3 ones too + assert len(generator_types) >= 1 + + +if __name__ == '__main__': + pytest.main([__file__, '-v']) diff --git a/tests/muscle3/Makefile b/tests/muscle3/Makefile new file mode 100644 index 0000000..b6812db --- /dev/null +++ b/tests/muscle3/Makefile @@ -0,0 +1,54 @@ +.PHONY: wrapped actor actor-gui wf-run check-output test +export + +export TEST_DIR:=$(TESTS_DIR)/actors/$(CODE_LANGUAGE)/$(ACTOR_NAME) +LAST_SNAPSHOT:=$(lastword $(sort $(wildcard $(TEST_DIR)/run_$(ACTOR_NAME)_*/snapshots/snapshot_*.ymmsl))) + + +ACTOR_DIR:=$(TEST_DIR)/m3_actor +export PYTHONPATH:=$(ACTOR_DIR):$(PYTHONPATH) + +test: | clean wrapped actor wf-run check-output + + +check-output: + @echo "================ CHECKING TEST OUTPUT =========================" + @cmp expected.out test.out &> /dev/null || \ + (echo 'ERROR: expected and real workflow outcome differs!' && exit 1 ) + @echo 'SUCCESS: Test PASSED' + + +wf-run: + @echo "================ TEST FIRST RUN =========================" + + @$(RM) -f test.out + muscle_manager --start-all --log-level DEBUG\ + $(TESTS_DIR)/workflow/test.ymmsl \ + $(TESTS_DIR)/workflow/test_macro.ymmsl \ + $(TESTS_DIR)/actors/$(CODE_LANGUAGE)/$(ACTOR_NAME)/test_micro.ymmsl + +wf-restart: + @echo "================ TEST RESTART =========================" + @$(RM) -f test.out + muscle_manager --start-all --log-level DEBUG\ + $(TESTS_DIR)/workflow/test.ymmsl \ + $(TESTS_DIR)/workflow/test_macro.ymmsl \ + $(TESTS_DIR)/actors/$(CODE_LANGUAGE)/$(ACTOR_NAME)/test_micro.ymmsl \ + $(LAST_SNAPSHOT) + +actor: + @echo "================ ACTOR BUILD =========================" + iwrap --actor-type $(ACTOR_TYPE) --actor-name $(ACTOR_NAME) --install-dir $(ACTOR_DIR) --file $(TEST_DIR)/$(ACTOR_NAME).yaml + +actor-gui: + iwrap-gui --actor-type $(ACTOR_TYPE) --actor-name $(ACTOR_NAME) --install-dir $(ACTOR_DIR) --file $(TEST_DIR)/$(ACTOR_NAME).yaml + +wrapped: + @echo "================ WRAPPED CODE BUILD =========================" + @$(MAKE) -C $(TESTS_DIR)/wrapped_codes/$(CODE_LANGUAGE)/$(WRAPPED_CODE_NAME) clean + @$(MAKE) -C $(TESTS_DIR)/wrapped_codes/$(CODE_LANGUAGE)/$(WRAPPED_CODE_NAME) + +clean: + #@$(RM) -rf ./reports + @$(RM) -rf $(ACTOR_DIR) + @$(RM) -rf ./run_$(ACTOR_NAME)_* diff --git a/tests/muscle3/actors/cpp/basic_cpp/Makefile b/tests/muscle3/actors/cpp/basic_cpp/Makefile new file mode 100644 index 0000000..db80faf --- /dev/null +++ b/tests/muscle3/actors/cpp/basic_cpp/Makefile @@ -0,0 +1,10 @@ +export # export all variables to 'sub-Makefiles' + +ACTOR_NAME=basic_cpp +ACTOR_TYPE=muscle3-cpp +CODE_LANGUAGE=cpp +WRAPPED_CODE_NAME=basic + +clean wrapped actor actor-gui wf-run check-output test: + $(MAKE) -f $(TESTS_DIR)/Makefile $@ + diff --git a/tests/muscle3/actors/cpp/basic_cpp/basic_cpp.yaml b/tests/muscle3/actors/cpp/basic_cpp/basic_cpp.yaml new file mode 100644 index 0000000..68e0f26 --- /dev/null +++ b/tests/muscle3/actors/cpp/basic_cpp/basic_cpp.yaml @@ -0,0 +1,31 @@ +code_description: + implementation: + subroutines: + init: init_code + main: code_step + finalize: clean_up + data_type: legacy + code_path: $TESTS_DIR/wrapped_codes/cpp/basic/libbasic.a + include_path: $TESTS_DIR/wrapped_codes/cpp/basic/basic.h + programming_language: cpp + data_dictionary_compliant: 3.37.0 + arguments: + - name: core_profiles_in + type: core_profiles + intent: IN + - name: distribution_sources_out + type: distribution_sources + intent: OUT + documentation: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do + eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim + veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo + consequat. ' + settings: + compiler_cmd: g++ + mpi_compiler_cmd: + open_mp_switch: -DAL_MAJOR=$(AL_MAJOR) + extra_libraries: + pkg_config_defined: + path_defined: + + diff --git a/tests/muscle3/actors/cpp/basic_cpp/expected.out b/tests/muscle3/actors/cpp/basic_cpp/expected.out new file mode 100644 index 0000000..39f4a47 --- /dev/null +++ b/tests/muscle3/actors/cpp/basic_cpp/expected.out @@ -0,0 +1,3 @@ + 20001.000000000000 + 40002.000000000000 + 60003.000000000000 diff --git a/tests/muscle3/actors/cpp/basic_cpp/input_physics.xml b/tests/muscle3/actors/cpp/basic_cpp/input_physics.xml new file mode 100644 index 0000000..362c4bf --- /dev/null +++ b/tests/muscle3/actors/cpp/basic_cpp/input_physics.xml @@ -0,0 +1,31 @@ + + + + 131024 + + + 1 + + + 10 + + + 2 + + + 1.5 + + + public + + + iter + + + user_environment_variable + + + iter + + + diff --git a/tests/muscle3/actors/cpp/basic_cpp/input_physics.xsd b/tests/muscle3/actors/cpp/basic_cpp/input_physics.xsd new file mode 100644 index 0000000..3102010 --- /dev/null +++ b/tests/muscle3/actors/cpp/basic_cpp/input_physics.xsd @@ -0,0 +1,120 @@ + + + Code parameters for Physics Code + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Input/output shot number + + + + + + + Input run number + + + + + + + Output run number + + + + + + + Number of times the multiplication factor is applied + + + + + + + Multiplication factor for the major radius + + + + + + + Input database name to read from external database + + + + + + + Input machine name to read from external database + + + + + + + Local database name to write into local IMAS database + + + + + + + Local machine name to write into local IMAS database + + + + + diff --git a/tests/muscle3/actors/cpp/basic_cpp/test_micro.ymmsl b/tests/muscle3/actors/cpp/basic_cpp/test_micro.ymmsl new file mode 100644 index 0000000..bd9c8a0 --- /dev/null +++ b/tests/muscle3/actors/cpp/basic_cpp/test_micro.ymmsl @@ -0,0 +1,13 @@ +ymmsl_version: v0.1 + +model: + name: basic_cpp + +implementations: + micro: + executable: ${ACTOR_DIR}/basic_cpp/bin/basic_cpp.exe + + +resources: + micro: + threads: 1 \ No newline at end of file diff --git a/tests/muscle3/actors/cpp/basic_mpi_cpp/Makefile b/tests/muscle3/actors/cpp/basic_mpi_cpp/Makefile new file mode 100644 index 0000000..35143f4 --- /dev/null +++ b/tests/muscle3/actors/cpp/basic_mpi_cpp/Makefile @@ -0,0 +1,11 @@ +export # export all variables to 'sub-Makefiles' + +ACTOR_NAME=basic_mpi_cpp +ACTOR_TYPE=muscle3-cpp +CODE_LANGUAGE=cpp +WRAPPED_CODE_NAME=parallel_mpi + +clean wrapped actor actor-gui wf-run check-output test: + $(MAKE) -f $(TESTS_DIR)/Makefile $@ + + diff --git a/tests/muscle3/actors/cpp/basic_mpi_cpp/basic_mpi_cpp.yaml b/tests/muscle3/actors/cpp/basic_mpi_cpp/basic_mpi_cpp.yaml new file mode 100644 index 0000000..b243f6f --- /dev/null +++ b/tests/muscle3/actors/cpp/basic_mpi_cpp/basic_mpi_cpp.yaml @@ -0,0 +1,34 @@ +code_description: + implementation: + subroutines: + init: init_code + finalize: clean_up + main: code_step + data_type: legacy + programming_language: cpp + data_dictionary_compliant: 3.37.0 + code_path: $TESTS_DIR/wrapped_codes/cpp/parallel_mpi/libparallel_mpi.a + include_path: $TESTS_DIR/wrapped_codes/cpp/parallel_mpi/parallel_mpi.h + code_parameters: + parameters: + schema: + arguments: + - name: core_profiles_in + type: core_profiles + intent: IN + - name: distribution_sources_out + type: distribution_sources + intent: OUT + documentation: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do + eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim + veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo + consequat. ' + settings: + compiler_cmd: g++ + mpi_compiler_cmd: mpicxx + open_mp_switch: -DAL_MAJOR=$(AL_MAJOR) + extra_libraries: + pkg_config_defined: + path_defined: + + diff --git a/tests/muscle3/actors/cpp/basic_mpi_cpp/expected.out b/tests/muscle3/actors/cpp/basic_mpi_cpp/expected.out new file mode 100644 index 0000000..9ab02f0 --- /dev/null +++ b/tests/muscle3/actors/cpp/basic_mpi_cpp/expected.out @@ -0,0 +1,3 @@ + 2001.0000000000000 + 4002.0000000000000 + 6003.0000000000000 diff --git a/tests/muscle3/actors/cpp/basic_mpi_cpp/test_micro.ymmsl b/tests/muscle3/actors/cpp/basic_mpi_cpp/test_micro.ymmsl new file mode 100644 index 0000000..5d3fde8 --- /dev/null +++ b/tests/muscle3/actors/cpp/basic_mpi_cpp/test_micro.ymmsl @@ -0,0 +1,15 @@ +ymmsl_version: v0.1 + +model: + name: basic_mpi_cpp + +implementations: + micro: + execution_model: openmpi + executable: bash + args: ['-c', '${ACTOR_DIR}/basic_mpi_cpp/bin/basic_mpi_cpp.exe'] + +resources: + micro: + mpi_processes: 2 + diff --git a/tests/muscle3/actors/cpp/restart_cpp/Makefile b/tests/muscle3/actors/cpp/restart_cpp/Makefile new file mode 100644 index 0000000..8ac0ea4 --- /dev/null +++ b/tests/muscle3/actors/cpp/restart_cpp/Makefile @@ -0,0 +1,12 @@ +export # export all variables to 'sub-Makefiles' + +ACTOR_NAME=restart_cpp +ACTOR_TYPE=muscle3-cpp + +CODE_LANGUAGE=cpp +WRAPPED_CODE_NAME=basic + +test: | clean wrapped actor wf-run wf-restart check-output + +clean wrapped actor actor-gui wf-run wf-restart check-output: + $(MAKE) -f $(TESTS_DIR)/Makefile $@ \ No newline at end of file diff --git a/tests/muscle3/actors/cpp/restart_cpp/expected.out b/tests/muscle3/actors/cpp/restart_cpp/expected.out new file mode 100644 index 0000000..cb3ad6a --- /dev/null +++ b/tests/muscle3/actors/cpp/restart_cpp/expected.out @@ -0,0 +1,3 @@ + 80001.000000000000 + 100002.00000000000 + 120003.00000000000 diff --git a/tests/muscle3/actors/cpp/restart_cpp/restart_cpp.yaml b/tests/muscle3/actors/cpp/restart_cpp/restart_cpp.yaml new file mode 100644 index 0000000..d30b9ed --- /dev/null +++ b/tests/muscle3/actors/cpp/restart_cpp/restart_cpp.yaml @@ -0,0 +1,34 @@ +code_description: + implementation: + subroutines: + init: init_code + finalize: clean_up + main: code_step + get_state: get_code_state + set_state: restore_code_state + get_timestamp: get_timestamp_cpp + data_type: legacy + code_path: $TESTS_DIR/wrapped_codes/cpp/basic/libbasic.a + include_path: $TESTS_DIR/wrapped_codes/cpp/basic/basic.h + programming_language: cpp + data_dictionary_compliant: 3.37.0 + arguments: + - name: core_profiles_in + type: core_profiles + intent: IN + - name: distribution_sources_out + type: distribution_sources + intent: OUT + documentation: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do + eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim + veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo + consequat. ' + settings: + compiler_cmd: g++ + mpi_compiler_cmd: + open_mp_switch: -DAL_MAJOR=$(AL_MAJOR) + extra_libraries: + pkg_config_defined: + path_defined: + + diff --git a/tests/muscle3/actors/cpp/restart_cpp/test_micro.ymmsl b/tests/muscle3/actors/cpp/restart_cpp/test_micro.ymmsl new file mode 100644 index 0000000..91d077c --- /dev/null +++ b/tests/muscle3/actors/cpp/restart_cpp/test_micro.ymmsl @@ -0,0 +1,16 @@ +ymmsl_version: v0.1 + +model: + name: restart_cpp + +checkpoints: + at_end: true + +implementations: + micro: + executable: ${ACTOR_DIR}/restart_cpp/bin/restart_cpp.exe + + +resources: + micro: + threads: 1 \ No newline at end of file diff --git a/tests/muscle3/actors/cpp/restart_mpi_cpp/Makefile b/tests/muscle3/actors/cpp/restart_mpi_cpp/Makefile new file mode 100644 index 0000000..8cc0cb7 --- /dev/null +++ b/tests/muscle3/actors/cpp/restart_mpi_cpp/Makefile @@ -0,0 +1,12 @@ +export # export all variables to 'sub-Makefiles' + +ACTOR_NAME=restart_mpi_cpp +ACTOR_TYPE=muscle3-cpp + +CODE_LANGUAGE=cpp +WRAPPED_CODE_NAME=parallel_mpi + +test: | clean wrapped actor wf-run wf-restart check-output + +clean wrapped actor actor-gui wf-run wf-restart check-output: + $(MAKE) -f $(TESTS_DIR)/Makefile $@ \ No newline at end of file diff --git a/tests/muscle3/actors/cpp/restart_mpi_cpp/expected.out b/tests/muscle3/actors/cpp/restart_mpi_cpp/expected.out new file mode 100644 index 0000000..c63cb63 --- /dev/null +++ b/tests/muscle3/actors/cpp/restart_mpi_cpp/expected.out @@ -0,0 +1,3 @@ + 8001.0000000000000 + 10002.000000000000 + 12003.000000000000 diff --git a/tests/muscle3/actors/cpp/restart_mpi_cpp/restart_mpi_cpp.yaml b/tests/muscle3/actors/cpp/restart_mpi_cpp/restart_mpi_cpp.yaml new file mode 100644 index 0000000..59f42c6 --- /dev/null +++ b/tests/muscle3/actors/cpp/restart_mpi_cpp/restart_mpi_cpp.yaml @@ -0,0 +1,34 @@ +code_description: + implementation: + subroutines: + init: init_code + finalize: clean_up + main: code_step + get_state: get_code_state + set_state: restore_code_state + get_timestamp: get_timestamp_cpp + data_type: legacy + code_path: $TESTS_DIR/wrapped_codes/cpp/parallel_mpi/libparallel_mpi.a + include_path: $TESTS_DIR/wrapped_codes/cpp/parallel_mpi/parallel_mpi.h + programming_language: cpp + data_dictionary_compliant: 3.37.0 + arguments: + - name: core_profiles_in + type: core_profiles + intent: IN + - name: distribution_sources_out + type: distribution_sources + intent: OUT + documentation: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do + eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim + veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo + consequat. ' + settings: + compiler_cmd: g++ + mpi_compiler_cmd: mpicxx + open_mp_switch: -DAL_MAJOR=$(AL_MAJOR) + extra_libraries: + pkg_config_defined: + path_defined: + + diff --git a/tests/muscle3/actors/cpp/restart_mpi_cpp/test_micro.ymmsl b/tests/muscle3/actors/cpp/restart_mpi_cpp/test_micro.ymmsl new file mode 100644 index 0000000..5206a57 --- /dev/null +++ b/tests/muscle3/actors/cpp/restart_mpi_cpp/test_micro.ymmsl @@ -0,0 +1,18 @@ +ymmsl_version: v0.1 + +model: + name: restart_mpi_cpp + +checkpoints: + at_end: true + +implementations: + micro: + execution_model: openmpi + executable: bash + args: ['-c', '${ACTOR_DIR}/restart_mpi_cpp/bin/restart_mpi_cpp.exe'] + + +resources: + micro: + mpi_processes: 2 \ No newline at end of file diff --git a/tests/muscle3/actors/fortran/basic_mpi/Makefile b/tests/muscle3/actors/fortran/basic_mpi/Makefile new file mode 100644 index 0000000..7ab0274 --- /dev/null +++ b/tests/muscle3/actors/fortran/basic_mpi/Makefile @@ -0,0 +1,9 @@ +export # export all variables to 'sub-Makefiles' + +ACTOR_NAME=basic_mpi +ACTOR_TYPE=muscle3-fortran +CODE_LANGUAGE=fortran +WRAPPED_CODE_NAME=parallel_mpi + +clean wrapped actor actor-gui wf-run wf-restart check-output test: + $(MAKE) -f $(TESTS_DIR)/Makefile $@ diff --git a/tests/muscle3/actors/fortran/basic_mpi/basic_mpi.yaml b/tests/muscle3/actors/fortran/basic_mpi/basic_mpi.yaml new file mode 100644 index 0000000..3aef354 --- /dev/null +++ b/tests/muscle3/actors/fortran/basic_mpi/basic_mpi.yaml @@ -0,0 +1,33 @@ +code_description: + implementation: + subroutines: + init: init_code + finalize: clean_up + main: code_step + programming_language: Fortran + data_dictionary_compliant: 3.37.0 + code_path: $TESTS_DIR/wrapped_codes/fortran/parallel_mpi/libparallel_mpi.a + include_path: $TESTS_DIR/wrapped_codes/fortran/parallel_mpi/mod_parallel_mpi.mod + data_type: legacy + code_parameters: + parameters: + schema: + arguments: + - name: core_profiles_in + type: core_profiles + intent: IN + - name: distribution_sources_out + type: distribution_sources + intent: OUT + documentation: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do + eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim + veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo + consequat. ' + settings: + compiler_cmd: gfortran + mpi_compiler_cmd: mpifort + extra_libraries: + pkg_config_defined: + path_defined: + + diff --git a/tests/muscle3/actors/fortran/basic_mpi/expected.out b/tests/muscle3/actors/fortran/basic_mpi/expected.out new file mode 100644 index 0000000..e4373b0 --- /dev/null +++ b/tests/muscle3/actors/fortran/basic_mpi/expected.out @@ -0,0 +1,3 @@ + 2002.0000000000000 + 4003.0000000000000 + 6004.0000000000000 diff --git a/tests/muscle3/actors/fortran/basic_mpi/test_micro.ymmsl b/tests/muscle3/actors/fortran/basic_mpi/test_micro.ymmsl new file mode 100644 index 0000000..c704a1a --- /dev/null +++ b/tests/muscle3/actors/fortran/basic_mpi/test_micro.ymmsl @@ -0,0 +1,15 @@ +ymmsl_version: v0.1 + +model: + name: basic_mpi + +implementations: + micro: + execution_model: openmpi + executable: bash + args: ['-c', '${ACTOR_DIR}/basic_mpi/bin/basic_mpi.exe'] + +resources: + micro: + mpi_processes: 2 + diff --git a/tests/muscle3/actors/fortran/code_lifecycle/Makefile b/tests/muscle3/actors/fortran/code_lifecycle/Makefile new file mode 100644 index 0000000..b7de788 --- /dev/null +++ b/tests/muscle3/actors/fortran/code_lifecycle/Makefile @@ -0,0 +1,9 @@ +export # export all variables to 'sub-Makefiles' + +ACTOR_NAME=code_lifecycle +ACTOR_TYPE=muscle3-fortran +CODE_LANGUAGE=fortran +WRAPPED_CODE_NAME=basic + +clean wrapped actor actor-gui wf-run check-output test: + $(MAKE) -f $(TESTS_DIR)/Makefile $@ \ No newline at end of file diff --git a/tests/muscle3/actors/fortran/code_lifecycle/code_lifecycle.yaml b/tests/muscle3/actors/fortran/code_lifecycle/code_lifecycle.yaml new file mode 100644 index 0000000..ae6a9ac --- /dev/null +++ b/tests/muscle3/actors/fortran/code_lifecycle/code_lifecycle.yaml @@ -0,0 +1,34 @@ +code_description: + implementation: + subroutines: + init: init_code + main: code_step + finalize: clean_up + programming_language: Fortran + data_dictionary_compliant: 3.37.0 + data_type: legacy + code_path: $TESTS_DIR/wrapped_codes/fortran/basic/libbasic.a + include_path: $TESTS_DIR/wrapped_codes/fortran/basic/mod_basic.mod + code_parameters: + parameters: $TESTS_DIR/code_parameters/xml/input/input_physics.xml + schema: $TESTS_DIR/code_parameters/xml/input/input_physics.xsd + arguments: + - name: core_profiles_in + type: core_profiles + intent: IN + - name: distribution_sources_out + type: distribution_sources + intent: OUT + documentation: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do + eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim + veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo + consequat. ' + settings: + compiler_cmd: gfortran + mpi_compiler_cmd: + extra_libraries: + pkg_config_defined: + - xmllib + path_defined: + + diff --git a/tests/muscle3/actors/fortran/code_lifecycle/expected.out b/tests/muscle3/actors/fortran/code_lifecycle/expected.out new file mode 100644 index 0000000..39f4a47 --- /dev/null +++ b/tests/muscle3/actors/fortran/code_lifecycle/expected.out @@ -0,0 +1,3 @@ + 20001.000000000000 + 40002.000000000000 + 60003.000000000000 diff --git a/tests/muscle3/actors/fortran/code_lifecycle/input_physics.xml b/tests/muscle3/actors/fortran/code_lifecycle/input_physics.xml new file mode 100644 index 0000000..362c4bf --- /dev/null +++ b/tests/muscle3/actors/fortran/code_lifecycle/input_physics.xml @@ -0,0 +1,31 @@ + + + + 131024 + + + 1 + + + 10 + + + 2 + + + 1.5 + + + public + + + iter + + + user_environment_variable + + + iter + + + diff --git a/tests/muscle3/actors/fortran/code_lifecycle/input_physics.xsd b/tests/muscle3/actors/fortran/code_lifecycle/input_physics.xsd new file mode 100644 index 0000000..3102010 --- /dev/null +++ b/tests/muscle3/actors/fortran/code_lifecycle/input_physics.xsd @@ -0,0 +1,120 @@ + + + Code parameters for Physics Code + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Input/output shot number + + + + + + + Input run number + + + + + + + Output run number + + + + + + + Number of times the multiplication factor is applied + + + + + + + Multiplication factor for the major radius + + + + + + + Input database name to read from external database + + + + + + + Input machine name to read from external database + + + + + + + Local database name to write into local IMAS database + + + + + + + Local machine name to write into local IMAS database + + + + + diff --git a/tests/muscle3/actors/fortran/code_lifecycle/test_micro.ymmsl b/tests/muscle3/actors/fortran/code_lifecycle/test_micro.ymmsl new file mode 100644 index 0000000..d49d603 --- /dev/null +++ b/tests/muscle3/actors/fortran/code_lifecycle/test_micro.ymmsl @@ -0,0 +1,13 @@ +ymmsl_version: v0.1 + +model: + name: code_lifecycle + +implementations: + micro: + executable: ${ACTOR_DIR}/code_lifecycle/bin/code_lifecycle.exe + + +resources: + micro: + threads: 1 \ No newline at end of file diff --git a/tests/muscle3/actors/fortran/code_restart/Makefile b/tests/muscle3/actors/fortran/code_restart/Makefile new file mode 100644 index 0000000..dc4b38c --- /dev/null +++ b/tests/muscle3/actors/fortran/code_restart/Makefile @@ -0,0 +1,11 @@ +export # export all variables to 'sub-Makefiles' + +ACTOR_NAME=code_restart +ACTOR_TYPE=muscle3-fortran +CODE_LANGUAGE=fortran +WRAPPED_CODE_NAME=basic + +test: | clean wrapped actor wf-run wf-restart check-output + +clean wrapped actor actor-gui wf-run wf-restart check-output: + $(MAKE) -f $(TESTS_DIR)/Makefile $@ \ No newline at end of file diff --git a/tests/muscle3/actors/fortran/code_restart/code_restart.yaml b/tests/muscle3/actors/fortran/code_restart/code_restart.yaml new file mode 100644 index 0000000..e8968f7 --- /dev/null +++ b/tests/muscle3/actors/fortran/code_restart/code_restart.yaml @@ -0,0 +1,33 @@ +code_description: + implementation: + subroutines: + init: init_code + finalize: clean_up + main: code_step + get_state: get_code_state + set_state: restore_code_state + get_timestamp: get_timestamp + programming_language: Fortran + data_dictionary_compliant: 3.37.0 + data_type: legacy + code_path: $TESTS_DIR/wrapped_codes/fortran/basic/libbasic.a + include_path: $TESTS_DIR/wrapped_codes/fortran/basic/mod_basic.mod + arguments: + - name: core_profiles_in + type: core_profiles + intent: IN + - name: distribution_sources_out + type: distribution_sources + intent: OUT + documentation: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do + eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim + veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo + consequat. ' + settings: + compiler_cmd: gfortran + mpi_compiler_cmd: + extra_libraries: + pkg_config_defined: + path_defined: + + diff --git a/tests/muscle3/actors/fortran/code_restart/expected.out b/tests/muscle3/actors/fortran/code_restart/expected.out new file mode 100644 index 0000000..cb3ad6a --- /dev/null +++ b/tests/muscle3/actors/fortran/code_restart/expected.out @@ -0,0 +1,3 @@ + 80001.000000000000 + 100002.00000000000 + 120003.00000000000 diff --git a/tests/muscle3/actors/fortran/code_restart/test_micro.ymmsl b/tests/muscle3/actors/fortran/code_restart/test_micro.ymmsl new file mode 100644 index 0000000..94ac2ed --- /dev/null +++ b/tests/muscle3/actors/fortran/code_restart/test_micro.ymmsl @@ -0,0 +1,16 @@ +ymmsl_version: v0.1 + +model: + name: code_restart + +checkpoints: + at_end: true + +implementations: + micro: + executable: ${ACTOR_DIR}/code_restart/bin/code_restart.exe + + +resources: + micro: + threads: 1 \ No newline at end of file diff --git a/tests/muscle3/actors/fortran/code_restart_mpi/Makefile b/tests/muscle3/actors/fortran/code_restart_mpi/Makefile new file mode 100644 index 0000000..ea8da26 --- /dev/null +++ b/tests/muscle3/actors/fortran/code_restart_mpi/Makefile @@ -0,0 +1,11 @@ +export # export all variables to 'sub-Makefiles' + +ACTOR_NAME=code_restart_mpi +ACTOR_TYPE=muscle3-fortran +CODE_LANGUAGE=fortran +WRAPPED_CODE_NAME=parallel_mpi + +test: | clean wrapped actor wf-run wf-restart check-output + +clean wrapped actor actor-gui wf-run wf-restart check-output: + $(MAKE) -f $(TESTS_DIR)/Makefile $@ \ No newline at end of file diff --git a/tests/muscle3/actors/fortran/code_restart_mpi/code_restart_mpi.yaml b/tests/muscle3/actors/fortran/code_restart_mpi/code_restart_mpi.yaml new file mode 100644 index 0000000..1c66e29 --- /dev/null +++ b/tests/muscle3/actors/fortran/code_restart_mpi/code_restart_mpi.yaml @@ -0,0 +1,33 @@ +code_description: + implementation: + subroutines: + init: init_code + finalize: clean_up + main: code_step + get_state: get_code_state + set_state: restore_code_state + get_timestamp: get_timestamp + programming_language: Fortran + data_dictionary_compliant: 3.37.0 + data_type: legacy + code_path: $TESTS_DIR/wrapped_codes/fortran/parallel_mpi/libparallel_mpi.a + include_path: $TESTS_DIR/wrapped_codes/fortran/parallel_mpi/mod_parallel_mpi.mod + arguments: + - name: core_profiles_in + type: core_profiles + intent: IN + - name: distribution_sources_out + type: distribution_sources + intent: OUT + documentation: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do + eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim + veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo + consequat. ' + settings: + compiler_cmd: gfortran + mpi_compiler_cmd: mpifort + extra_libraries: + pkg_config_defined: + path_defined: + + diff --git a/tests/muscle3/actors/fortran/code_restart_mpi/expected.out b/tests/muscle3/actors/fortran/code_restart_mpi/expected.out new file mode 100644 index 0000000..261d3b4 --- /dev/null +++ b/tests/muscle3/actors/fortran/code_restart_mpi/expected.out @@ -0,0 +1,3 @@ + 8002.0000000000000 + 10003.000000000000 + 12004.000000000000 diff --git a/tests/muscle3/actors/fortran/code_restart_mpi/test_micro.ymmsl b/tests/muscle3/actors/fortran/code_restart_mpi/test_micro.ymmsl new file mode 100644 index 0000000..f1a7d67 --- /dev/null +++ b/tests/muscle3/actors/fortran/code_restart_mpi/test_micro.ymmsl @@ -0,0 +1,18 @@ +ymmsl_version: v0.1 + +model: + name: code_restart_mpi + +checkpoints: + at_end: true + +implementations: + micro: + execution_model: openmpi + executable: bash + args: ['-c', '${ACTOR_DIR}/code_restart_mpi/bin/code_restart_mpi.exe'] + + +resources: + micro: + mpi_processes: 2 \ No newline at end of file diff --git a/tests/muscle3/actors/python/basic_python/Makefile b/tests/muscle3/actors/python/basic_python/Makefile new file mode 100644 index 0000000..87f868e --- /dev/null +++ b/tests/muscle3/actors/python/basic_python/Makefile @@ -0,0 +1,9 @@ +export # export all variables to 'sub-Makefiles' + +ACTOR_NAME=basic_python +ACTOR_TYPE=muscle3-python +CODE_LANGUAGE=python +WRAPPED_CODE_NAME=basic + +clean wrapped actor actor-gui wf-run check-output test: + $(MAKE) -f $(TESTS_DIR)/Makefile $@ diff --git a/tests/muscle3/actors/python/basic_python/basic_python.yaml b/tests/muscle3/actors/python/basic_python/basic_python.yaml new file mode 100644 index 0000000..0ca85b2 --- /dev/null +++ b/tests/muscle3/actors/python/basic_python/basic_python.yaml @@ -0,0 +1,25 @@ +code_description: + implementation: + subroutines: + init: init_code + main: code_step + finalize: clean_up + programming_language: Python + data_dictionary_compliant: 3.37.0 + data_type: legacy + code_path: $TESTS_DIR/wrapped_codes/python/basic/basic.py + include_path: $TESTS_DIR/wrapped_codes/python/basic/basic.py + arguments: + - name: core_profiles_in + type: core_profiles + intent: IN + - name: distribution_sources_out + type: distribution_sources + intent: OUT + documentation: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do + eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim + veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo + consequat. ' + + + diff --git a/tests/muscle3/actors/python/basic_python/expected.out b/tests/muscle3/actors/python/basic_python/expected.out new file mode 100644 index 0000000..b0e1d76 --- /dev/null +++ b/tests/muscle3/actors/python/basic_python/expected.out @@ -0,0 +1,3 @@ + 19.000000000000000 + 38.000000000000000 + 57.000000000000000 diff --git a/tests/muscle3/actors/python/basic_python/test_micro.ymmsl b/tests/muscle3/actors/python/basic_python/test_micro.ymmsl new file mode 100644 index 0000000..b529e3a --- /dev/null +++ b/tests/muscle3/actors/python/basic_python/test_micro.ymmsl @@ -0,0 +1,13 @@ +ymmsl_version: v0.1 + +model: + name: basic_python + +implementations: + micro: + executable: python + args: ${ACTOR_DIR}/basic_python/standalone.py + +resources: + micro: + threads: 1 \ No newline at end of file diff --git a/tests/muscle3/actors/python/restart_python/Makefile b/tests/muscle3/actors/python/restart_python/Makefile new file mode 100644 index 0000000..c7253ab --- /dev/null +++ b/tests/muscle3/actors/python/restart_python/Makefile @@ -0,0 +1,12 @@ +export # export all variables to 'sub-Makefiles' + +ACTOR_NAME=restart_python +ACTOR_TYPE=muscle3-python + +CODE_LANGUAGE=python +WRAPPED_CODE_NAME=basic + +test: | clean wrapped actor wf-run wf-restart check-output + +clean wrapped actor actor-gui wf-run wf-restart check-output: + $(MAKE) -f $(TESTS_DIR)/Makefile $@ diff --git a/tests/muscle3/actors/python/restart_python/expected.out b/tests/muscle3/actors/python/restart_python/expected.out new file mode 100644 index 0000000..772ced3 --- /dev/null +++ b/tests/muscle3/actors/python/restart_python/expected.out @@ -0,0 +1,3 @@ + 76.000000000000000 + 95.000000000000000 + 114.00000000000000 diff --git a/tests/muscle3/actors/python/restart_python/restart_python.yaml b/tests/muscle3/actors/python/restart_python/restart_python.yaml new file mode 100644 index 0000000..d64f286 --- /dev/null +++ b/tests/muscle3/actors/python/restart_python/restart_python.yaml @@ -0,0 +1,28 @@ +code_description: + implementation: + subroutines: + init: init_code + main: code_step + finalize: clean_up + get_state: get_code_state + set_state: restore_code_state + get_timestamp: get_timestamp + programming_language: Python + data_dictionary_compliant: 3.37.0 + data_type: legacy + code_path: $TESTS_DIR/wrapped_codes/python/basic/basic.py + include_path: $TESTS_DIR/wrapped_codes/python/basic/basic.py + arguments: + - name: core_profiles_in + type: core_profiles + intent: IN + - name: distribution_sources_out + type: distribution_sources + intent: OUT + documentation: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do + eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim + veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo + consequat. ' + + + diff --git a/tests/muscle3/actors/python/restart_python/test_micro.ymmsl b/tests/muscle3/actors/python/restart_python/test_micro.ymmsl new file mode 100644 index 0000000..588920e --- /dev/null +++ b/tests/muscle3/actors/python/restart_python/test_micro.ymmsl @@ -0,0 +1,17 @@ +ymmsl_version: v0.1 + +model: + name: restart_python + +checkpoints: + at_end: true + +implementations: + micro: + executable: python + args: ${ACTOR_DIR}/restart_python/standalone.py + + +resources: + micro: + threads: 1 \ No newline at end of file diff --git a/tests/muscle3/code_parameters/xml/input/input_physics.xml b/tests/muscle3/code_parameters/xml/input/input_physics.xml new file mode 100644 index 0000000..362c4bf --- /dev/null +++ b/tests/muscle3/code_parameters/xml/input/input_physics.xml @@ -0,0 +1,31 @@ + + + + 131024 + + + 1 + + + 10 + + + 2 + + + 1.5 + + + public + + + iter + + + user_environment_variable + + + iter + + + diff --git a/tests/muscle3/code_parameters/xml/input/input_physics.xsd b/tests/muscle3/code_parameters/xml/input/input_physics.xsd new file mode 100644 index 0000000..3102010 --- /dev/null +++ b/tests/muscle3/code_parameters/xml/input/input_physics.xsd @@ -0,0 +1,120 @@ + + + Code parameters for Physics Code + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Input/output shot number + + + + + + + Input run number + + + + + + + Output run number + + + + + + + Number of times the multiplication factor is applied + + + + + + + Multiplication factor for the major radius + + + + + + + Input database name to read from external database + + + + + + + Input machine name to read from external database + + + + + + + Local database name to write into local IMAS database + + + + + + + Local machine name to write into local IMAS database + + + + + diff --git a/tests/muscle3/macro/Makefile b/tests/muscle3/macro/Makefile new file mode 100644 index 0000000..bc4dede --- /dev/null +++ b/tests/muscle3/macro/Makefile @@ -0,0 +1,27 @@ +AL_VERSION ?= $(UAL_VERSION) +AL_MAJOR := $(firstword $(subst ., ,$(AL_VERSION))) + +FLAGS= -g -fPIC # FPIC AND PREPROCESSING OPTIONS + +ifeq ($(AL_MAJOR),) + $(error Could not extract major AL version from UAL_VERSION/AL_VERSION system variable. Is your IMAS module loaded?) +else ifeq ($(AL_MAJOR), 5) + IMAS_LIB_NAME=al-fortran +else ifeq ($(AL_MAJOR), 4) + IMAS_LIB_NAME=imas-$(FC) +endif + +LIBS = `pkg-config --libs $(IMAS_LIB_NAME) ymmsl libmuscle_fortran` +INCLUDES = `pkg-config --cflags $(IMAS_LIB_NAME) ymmsl libmuscle_fortran` + +all: test_macro.exe + +%.o: %.f90 + $(FC) $(FLAGS) -c -o $@ $^ ${INCLUDES} + +%.exe: %.o + $(FC) $(FLAGS) -o $@ $^ $(LIBS) + +clean: + @$(RM) -f *.o *.exe + diff --git a/tests/muscle3/macro/test_macro.f90 b/tests/muscle3/macro/test_macro.f90 new file mode 100644 index 0000000..a8ce709 --- /dev/null +++ b/tests/muscle3/macro/test_macro.f90 @@ -0,0 +1,144 @@ +program helloworld + use ids_routines + use ymmsl + use libmuscle + implicit none + + + + + + type (ids_core_profiles) :: core_profiles_out + type (ids_distribution_sources) :: distribution_sources_in + + real (selected_real_kind(15)) :: t_cur, t_next, t_max, dt + character(len=1), dimension(:), allocatable :: serialized_ids + real :: time + + type(LIBMUSCLE_PortsDescription) :: ports + type(LIBMUSCLE_Instance) :: instance + + type(LIBMUSCLE_Message) :: rmsg + type(LIBMUSCLE_DataConstRef) :: ritem + + type(LIBMUSCLE_Message) :: smsg + type(LIBMUSCLE_Data) :: sitem + + character(:), allocatable :: test_dir_path + + write(*,*) "Starting Fortran M3 macro" + + test_dir_path = get_test_dir() + + open(10, file=test_dir_path//"/test.out", status="NEW") + + ports = LIBMUSCLE_PortsDescription_create() + call LIBMUSCLE_PortsDescription_add(ports, YMMSL_OPERATOR_O_I, 'core_profiles_out') + call LIBMUSCLE_PortsDescription_add(ports, YMMSL_OPERATOR_S, 'distribution_sources_in') + instance = LIBMUSCLE_Instance_create(ports, & + LIBMUSCLE_InstanceFlags(KEEPS_NO_STATE_FOR_NEXT_USE=.true.)) + call LIBMUSCLE_PortsDescription_free(ports) + + do while (LIBMUSCLE_Instance_reuse_instance(instance)) + ! F_INIT + t_max = LIBMUSCLE_Instance_get_setting_as_real8(instance, 't_max') + dt = LIBMUSCLE_Instance_get_setting_as_real8(instance, 'dt') + + time = 1 + + t_cur = 0.0 + do while (t_cur + dt < t_max) + ! O_I + t_next = t_cur + dt + + + ! SENDING DATA + allocate(core_profiles_out%ids_properties%comment(1)) + core_profiles_out%ids_properties%comment(1) = 'Hello world!' + core_profiles_out%ids_properties%homogeneous_time = IDS_TIME_MODE_HOMOGENEOUS + allocate(core_profiles_out%code%name(1)) + core_profiles_out%code%name = 'HelloWorld Actor in Fortran' + + allocate(core_profiles_out%time(1)) + core_profiles_out%time(1) = t_next + + + call ids_serialize(core_profiles_out, serialized_ids) + call ids_deallocate(core_profiles_out) + + sitem = LIBMUSCLE_Data_create_byte_array(serialized_ids) + smsg = LIBMUSCLE_Message_create(t_cur, sitem) + + if (t_next + dt <= t_max) then + call LIBMUSCLE_Message_set_next_timestamp(smsg, t_next) + end if + call LIBMUSCLE_Instance_send(instance, 'core_profiles_out', smsg) + call LIBMUSCLE_Message_free(smsg) + call LIBMUSCLE_Data_free(sitem) + deallocate(serialized_ids) + + + ! > > > - - - RECEIVING DATA - - - < < < + + rmsg = LIBMUSCLE_Instance_receive(instance, 'distribution_sources_in') + ritem = LIBMUSCLE_Message_get_data(rmsg); + + allocate(serialized_ids(LIBMUSCLE_DataConstRef_size(ritem))) + call LIBMUSCLE_DataConstRef_as_byte_array(ritem, serialized_ids) + call LIBMUSCLE_DataConstRef_free(ritem) + + + ! deserialize and verify received IDS + call ids_deserialize(serialized_ids, distribution_sources_in) + deallocate(serialized_ids) + + if (.not. associated(distribution_sources_in%time)) then + call LIBMUSCLE_Instance_error_shutdown(instance, "Received TIME is not associated") + call exit(1) + end if + + write(*,*) "TIME received ", distribution_sources_in%time + write(10,*) distribution_sources_in%time + + time = distribution_sources_in%time(1) + + call ids_deallocate(distribution_sources_in) + + if (LIBMUSCLE_Message_has_next_timestamp(rmsg)) then + t_max = LIBMUSCLE_Message_next_timestamp(rmsg) + end if + + call LIBMUSCLE_Message_free(rmsg) + + ! a simulation would actually update something here, but we're just saying + ! hi to connected actors and don't do anything else + t_cur = t_cur + dt + end do + + + + end do + + close(10) + +contains + + FUNCTION get_test_dir() RESULT(test_dir_path) + + CHARACTER(:), allocatable :: test_dir_path + CHARACTER(len=1000) :: tmp_str + INTEGER :: var_size, status + + CALL get_environment_variable ("TEST_DIR", value=tmp_str, length=var_size, status=status) + + if (status .ne. 0) then + write (*,*) 'Getting environment variable "TEST_DIR" failed: status = ', status + stop + end if + + ALLOCATE(character(len=var_size) :: test_dir_path) + + test_dir_path =tmp_str(1:var_size) + + END FUNCTION get_test_dir +end program helloworld diff --git a/tests/muscle3/pytest.ini b/tests/muscle3/pytest.ini new file mode 100644 index 0000000..8176c20 --- /dev/null +++ b/tests/muscle3/pytest.ini @@ -0,0 +1,11 @@ +[pytest] +markers = + slow: Mark test as slow + fast: Mark test as fast + +log_cli = True +log_cli_level = INFO +log_cli_format = [%(levelname)s] + %(message)s + +addopts = -v --junitxml=test_results.xml \ No newline at end of file diff --git a/tests/muscle3/pytests.py b/tests/muscle3/pytests.py new file mode 100755 index 0000000..9da8c49 --- /dev/null +++ b/tests/muscle3/pytests.py @@ -0,0 +1,4 @@ +#!/bin/env python3 + + +print("I am placeholder for a future tests!") diff --git a/tests/muscle3/run-tests.sh b/tests/muscle3/run-tests.sh new file mode 100755 index 0000000..cbe4f67 --- /dev/null +++ b/tests/muscle3/run-tests.sh @@ -0,0 +1,5 @@ +#!/bin/bash + + +echo "I am placeholder for a future tests!" + diff --git a/tests/muscle3/workflow/test.ymmsl b/tests/muscle3/workflow/test.ymmsl new file mode 100644 index 0000000..fc031f6 --- /dev/null +++ b/tests/muscle3/workflow/test.ymmsl @@ -0,0 +1,16 @@ +ymmsl_version: v0.1 + +model: + name: helloworld + components: + macro: macro + micro: micro + conduits: + macro.core_profiles_out: micro.core_profiles_in + micro.distribution_sources_out: macro.distribution_sources_in +settings: + t_max: 4.0 + dt: 1.0 + muscle_remote_log_level: DEBUG + + diff --git a/tests/muscle3/workflow/test_macro.ymmsl b/tests/muscle3/workflow/test_macro.ymmsl new file mode 100644 index 0000000..cd41116 --- /dev/null +++ b/tests/muscle3/workflow/test_macro.ymmsl @@ -0,0 +1,9 @@ +ymmsl_version: v0.1 + +implementations: + macro: + executable: ${TESTS_DIR}/macro/test_macro.exe + +resources: + macro: + threads: 1 \ No newline at end of file diff --git a/tests/muscle3/wrapped_codes/cpp/basic/Makefile b/tests/muscle3/wrapped_codes/cpp/basic/Makefile new file mode 100644 index 0000000..c9c051f --- /dev/null +++ b/tests/muscle3/wrapped_codes/cpp/basic/Makefile @@ -0,0 +1,44 @@ +AL_VERSION ?= $(UAL_VERSION) +AL_MAJOR := $(firstword $(subst ., ,$(AL_VERSION))) + +CFLAGS = -pthread -g -O0 -D__USE_XOPEN2K8 -fPIC -DAL_MAJOR=$(AL_MAJOR) + +ifeq ($(AL_MAJOR),) + $(error Could not extract major AL version from UAL_VERSION/AL_VERSION system variable. Is your IMAS module loaded?) +else ifeq ($(AL_MAJOR), 5) + IMAS_LIB_NAME=al-cpp +else ifeq ($(AL_MAJOR), 4) + IMAS_LIB_NAME=imas-cpp +endif + +LIBS = `pkg-config --libs $(IMAS_LIB_NAME)` +INCLUDES = `pkg-config --cflags $(IMAS_LIB_NAME)` + +all: libbasic.a + +# LIBRARY COMPILATION +lib%.a: %.o + $(AR) -rcs $@ $^ + +# RULES FOR ALL OBJECT FILES +%.o: %.cpp + $(CXX) $(CFLAGS) -c -o $@ $^ $(INCLUDES) $(LIBS) + + +# CLEAN DIRECTORY +clean: + $(RM) *.o *.a + + + + + + + + + + + + + + diff --git a/tests/muscle3/wrapped_codes/cpp/basic/basic.cpp b/tests/muscle3/wrapped_codes/cpp/basic/basic.cpp new file mode 100644 index 0000000..73bca36 --- /dev/null +++ b/tests/muscle3/wrapped_codes/cpp/basic/basic.cpp @@ -0,0 +1,134 @@ +#include + +#include "basic.h" + +// ======================================= +// GET STATE +//======================================= + +int code_state = 0; + +void get_code_state( std::string& state_out, int& status_code, std::string& status_message) +{ + status_code = 0; + + status_message = "INITIALISATION: OK"; + state_out = std::to_string(code_state); + + std::cout << "=======================================================" << std::endl; + std::cout << "Code restart CPP: GET STATE called" << std::endl; + std::cout << "STATE is : " << state_out << std::endl; + std::cout << "=======================================================" << std::endl; +} + +// ======================================= +// SET STATE +//======================================= +void restore_code_state( std::string state, int& status_code, std::string& status_message) +{ + status_code = 0; + status_message = "FINALISATION: OK"; + + code_state = std::stoi( state ); + std::cout << "=======================================================" << std::endl; + std::cout << "Code lifecycle CPP: RESTORE STATE called" << std::endl; + std::cout << "STATE TO BE RESTORED : " << code_state << std::endl; + std::cout << "=======================================================" << std::endl; +} + +// ======================================= +// GET TIMESTAMP +//======================================= +void get_timestamp_cpp(double& timestamp_out, int& status_code, std::string& status_message) +{ + timestamp_out = (double) code_state; +} + +// ======================================= +// INITIALISATION +//======================================= + +void init_code (int& status_code, std::string& status_message) +{ + status_code = 0; + + status_message = "INITIALISATION: OK"; + + printf("=======================================================\n"); + printf("Code lifecycle CPP: INITIALISATION called\n"); + printf("=======================================================\n"); + // printf( "%s\n", *(codeparam.parameters)); + printf( "\n=======================================\n"); +} + +// ======================================= +// FINALISATION +//======================================= +void clean_up( int& status_code, std::string& status_message) +{ + status_code = 0; + status_message = "FINALISATION: OK"; + + printf("=======================================================\n"); + printf("Code lifecycle CPP: FINALISATION called\n"); + printf("=======================================================\n"); +} + +// ======================================= +// MAIN +//======================================= +void code_step(const IdsNs::IDS::core_profiles& core_profiles_in, + IdsNs::IDS::distribution_sources& distribution_sources_out, + int& status_code, std::string& status_message) +{ + int idsSize = 0; + + // INITIALISATION OF ERROR FLAG + status_code = 0; + + // INITIAL DISPLAY + std::cout << "=======================================" << std::endl; + std::cout << "START OF PHYSICS CODE" << std::endl; + + + std::cout << "Starting from: " << code_state << std::endl; + std::cout << "PWD: " << get_current_dir_name() << std::endl; + + + + for (int i = 0; i < 20; i++) + { + // COMPUTATIONS + code_state++; + } + + + std::cout << "Counting to : " << code_state << std::endl; + + distribution_sources_out.ids_properties.homogeneous_time = IDS_TIME_MODE_HOMOGENEOUS; + idsSize = core_profiles_in.time.extent(0); + std::cout << "Size of input IDS: " < 0) { + distribution_sources_out.time.resize(idsSize); + // Fill in the output IDS (Physical data) + for(int i=0; i < idsSize; i++) + { + // Time : copy from input IDS + distribution_sources_out.time(i) = 1000 * code_state + core_profiles_in.time(i); + } + } + else { + distribution_sources_out.time.resize(1); + distribution_sources_out.time(1) = 1000 * code_state; + } + + // INITIALISATION OF STATUS INFO + status_message = "Status info of code_restart CPP"; + + + // FINAL DISPLAY + std::cout << "END OF PHYSICS CODE" << std::endl; + std::cout << "=======================================" << std::endl; + std::cout << " " << std::endl; +} diff --git a/tests/muscle3/wrapped_codes/cpp/basic/basic.h b/tests/muscle3/wrapped_codes/cpp/basic/basic.h new file mode 100644 index 0000000..976203e --- /dev/null +++ b/tests/muscle3/wrapped_codes/cpp/basic/basic.h @@ -0,0 +1,27 @@ +#ifndef _CODE_RESTART_CPP +#define _CODE_RESTART_CPP + +#if 5 == AL_MAJOR + #include "ALClasses.h" +#elif AL_MAJOR == 4 + #include "UALClasses.h" +#else + #warning Could not find AL_MAJOR variable. Assuming AL version = 5.x.x + #include "ALClasses.h" +#endif + +void get_code_state( std::string& state_out, int& status_code, std::string& status_message); + +void restore_code_state( std::string state, int& status_code, std::string& status_message); + +void get_timestamp_cpp(double& timestamp_out, int& status_code, std::string& status_message); + + +void init_code (int& status_code, std::string& status_message); +void clean_up( int& status_code, std::string& status_message); + +void code_step(const IdsNs::IDS::core_profiles& core_profiles_in, + IdsNs::IDS::distribution_sources& distribution_sources_out, + int& status_code, std::string& status_message); + +#endif // _CODE_RESTART_CPP \ No newline at end of file diff --git a/tests/muscle3/wrapped_codes/cpp/parallel_mpi/Makefile b/tests/muscle3/wrapped_codes/cpp/parallel_mpi/Makefile new file mode 100644 index 0000000..99d61c7 --- /dev/null +++ b/tests/muscle3/wrapped_codes/cpp/parallel_mpi/Makefile @@ -0,0 +1,27 @@ +AL_VERSION ?= $(UAL_VERSION) +AL_MAJOR := $(firstword $(subst ., ,$(AL_VERSION))) + +FLAGS = -pthread -g -O0 -D__USE_XOPEN2K8 -fPIC -DAL_MAJOR=$(AL_MAJOR) + +ifeq ($(AL_MAJOR),) + $(error Could not extract major AL version from UAL_VERSION/AL_VERSION system variable. Is your IMAS module loaded?) +else ifeq ($(AL_MAJOR), 5) + IMAS_LIB_NAME=al-cpp +else ifeq ($(AL_MAJOR), 4) + IMAS_LIB_NAME=imas-cpp +endif + +LIBS = `pkg-config --libs $(IMAS_LIB_NAME)` +INCLUDES = `pkg-config --cflags $(IMAS_LIB_NAME)` + + +all: libparallel_mpi.a + +lib%.a: %.o + @$(AR) -rvcs $@ $^ + +%.o: %.cpp + $(MPICXX) $(FLAGS) -c -o $@ $^ $(INCLUDES) $(LIBS) + +clean: + @$(RM) -f *.o *.a diff --git a/tests/muscle3/wrapped_codes/cpp/parallel_mpi/parallel_mpi.cpp b/tests/muscle3/wrapped_codes/cpp/parallel_mpi/parallel_mpi.cpp new file mode 100644 index 0000000..c946f59 --- /dev/null +++ b/tests/muscle3/wrapped_codes/cpp/parallel_mpi/parallel_mpi.cpp @@ -0,0 +1,154 @@ +#include +#include "parallel_mpi.h" + +int code_state = 0; + +// ======================================= +// INITIALISATION +//======================================= + +void init_code (int& status_code, std::string& status_message) +{ + int mpi_size, mpi_rank; + + status_code = 0; + status_message = "INITIALISATION: OK"; + + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + + std::cout << "=======================================================" << std::endl; + std::cout << "CPP MPI: INITIALISATION called <" << mpi_rank << "/" << mpi_size << ">" << std::endl; + std::cout << "=======================================================" << std::endl; +} + +// ======================================= +// FINALISATION +//======================================= +void clean_up( int& status_code, std::string& status_message) +{ + int mpi_size, mpi_rank; + + status_code = 0; + status_message = "FINALISATION: OK"; + + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + + std::cout << "=======================================================" << std::endl; + std::cout << "CPP MPI: FINALISATION called <" << mpi_rank << "/" << mpi_size << ">" << std::endl; + std::cout << "=======================================================" << std::endl; +} + +// ======================================= +// MAIN +//======================================= +void code_step(const IdsNs::IDS::core_profiles& in_core_profiles, + IdsNs::IDS::distribution_sources& out_distribution_sources, + int& status_code, std::string& status_message) +{ + int mpi_size, mpi_rank; + int idsSize = -1; + + status_code = 0; + status_message = "STEP OK"; + + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + + std::cout << "=======================================" << std::endl; + std::cout << "START OF PHYSICS CODE<" << mpi_rank << "/" << mpi_size << ">" << std::endl; + std::cout << "Starting from: " << code_state << std::endl; + + for (int i = 0; i < 20; i++) + { + // ANY COMPUTATIONS + code_state++; + } + + std::cout << "Counting to : " << code_state << std::endl; + + idsSize = in_core_profiles.time.extent(0); + std::cout << "Size of input IDS: " <" << std::endl; + + if (idsSize > 0) { + out_distribution_sources.time.resize(idsSize); + // Fill in the output IDS (Physical data) + for(int i=0; i < idsSize; i++) + { + // Time : copy from input IDS + out_distribution_sources.time(i) = 1000000 * mpi_rank + 100 * code_state + in_core_profiles.time(i); + } + } + else { + out_distribution_sources.time.resize(1); + out_distribution_sources.time(1) = 1000000 * mpi_rank + 100 * code_state; + } + out_distribution_sources.ids_properties.homogeneous_time = IDS_TIME_MODE_HOMOGENEOUS; + out_distribution_sources.code.name = "cp2ds_mpi_cpp"; + out_distribution_sources.code.version = "1.0"; + out_distribution_sources.code.output_flag = 0 ; + + + std::cout << "END OF PHYSICS CODE" << std::endl; + std::cout << "=======================================" << std::endl; + +} + +// ======================================= +// GET STATE +//======================================= + + +void get_code_state( std::string& state_out, int& status_code, std::string& status_message) +{ + int mpi_size, mpi_rank; + status_code = 0; + + status_message = "INITIALISATION: OK"; + state_out = std::to_string(code_state); + + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + + std::cout << "=======================================================" << std::endl; + std::cout << "CPP MPI: GET STATE called<" << mpi_rank << "/" << mpi_size << ">" << std::endl; + std::cout << "STATE is : " << state_out << std::endl; + std::cout << "=======================================================" << std::endl; +} + +// ======================================= +// SET STATE +//======================================= +void restore_code_state( std::string state, int& status_code, std::string& status_message) +{ + int mpi_size, mpi_rank; + status_code = 0; + status_message = "FINALISATION: OK"; + + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + + code_state = std::stoi( state ); + std::cout << "=======================================================" << std::endl; + std::cout << "CPP MPI: RESTORE STATE called<" << mpi_rank << "/" << mpi_size << ">" << std::endl; + std::cout << "STATE TO BE RESTORED : " << code_state << std::endl; + std::cout << "=======================================================" << std::endl; +} + +// ======================================= +// GET TIMESTAMP +//======================================= +void get_timestamp_cpp(double& timestamp_out, int& status_code, std::string& status_message) +{ + int mpi_size, mpi_rank; + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + + timestamp_out = (double) code_state; + + std::cout << "=======================================================" << std::endl; + std::cout << "CPP MPI: GET TIMESTAMP called<" << mpi_rank << "/" << mpi_size << ">" << std::endl; + std::cout << "TIMESTAMP : " << timestamp_out << std::endl; + std::cout << "=======================================================" << std::endl; +} \ No newline at end of file diff --git a/tests/muscle3/wrapped_codes/cpp/parallel_mpi/parallel_mpi.h b/tests/muscle3/wrapped_codes/cpp/parallel_mpi/parallel_mpi.h new file mode 100644 index 0000000..1154bec --- /dev/null +++ b/tests/muscle3/wrapped_codes/cpp/parallel_mpi/parallel_mpi.h @@ -0,0 +1,27 @@ +#ifndef _PARALLEL_MPI_CPP +#define _PARALLEL_MPI_CPP + +#if 5 == AL_MAJOR + #include "ALClasses.h" +#elif AL_MAJOR == 4 + #include "UALClasses.h" +#else + #warning Could not find AL_MAJOR variable. Assuming AL version = 5.x.x + #include "ALClasses.h" +#endif + +void init_code (int& status_code, std::string& status_message); + +void clean_up( int& status_code, std::string& status_message); + +void code_step(const IdsNs::IDS::core_profiles& core_profiles_in, + IdsNs::IDS::distribution_sources& distribution_sources_out, + int& status_code, std::string& status_message); + +void get_code_state( std::string& state_out, int& status_code, std::string& status_message); + +void restore_code_state( std::string state, int& status_code, std::string& status_message); + +void get_timestamp_cpp(double& timestamp_out, int& status_code, std::string& status_message); + +#endif // _PARALLEL_MPI_CPP \ No newline at end of file diff --git a/tests/muscle3/wrapped_codes/fortran/basic/Makefile b/tests/muscle3/wrapped_codes/fortran/basic/Makefile new file mode 100644 index 0000000..d2979e3 --- /dev/null +++ b/tests/muscle3/wrapped_codes/fortran/basic/Makefile @@ -0,0 +1,30 @@ +AL_VERSION ?= $(UAL_VERSION) +AL_MAJOR := $(firstword $(subst ., ,$(AL_VERSION))) + +FLAGS= -g -fPIC # FPIC AND PREPROCESSING OPTIONS + +ifeq ($(AL_MAJOR),) + $(error Could not extract major AL version from UAL_VERSION/AL_VERSION system variable. Is your IMAS module loaded?) +else ifeq ($(AL_MAJOR), 5) + IMAS_LIB_NAME=al-fortran +else ifeq ($(AL_MAJOR), 4) + IMAS_LIB_NAME=imas-$(FC) +endif + +LIBS = `pkg-config --libs $(IMAS_LIB_NAME)` +INCLUDES = `pkg-config --cflags $(IMAS_LIB_NAME)` + +all: libbasic.a + +lib%.a: %.o + @$(AR) -rcs $@ $^ + +%.o: %.f90 + $(FC) $(FLAGS) -c -o $@ $^ $(INCLUDES) $(LIBS) + +clean: + @$(RM) -f *.exe *.a *.mod *.o *~ + + + + diff --git a/tests/muscle3/wrapped_codes/fortran/basic/basic.f90 b/tests/muscle3/wrapped_codes/fortran/basic/basic.f90 new file mode 100644 index 0000000..8aae928 --- /dev/null +++ b/tests/muscle3/wrapped_codes/fortran/basic/basic.f90 @@ -0,0 +1,381 @@ +module mod_basic + +implicit none + + type type_codeparam_physics_data + integer :: ntimes + double precision:: multiplication_factor + end type type_codeparam_physics_data + + integer :: code_state = 0 + + interface code_step + module procedure codeStepIdsNoParameters, codeStepIdsParameters + end interface code_step + + interface init_code + module procedure initNoIdsNoParameters, initOnlyParameters, initIdsOnly, initIdsParameters + end interface init_code + + interface clean_up + module procedure cleanUpNoIds, cleanUpIds + end interface clean_up + +contains + ! + ! RESTORE_CODE_STATE SUBROUTINE + ! + subroutine restore_code_state (state_str, status_code, status_message) + + implicit none + character(len=:), allocatable, intent(in) :: state_str + integer, intent(out) :: status_code + character(len=:), pointer, intent(out) :: status_message + + + ! Setting status to SUCCESS + status_code = 0 + allocate(character(50):: status_message) + status_message = 'OK' + + read(state_str , *) code_state + write(*,*) '=======================================' + write(*,*) 'Basic: RESTORE STATE called' + write(*,*) 'STATE TO BE RESTORED :', code_state + write(*,*) '=======================================' + + end subroutine restore_code_state + + ! + ! GET_CODE_STATE SUBROUTINE + ! + subroutine get_code_state (state_str, status_code, status_message) + + implicit none + character(len=:), allocatable, intent(out) :: state_str + integer, intent(out) :: status_code + character(len=:), pointer, intent(out) :: status_message + + + ! Setting status to SUCCESS + status_code = 0 + allocate(character(50):: status_message) + status_message = 'OK' + + allocate(character(50):: state_str) + write(state_str,*) code_state + + + + write(*,*) '=======================================' + write(*,*) 'Basic: GET CODE STATE called' + write(*,*) 'STATE is :', state_str + write(*,*) '=======================================' + + end subroutine get_code_state + + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + ! GET TIMESTAMP SUBROUTINE + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + subroutine get_timestamp(timestamp_out, error_flag, error_message) + + integer,parameter :: DP=kind(1.0D0) + real(kind=DP), intent(out) :: timestamp_out + !---- Status info ---- + integer, intent(out) :: error_flag + character(len=:), pointer, intent(out) :: error_message + + error_flag = 0 + timestamp_out = code_state + + end subroutine get_timestamp + + ! + ! INITIALISATION SUBROUTINE + ! + subroutine initNoIdsNoParameters(status_code, status_message) + use ids_schemas, only: ids_parameters_input + implicit none + type(ids_parameters_input) :: xml_parameters + integer, intent(out) :: status_code + character(len=:), pointer, intent(out) :: status_message + + call initOnlyParameters (xml_parameters, status_code, status_message) + + end subroutine initNoIdsNoParameters + + ! + ! INITIALISATION SUBROUTINE + ! + subroutine initOnlyParameters(xml_parameters, status_code, status_message) + use ids_schemas, only: ids_parameters_input, ids_core_profiles, ids_distribution_sources + implicit none + type (ids_core_profiles) :: core_profiles_in + type (ids_distribution_sources) :: distribution_sources_out + type(ids_parameters_input), intent(in) :: xml_parameters + integer, intent(out) :: status_code + character(len=:), pointer, intent(out) :: status_message + + + call initIdsParameters (core_profiles_in, distribution_sources_out, xml_parameters, status_code, status_message) + + end subroutine initOnlyParameters + + ! + ! INITIALISATION SUBROUTINE + ! + subroutine initIdsOnly (core_profiles_in, distribution_sources_out, status_code, status_message) + use ids_schemas, only: ids_parameters_input, ids_core_profiles, ids_distribution_sources + implicit none + type (ids_core_profiles), intent(in) :: core_profiles_in + type (ids_distribution_sources), intent(out) :: distribution_sources_out + type (ids_parameters_input) :: xml_parameters + integer, intent(out) :: status_code + character(len=:), pointer, intent(out) :: status_message + + call initIdsParameters (core_profiles_in, distribution_sources_out, xml_parameters, status_code, status_message) + + end subroutine initIdsOnly + + ! + ! INITIALISATION SUBROUTINE + ! + subroutine initIdsParameters (core_profiles_in, distribution_sources_out, xml_parameters, status_code, status_message) + use ids_schemas, only: ids_parameters_input, ids_core_profiles, ids_distribution_sources + use ids_routines, only: IDS_TIME_MODE_UNKNOWN + implicit none + type (ids_core_profiles), intent(in) :: core_profiles_in + type (ids_distribution_sources), intent(out) :: distribution_sources_out + type (ids_parameters_input), intent(in) :: xml_parameters + integer, intent(inout) :: status_code + character(len=:), pointer, intent(out) :: status_message + integer :: i + + + ! Setting status to SUCCESS + status_code = 0 + allocate(character(50):: status_message) + status_message = 'OK' + + write(*,*) '=======================================' + write(*,*) 'Code lifecycle: INITIALISATION called' + write(*,*) '---------------------------------------' + + + distribution_sources_out%ids_properties%homogeneous_time = 1 + if (core_profiles_in%ids_properties%homogeneous_time /= IDS_TIME_MODE_UNKNOWN) then + + if (associated(core_profiles_in%time)) then + allocate(distribution_sources_out%time(size(core_profiles_in%time))) + + write(0,*) 'Received size of input time from core_profiles_in : ', SIZE(core_profiles_in%time) + + ! Fill in the output IDS (Physical data) + do i=1,size(core_profiles_in%time) + ! Time : copy from input IDS + distribution_sources_out%time(i) = core_profiles_in%time(i) + 1.0 + enddo + + else + allocate(distribution_sources_out%time(1)) + distribution_sources_out%time(1) = 1000 * code_state + end if + + allocate(distribution_sources_out%code%name(1)) ! For a string of 132 characters max. + distribution_sources_out%code%name(1) = 'basic methods: INIT' + allocate(distribution_sources_out%code%output_flag(1)) + distribution_sources_out%code%output_flag(1) = 0 + + end if + + write(*,*) '---------------------------------------' + write(*,*) 'Basic methods: INITIALISATION ends' + write(*,*) '=======================================' + + end subroutine initIdsParameters + + ! + ! FINALISATION SUBROUTINE + ! + subroutine cleanUpNoIds(status_code, status_message) + use ids_schemas, only: ids_core_profiles, ids_distribution_sources + implicit none + + type (ids_distribution_sources) :: distribution_sources_in + type (ids_core_profiles) :: core_profiles_out + + integer, intent(out) :: status_code + character(len=:), pointer, intent(out) :: status_message + + call cleanUpIds(distribution_sources_in, core_profiles_out, status_code, status_message) + + end subroutine cleanUpNoIds + + ! + ! FINALISATION SUBROUTINE + ! + subroutine cleanUpIds(distribution_sources_in, core_profiles_out, status_code, status_message) + use ids_schemas, only: ids_core_profiles, ids_distribution_sources + use ids_routines, only: IDS_TIME_MODE_UNKNOWN + + implicit none + + type (ids_distribution_sources), intent(IN) :: distribution_sources_in + type (ids_core_profiles), intent(OUT) :: core_profiles_out + + integer, intent(out) :: status_code + character(len=:), pointer, intent(out) :: status_message + integer :: i + + + ! Setting status to SUCCESS + status_code = 0 + allocate(character(50):: status_message) + status_message = 'OK' + + write(*,*) '=======================================' + write(*,*) 'Basic methods: FINALISATION called' + write(*,*) '---------------------------------------' + + + + if (distribution_sources_in%ids_properties%homogeneous_time /= IDS_TIME_MODE_UNKNOWN) then + core_profiles_out%ids_properties%homogeneous_time = distribution_sources_in%ids_properties%homogeneous_time + + allocate(core_profiles_out%time(size(distribution_sources_in%time))) + + write(0,*) 'Received size of input time from core_profiles_in : ', SIZE(distribution_sources_in%time) + + ! Fill in the output IDS (Physical data) + do i=1,size(distribution_sources_in%time) + ! Time : copy from input IDS + core_profiles_out%time(i) = distribution_sources_in%time(i) + 1.0 + enddo + + allocate(core_profiles_out%code%name(1)) ! For a string of 132 characters max. + core_profiles_out%code%name(1) = 'basic methods: FINALIZE' + allocate(core_profiles_out%code%output_flag(1)) + core_profiles_out%code%output_flag(1) = 0 + + end if + + write(*,*) '---------------------------------------' + write(*,*) 'Basic methods: FINALISATION ends' + write(*,*) '=======================================' + + end subroutine cleanUpIds + + ! + ! MAIN SUBROUTINE + ! + subroutine codeStepNoIdsNoParameters(status_code, status_message) + use ids_schemas, only: ids_parameters_input + implicit none + + type(ids_parameters_input) :: xml_parameters + integer, intent(out) :: status_code + character(len=:), pointer, intent(out) :: status_message + + call codeStepNoIdsParameters(xml_parameters, status_code, status_message) + end subroutine codeStepNoIdsNoParameters + + ! + ! MAIN SUBROUTINE + ! + subroutine codeStepNoIdsParameters(xml_parameters, status_code, status_message) + use ids_schemas, only: ids_core_profiles, ids_distribution_sources, ids_parameters_input + implicit none + + type(ids_core_profiles):: core_profiles_in + type(ids_distribution_sources):: distribution_sources_out + type(ids_parameters_input) :: xml_parameters + integer, intent(out) :: status_code + character(len=:), pointer, intent(out) :: status_message + + call codeStepIdsParameters(core_profiles_in, distribution_sources_out, xml_parameters, status_code, status_message) + end subroutine codeStepNoIdsParameters + + ! + ! MAIN SUBROUTINE + ! + subroutine codeStepIdsNoParameters(core_profiles_in, distribution_sources_out, status_code, status_message) + use ids_schemas, only: ids_core_profiles, ids_distribution_sources, ids_parameters_input + implicit none + + type(ids_core_profiles):: core_profiles_in + type(ids_distribution_sources):: distribution_sources_out + type(ids_parameters_input) :: xml_parameters + integer, intent(out) :: status_code + character(len=:), pointer, intent(out) :: status_message + + call codeStepIdsParameters(core_profiles_in, distribution_sources_out, xml_parameters, status_code, status_message) + end subroutine codeStepIdsNoParameters + + ! + ! MAIN SUBROUTINE + ! + subroutine codeStepIdsParameters(core_profiles_in, distribution_sources_out, xml_parameters, status_code, status_message) + use ids_schemas, only: ids_core_profiles, ids_distribution_sources, ids_parameters_input + use ids_routines, only: IDS_TIME_MODE_UNKNOWN + + implicit none + + type(ids_core_profiles), intent(IN) :: core_profiles_in + type(ids_distribution_sources), intent(OUT) :: distribution_sources_out + type(ids_parameters_input) :: xml_parameters + integer, intent(out) :: status_code + character(len=:), pointer, intent(out) :: status_message + integer :: i + + ! INITIALISATION OF ERROR FLAG + status_code = 0 + allocate(character(50):: status_message) + + ! INITIAL DISPLAY + write(*,*) '=======================================' + write(*,*) 'START OF PHYSICS CODE' + write(*,*) '---------------------------------------' + + write(*,*) 'Starting from: ', code_state + do i = 1, 20 + ! COMPUTATIONS + code_state = code_state + 1 + end do + + write(*,*) 'Counting to: ', code_state + + ! CHECK IF INPUT IDS IS VALID + if ( core_profiles_in%ids_properties%homogeneous_time /= IDS_TIME_MODE_UNKNOWN & + .and.size(core_profiles_in%time)>0) then + + + ! MANDATORY FLAG (UNIFORM TIME HERE) + distribution_sources_out%ids_properties%homogeneous_time = 1 + allocate(distribution_sources_out%code%name(1)) ! For a string of 132 characters max. + + distribution_sources_out%code%name(1) = 'EXAMPLE: STEP method' + allocate(distribution_sources_out%code%version(1)) ! For a string of 132 characters max. + + distribution_sources_out%code%version(1) = '1.0' + allocate(distribution_sources_out%code%output_flag(1)) + distribution_sources_out%code%output_flag(1) = 0 ! Integer output flag, 0 means the run was successful and can be used in the rest of the workflow, <0 means failure + + ! Fill in the output IDS (Physical data) + allocate(distribution_sources_out%time(size(core_profiles_in%time))) + do i=1,size(core_profiles_in%time) + ! Time : copy from input IDS + distribution_sources_out%time(i) = 1000 * code_state + core_profiles_in%time(i) + enddo + + endif + + status_message = 'STEP: OK' + + ! FINAL DISPLAY + write(*,*) '---------------------------------------' + write(*,*) 'END OF PHYSICS CODE' + write(*,*) '=======================================' + + end subroutine codeStepIdsParameters + +end module mod_basic \ No newline at end of file diff --git a/tests/muscle3/wrapped_codes/fortran/basic_parametrizable/Makefile b/tests/muscle3/wrapped_codes/fortran/basic_parametrizable/Makefile new file mode 100644 index 0000000..a97fcd9 --- /dev/null +++ b/tests/muscle3/wrapped_codes/fortran/basic_parametrizable/Makefile @@ -0,0 +1,18 @@ +FLAGS = -g -fPIC +LIBS = `pkg-config --libs imas-$(FC)` +INCLUDES = `pkg-config --cflags imas-$(FC)` + +all: libbasic_parametrizable.a + +lib%.a: %.o + @$(AR) -rcs $@ $^ + +%.o: %.f90 + $(FC) $(FLAGS) -c -o $@ $^ $(INCLUDES) $(LIBS) + +clean: + @$(RM) -f *.exe *.a *.mod *.o *~ + + + + diff --git a/tests/muscle3/wrapped_codes/fortran/basic_parametrizable/basic.f90 b/tests/muscle3/wrapped_codes/fortran/basic_parametrizable/basic.f90 new file mode 100644 index 0000000..e050d72 --- /dev/null +++ b/tests/muscle3/wrapped_codes/fortran/basic_parametrizable/basic.f90 @@ -0,0 +1,172 @@ +module mod_basic + +integer :: code_state = 0 + +contains + + ! + ! INITIALISATION SUBROUTINE + ! + subroutine init_code (status_code, status_message) + use ids_schemas, only: ids_parameters_input + implicit none + integer, intent(out) :: status_code + character(len=:), pointer, intent(out) :: status_message + + + ! Setting status to SUCCESS + status_code = 0 + allocate(character(50):: status_message) + status_message = 'OK' + + write(*,*) '=======================================' + write(*,*) 'Code lifecycle: INITIALISATION called' + write(*,*) '=======================================' + + end subroutine init_code + + + ! + ! FINALISATION SUBROUTINE + ! + subroutine clean_up(status_code, status_message) + implicit none + integer, intent(out) :: status_code + character(len=:), pointer, intent(out) :: status_message + + ! Setting status to SUCCESS + status_code = 0 + allocate(character(50):: status_message) + status_message = 'OK' + + write(*,*) '=======================================' + write(*,*) 'Code lifecycle: FINALISATION called' + write(*,*) '=======================================' + + end subroutine clean_up + + ! + ! MAIN SUBROUTINE + ! + subroutine code_step(core_profiles_in, distribution_sources_out, status_code, status_message) + use ids_schemas, only: ids_core_profiles, ids_distribution_sources, ids_is_valid + + implicit none + + type(ids_core_profiles):: core_profiles_in + type(ids_distribution_sources):: distribution_sources_out + integer, intent(out) :: status_code + character(len=:), pointer, intent(out) :: status_message + integer :: i + + ! INITIAL DISPLAY + write(*,*) '=======================================' + write(*,*) 'START OF PHYSICS CODE' + + ! INITIALISATION OF ERROR FLAG + status_code = 0 + allocate(character(50):: status_message) + status_message = 'code_restart: OK' + + write(*,*) 'Starting from: ', code_state + + do i = 1, 20 + ! COMPUTATIONS + code_state = code_state + 1 + end do + + write(*,*) 'Counting to: ', code_state + + distribution_sources_out%ids_properties%homogeneous_time = 1 + if (associated(core_profiles_in%time)) then + + allocate(distribution_sources_out%time(size(core_profiles_in%time))) + write(0,*) 'Received size of input time from coreprofilesin : ', SIZE(core_profiles_in%time) + + ! Fill in the output IDS (Physical data) + do i=1,size(core_profiles_in%time) + ! Time : copy from input IDS + distribution_sources_out%time(i) = 1000 * code_state + core_profiles_in%time(i) + ! THE TIME FIELD MUST BE FILLED (MANDATORY) in case of multiple time slice mode for the IDS; + + enddo + else + allocate(distribution_sources_out%time(1)) + distribution_sources_out%time(1) = 1000 * code_state + end if + + ! FINAL DISPLAY + write(*,*) 'END OF PHYSICS CODE' + write(*,*) '=======================================' + write(*,*) ' ' + + end subroutine code_step + + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + ! GET TIMESTAMP SUBROUTINE + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + subroutine get_timestamp(timestamp_out, error_flag, error_message) + + integer,parameter :: DP=kind(1.0D0) + real(kind=DP), intent(out) :: timestamp_out + !---- Status info ---- + integer, intent(out) :: error_flag + character(len=:), pointer, intent(out) :: error_message + + error_flag = 0 + timestamp_out = code_state + + end subroutine get_timestamp + + ! + ! INITIALISATION SUBROUTINE + ! + subroutine get_code_state (state_str, status_code, status_message) + + implicit none + character(len=:), allocatable, intent(out) :: state_str + integer, intent(out) :: status_code + character(len=:), pointer, intent(out) :: status_message + + + ! Setting status to SUCCESS + status_code = 0 + allocate(character(50):: status_message) + status_message = 'OK' + + allocate(character(50):: state_str) + write(state_str,*) code_state + + + + write(*,*) '=======================================' + write(*,*) 'Code lifecycle: GET CODE STATE called' + write(*,*) 'STATE is :', state_str + write(*,*) '=======================================' + + end subroutine get_code_state + + subroutine restore_code_state (state_str, status_code, status_message) + ! + ! INITIALISATION SUBROUTINE + ! + implicit none + character(len=:), allocatable, intent(in) :: state_str + integer, intent(out) :: status_code + character(len=:), pointer, intent(out) :: status_message + + + ! Setting status to SUCCESS + status_code = 0 + allocate(character(50):: status_message) + status_message = 'OK' + + read(state_str , *) code_state + write(*,*) '=======================================' + write(*,*) 'Code lifecycle: RESTORE STATE called' + write(*,*) 'STATE TO BE RESTORED :', code_state + write(*,*) '=======================================' + + end subroutine restore_code_state + +end module mod_basic diff --git a/tests/muscle3/wrapped_codes/fortran/parallel_mpi/Makefile b/tests/muscle3/wrapped_codes/fortran/parallel_mpi/Makefile new file mode 100644 index 0000000..e4ff8fc --- /dev/null +++ b/tests/muscle3/wrapped_codes/fortran/parallel_mpi/Makefile @@ -0,0 +1,27 @@ +AL_VERSION ?= $(UAL_VERSION) +AL_MAJOR := $(firstword $(subst ., ,$(AL_VERSION))) + +FLAGS= -g -fPIC # FPIC AND PREPROCESSING OPTIONS + +ifeq ($(AL_MAJOR),) + $(error Could not extract major AL version from UAL_VERSION/AL_VERSION system variable. Is your IMAS module loaded?) +else ifeq ($(AL_MAJOR), 5) + IMAS_LIB_NAME=al-fortran +else ifeq ($(AL_MAJOR), 4) + IMAS_LIB_NAME=imas-$(FC) +endif + +LIBS = `pkg-config --libs $(IMAS_LIB_NAME)` +INCLUDES = `pkg-config --cflags $(IMAS_LIB_NAME)` + +all: libparallel_mpi.a + +lib%.a: %.o + @$(AR) -rcs $@ $^ + +%.o: %.f90 + $(MPIFC) $(FLAGS) -c -o $@ $^ $(INCLUDES) $(LIBS) + +clean: + @$(RM) -f *.o *.a *.mod *~ + diff --git a/tests/muscle3/wrapped_codes/fortran/parallel_mpi/parallel_mpi.f90 b/tests/muscle3/wrapped_codes/fortran/parallel_mpi/parallel_mpi.f90 new file mode 100644 index 0000000..63062d4 --- /dev/null +++ b/tests/muscle3/wrapped_codes/fortran/parallel_mpi/parallel_mpi.f90 @@ -0,0 +1,213 @@ +module mod_parallel_mpi + + use mpi + use ids_schemas + + integer :: code_state = 0 + +contains + + + ! + ! INITIALISATION SUBROUTINE + ! + subroutine init_code (status_code, status_message) + use ids_schemas, only: ids_parameters_input + implicit none + integer, intent(out) :: status_code + character(len=:), pointer, intent(out) :: status_message + integer :: mpi_world_size, mpi_rank, error + + + ! Setting status to SUCCESS + status_code = 0 + allocate(character(50):: status_message) + status_message = 'OK' + + call MPI_Comm_size ( MPI_COMM_WORLD, mpi_world_size, error ) + call MPI_Comm_rank ( MPI_COMM_WORLD, mpi_rank, error ) + + write(*,*) '=======================================' + write(*,*) 'Parallel MPI:: INITIALISATION called' + write(*,*) 'Process <', mpi_rank, '> out of: ', mpi_world_size + write(*,*) '=======================================' + + end subroutine init_code + + + ! + ! FINALISATION SUBROUTINE + ! + subroutine clean_up(status_code, status_message) + implicit none + integer, intent(out) :: status_code + character(len=:), pointer, intent(out) :: status_message + integer :: mpi_world_size, mpi_rank, error + + ! Setting status to SUCCESS + status_code = 0 + allocate(character(50):: status_message) + status_message = 'OK' + + call MPI_Comm_size ( MPI_COMM_WORLD, mpi_world_size, error ) + call MPI_Comm_rank ( MPI_COMM_WORLD, mpi_rank, error ) + + write(*,*) '=======================================' + write(*,*) 'Parallel MPI: FINALISATION called' + write(*,*) 'Process <', mpi_rank, '> out of: ', mpi_world_size + write(*,*) '=======================================' + + end subroutine clean_up + + subroutine code_step(coreprofilesin, distsourceout, error_flag, error_message) + + implicit none + + integer,parameter :: DP=kind(1.0D0) + + type (ids_core_profiles) :: coreprofilesin + type (ids_distribution_sources) :: distsourceout + integer, intent(out) :: error_flag + character(len=:), pointer, intent(out) :: error_message + + integer :: i + integer :: mpi_world_size, mpi_rank, error + + write(0,*) 'Entering subroutine parallel MPI: code_step' + + ! INITIALISATION OF ERROR FLAG + error_flag = 0 + allocate(character(50):: error_message) + error_message = 'Status info of parallel_mpi' + + call MPI_Comm_size ( MPI_COMM_WORLD, mpi_world_size, error ) + call MPI_Comm_rank ( MPI_COMM_WORLD, mpi_rank, error ) + + ! The output IDS must be allocated with its number of time slices (1 for a single time slice physics module) + ! Here we allocate the output IDS to the same size as the input IDS (but this is not a general rule) + + write(*,*) '=======================================' + write(*,*) 'Parallel MPI: STEP called' + write(*,*) 'Process <', mpi_rank, '> out of: ', mpi_world_size + write(*,*) '=======================================' + + write(*,*) 'Starting from: ', code_state + + do i = 1, 20 + ! COMPUTATIONS + code_state = code_state + 1 + end do + + write(*,*) 'Counting to: ', code_state + + if (associated(coreprofilesin%time)) then + + allocate(distsourceout%time(size(coreprofilesin%time))) + write(0,*) 'Received size of input time from coreprofilesin : ', SIZE(coreprofilesin%time) + + ! Fill in the output IDS (Physical data) + do i=1,size(coreprofilesin%time) + ! Time : copy from input IDS + distsourceout%time(i) = 1000000 * mpi_rank + 100 * code_state + coreprofilesin%time(i) + 1 + ! THE TIME FIELD MUST BE FILLED (MANDATORY) in case of multiple time slice mode for the IDS; + + enddo + else + allocate(distsourceout%time(1)) + distsourceout%time(1) = 1000000 * mpi_rank + 100 * code_state + end if + + + distsourceout%ids_properties%homogeneous_time = 1 + + allocate(distsourceout%code%name(1)) ! For a string of 132 characters max. + distsourceout%code%name(1) = 'parallel_mpi' + allocate(distsourceout%code%version(1)) ! For a string of 132 characters max. + distsourceout%code%version(1) = '1.0' + allocate(distsourceout%code%output_flag(1)) + distsourceout%code%output_flag(1) = 0 ! Integer output flag, 0 means the run was successful and can be used in the rest of the workflow, <0 means failure + + write(*,*) 'END OF PHYSICS CODE' + write(*,*) '=======================================' + + return + end subroutine + + + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + ! GET TIMESTAMP SUBROUTINE + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + subroutine get_timestamp(timestamp_out, error_flag, error_message) + + integer,parameter :: DP=kind(1.0D0) + real(kind=DP), intent(out) :: timestamp_out + !---- Status info ---- + integer, intent(out) :: error_flag + character(len=:), pointer, intent(out) :: error_message + integer :: mpi_world_size, mpi_rank, error + + error_flag = 0 + timestamp_out = code_state + + end subroutine get_timestamp + + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + ! GET STATE SUBROUTINE + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + subroutine get_code_state (state_str, status_code, status_message) + + implicit none + character(len=:), allocatable, intent(out) :: state_str + integer, intent(out) :: status_code + character(len=:), pointer, intent(out) :: status_message + integer :: mpi_world_size, mpi_rank, error + + + ! Setting status to SUCCESS + status_code = 0 + allocate(character(50):: status_message) + status_message = 'OK' + + allocate(character(50):: state_str) + write(state_str,*) code_state + + call MPI_Comm_size ( MPI_COMM_WORLD, mpi_world_size, error ) + call MPI_Comm_rank ( MPI_COMM_WORLD, mpi_rank, error ) + + write(*,*) '=======================================' + write(*,*) 'Parallel MPI:: GET CODE STATE called' + write(*,*) 'Process <', mpi_rank, '> out of: ', mpi_world_size + write(*,*) 'STATE is :', state_str + write(*,*) '=======================================' + + end subroutine get_code_state + + + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + ! SET STATE SUBROUTINE + ! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + subroutine restore_code_state (state_str, status_code, status_message) + + implicit none + character(len=:), allocatable, intent(in) :: state_str + integer, intent(out) :: status_code + character(len=:), pointer, intent(out) :: status_message + integer :: mpi_world_size, mpi_rank, error + + ! Setting status to SUCCESS + status_code = 0 + allocate(character(50):: status_message) + status_message = 'OK' + + call MPI_Comm_size ( MPI_COMM_WORLD, mpi_world_size, error ) + call MPI_Comm_rank ( MPI_COMM_WORLD, mpi_rank, error ) + + read(state_str , *) code_state + write(*,*) '=======================================' + write(*,*) 'Parallel MPI: RESTORE STATE called' + write(*,*) 'STATE TO BE RESTORED :', code_state + write(*,*) 'Process <', mpi_rank, '> out of: ', mpi_world_size + write(*,*) '=======================================' + + end subroutine restore_code_state +end module mod_parallel_mpi diff --git a/tests/muscle3/wrapped_codes/python/basic/Makefile b/tests/muscle3/wrapped_codes/python/basic/Makefile new file mode 100644 index 0000000..3d83c36 --- /dev/null +++ b/tests/muscle3/wrapped_codes/python/basic/Makefile @@ -0,0 +1,7 @@ + +all: + #do nothing + +clean: + #do nothing + diff --git a/tests/muscle3/wrapped_codes/python/basic/basic.py b/tests/muscle3/wrapped_codes/python/basic/basic.py new file mode 100644 index 0000000..12c9daa --- /dev/null +++ b/tests/muscle3/wrapped_codes/python/basic/basic.py @@ -0,0 +1,94 @@ +code_state = 0 + + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# GET CODE STATE +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +def get_code_state(): + state_str = code_state + + print( '=======================================' ) + print( 'Code lifecycle: GET CODE STATE called' ) + print( 'STATE is :', state_str ) + print( '=======================================' ) + + return state_str + + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# SET CODE STATE +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +def restore_code_state(state_str): + global code_state + + code_state = float( state_str ) + print( '=======================================' ) + print( 'Code lifecycle: RESTORE STATE called' ) + print( 'STATE TO BE RESTORED :', code_state ) + print( '=======================================' ) + + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# GET TIMESTAMP +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +def get_timestamp(): + + timestamp_out = code_state + + return timestamp_out + + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# INITIALISATION +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +def init_code (): + + print('=======================================') + print('Code lifecycle: INITIALISATION called') + print('=======================================') + + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# FINALISATION +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +def clean_up(): + + print('=======================================') + print('Code lifecycle: FINALISATION called') + print('=======================================') + + + +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +# MAIN +# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +def code_step(core_profiles_in, distribution_sources_out): + global code_state + print('=======================================') + print('START OF PHYSICS CODE') + print('=======================================') + print('Starting from: ', code_state) + + for i in range (1, 20): + # COMPUTATIONS + code_state = code_state + 1 + + print('Counting to: ', code_state) + + print('=======================================') + + # MANDATORY FLAG (UNIFORM TIME HERE) + distribution_sources_out.ids_properties.homogeneous_time = 1 + + distribution_sources_out.code.name = 'EXAMPLE: code_restart' + distribution_sources_out.code.version = '1.0' + + distribution_sources_out.time.resize(1) + distribution_sources_out.time[0] = code_state + + # FINAL DISPLAY + print('END OF PHYSICS CODE') + print('=======================================') + + + diff --git a/tests/pytest.ini b/tests/pytest.ini new file mode 100644 index 0000000..f46fde8 --- /dev/null +++ b/tests/pytest.ini @@ -0,0 +1,58 @@ +# pytest configuration for iWrap +# This file configures pytest for the iWrap test suite including MUSCLE3 tests + +[pytest] +# Test discovery patterns +python_files = test_*.py *_test.py pytests.py +python_classes = Test* +python_functions = test_* + +# Directories to search for tests +testpaths = tests + +# Markers for categorizing tests +markers = + unit: Unit tests (fast, isolated) + integration: Integration tests (slower, may require external dependencies) + muscle3: Tests requiring MUSCLE3 library + imas: Tests requiring IMAS library + slow: Slow running tests + core: Core iWrap functionality tests + generators: Generator tests + gui: GUI related tests + +# Minimum log level +log_level = INFO + +# Output options +addopts = + -v + --strict-markers + --tb=short + --durations=10 + +# Coverage options (if pytest-cov is installed) +# Uncomment to enable coverage reporting +# addopts = +# --cov=iwrap +# --cov-report=html +# --cov-report=term-missing + +# Ignore directories +norecursedirs = + .git + .tox + dist + build + *.egg + __pycache__ + docs + envs + +# Timeout for tests (in seconds) - requires pytest-timeout +# timeout = 300 + +# Filterwarnings +filterwarnings = + ignore::DeprecationWarning + ignore::PendingDeprecationWarning diff --git a/tests/run-all-tests-master.sh b/tests/run-all-tests-master.sh new file mode 100755 index 0000000..ff1d8a2 --- /dev/null +++ b/tests/run-all-tests-master.sh @@ -0,0 +1,394 @@ +#!/bin/bash +####################################################################################################################### +# Master Test Orchestrator for iWrap +# Runs both pytest-based tests (unit/integration) and IMAS integration tests +####################################################################################################################### + +set -e + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +NC='\033[0m' + +# Default configuration +RUN_UNIT_TESTS=1 +RUN_INTEGRATION_TESTS=1 +RUN_IMAS_TESTS=0 # IMAS tests are optional and slow +RUN_MUSCLE3_TESTS=1 # Run if MUSCLE3 is available +VERBOSE=0 +FAIL_FAST=0 + +# Results tracking +UNIT_TESTS_RESULT=0 +INTEGRATION_TESTS_RESULT=0 +IMAS_TESTS_RESULT=0 +MUSCLE3_TESTS_RESULT=0 + +####################################################################################################################### +# Help Function +####################################################################################################################### +print_help() { + cat << EOF +${BLUE}iWrap Master Test Suite${NC} + +Usage: $0 [OPTIONS] + +Options: + --unit-only Run only unit tests + --integration-only Run only integration tests + --imas Run IMAS integration tests (slow) + --muscle3-only Run only MUSCLE3 tests + --all Run all tests including IMAS + --skip-muscle3 Skip MUSCLE3 tests even if available + --fail-fast Stop on first failure + -v, --verbose Verbose output + -h, --help Show this help message + +Test Types: + Unit Tests: Fast, isolated component tests (pytest) + Integration Tests: Cross-component workflow tests (pytest) + MUSCLE3 Tests: Tests requiring MUSCLE3 library (pytest) + IMAS Tests: Full IMAS integration tests (legacy test suite) + +Examples: + $0 # Run unit + integration tests + $0 --all # Run all tests including IMAS + $0 --unit-only # Run only unit tests + $0 --muscle3-only # Run only MUSCLE3 tests + $0 --imas # Include IMAS tests + $0 --skip-muscle3 # Skip MUSCLE3 tests + +EOF +} + +####################################################################################################################### +# Parse Command Line Arguments +####################################################################################################################### +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + --unit-only) + RUN_INTEGRATION_TESTS=0 + RUN_IMAS_TESTS=0 + shift + ;; + --integration-only) + RUN_UNIT_TESTS=0 + RUN_IMAS_TESTS=0 + shift + ;; + --muscle3-only) + RUN_UNIT_TESTS=0 + RUN_INTEGRATION_TESTS=0 + RUN_IMAS_TESTS=0 + shift + ;; + --imas) + RUN_IMAS_TESTS=1 + shift + ;; + --all) + RUN_UNIT_TESTS=1 + RUN_INTEGRATION_TESTS=1 + RUN_IMAS_TESTS=1 + shift + ;; + --skip-muscle3) + RUN_MUSCLE3_TESTS=0 + shift + ;; + --fail-fast) + FAIL_FAST=1 + shift + ;; + -v|--verbose) + VERBOSE=1 + shift + ;; + -h|--help) + print_help + exit 0 + ;; + *) + echo -e "${RED}Unknown option: $1${NC}" + print_help + exit 1 + ;; + esac + done +} + +####################################################################################################################### +# Environment Setup +####################################################################################################################### +setup_environment() { + echo -e "${BLUE}========================================${NC}" + echo -e "${BLUE}Environment Setup${NC}" + echo -e "${BLUE}========================================${NC}" + + # Set PYTHONPATH + export PYTHONPATH=$(pwd):${PYTHONPATH} + echo -e "PYTHONPATH: ${PYTHONPATH}" + + # Check pytest + if ! command -v pytest &> /dev/null; then + echo -e "${RED}Error: pytest is not installed${NC}" + echo "Install with: pip install pytest" + exit 1 + fi + echo -e "${GREEN}✓ pytest is available${NC}" + + # Check MUSCLE3 + if python3 -c "import muscle3" 2>/dev/null; then + echo -e "${GREEN}✓ MUSCLE3 is available${NC}" + MUSCLE3_AVAILABLE=1 + else + echo -e "${YELLOW}⚠ MUSCLE3 is not available (tests will be skipped)${NC}" + MUSCLE3_AVAILABLE=0 + if [ $RUN_MUSCLE3_TESTS -eq 1 ]; then + RUN_MUSCLE3_TESTS=0 + fi + fi + + echo "" +} + +####################################################################################################################### +# Run Unit Tests +####################################################################################################################### +run_unit_tests() { + if [ $RUN_UNIT_TESTS -eq 0 ]; then + return 0 + fi + + echo -e "${BLUE}========================================${NC}" + echo -e "${BLUE}Running Unit Tests${NC}" + echo -e "${BLUE}========================================${NC}" + + local pytest_args="-v -m unit" + if [ $VERBOSE -eq 1 ]; then + pytest_args="$pytest_args -vv" + fi + + if [ $FAIL_FAST -eq 1 ]; then + pytest_args="$pytest_args -x" + fi + + # Run unit tests, excluding MUSCLE3 if not available + if [ $MUSCLE3_AVAILABLE -eq 0 ]; then + pytest_args="$pytest_args and not muscle3" + fi + + if pytest tests/unit/ $pytest_args 2>&1 | tee /tmp/iwrap_unit_tests.log; then + echo -e "${GREEN}✓ Unit tests PASSED${NC}" + UNIT_TESTS_RESULT=0 + else + echo -e "${RED}✗ Unit tests FAILED${NC}" + UNIT_TESTS_RESULT=1 + if [ $FAIL_FAST -eq 1 ]; then + return 1 + fi + fi + echo "" + + return $UNIT_TESTS_RESULT +} + +####################################################################################################################### +# Run Integration Tests +####################################################################################################################### +run_integration_tests() { + if [ $RUN_INTEGRATION_TESTS -eq 0 ]; then + return 0 + fi + + echo -e "${BLUE}========================================${NC}" + echo -e "${BLUE}Running Integration Tests${NC}" + echo -e "${BLUE}========================================${NC}" + + local pytest_args="-v -m integration" + if [ $VERBOSE -eq 1 ]; then + pytest_args="$pytest_args -vv" + fi + + if [ $FAIL_FAST -eq 1 ]; then + pytest_args="$pytest_args -x" + fi + + # Run integration tests, excluding MUSCLE3 if not available + if [ $MUSCLE3_AVAILABLE -eq 0 ]; then + pytest_args="$pytest_args and not muscle3" + fi + + if pytest tests/integration/ $pytest_args 2>&1 | tee /tmp/iwrap_integration_tests.log; then + echo -e "${GREEN}✓ Integration tests PASSED${NC}" + INTEGRATION_TESTS_RESULT=0 + else + echo -e "${RED}✗ Integration tests FAILED${NC}" + INTEGRATION_TESTS_RESULT=1 + if [ $FAIL_FAST -eq 1 ]; then + return 1 + fi + fi + echo "" + + return $INTEGRATION_TESTS_RESULT +} + +####################################################################################################################### +# Run MUSCLE3 Tests +####################################################################################################################### +run_muscle3_tests() { + if [ $RUN_MUSCLE3_TESTS -eq 0 ] || [ $MUSCLE3_AVAILABLE -eq 0 ]; then + return 0 + fi + + echo -e "${BLUE}========================================${NC}" + echo -e "${BLUE}Running MUSCLE3 Tests${NC}" + echo -e "${BLUE}========================================${NC}" + + local pytest_args="-v -m muscle3" + if [ $VERBOSE -eq 1 ]; then + pytest_args="$pytest_args -vv" + fi + + if [ $FAIL_FAST -eq 1 ]; then + pytest_args="$pytest_args -x" + fi + + if pytest tests/ $pytest_args 2>&1 | tee /tmp/iwrap_muscle3_tests.log; then + echo -e "${GREEN}✓ MUSCLE3 tests PASSED${NC}" + MUSCLE3_TESTS_RESULT=0 + else + echo -e "${RED}✗ MUSCLE3 tests FAILED${NC}" + MUSCLE3_TESTS_RESULT=1 + if [ $FAIL_FAST -eq 1 ]; then + return 1 + fi + fi + echo "" + + return $MUSCLE3_TESTS_RESULT +} + +####################################################################################################################### +# Run IMAS Tests +####################################################################################################################### +run_imas_tests() { + if [ $RUN_IMAS_TESTS -eq 0 ]; then + return 0 + fi + + echo -e "${BLUE}========================================${NC}" + echo -e "${BLUE}Running IMAS Integration Tests${NC}" + echo -e "${BLUE}========================================${NC}" + echo -e "${YELLOW}Note: IMAS tests are slow and require IMAS environment${NC}" + echo "" + + if [ -f "./tests/run-tests.sh" ]; then + if ./tests/run-tests.sh 2>&1 | tee /tmp/iwrap_imas_tests.log; then + echo -e "${GREEN}✓ IMAS tests PASSED${NC}" + IMAS_TESTS_RESULT=0 + else + echo -e "${RED}✗ IMAS tests FAILED${NC}" + IMAS_TESTS_RESULT=1 + fi + else + echo -e "${YELLOW}⚠ IMAS test script not found${NC}" + IMAS_TESTS_RESULT=0 + fi + echo "" + + return $IMAS_TESTS_RESULT +} + +####################################################################################################################### +# Print Summary +####################################################################################################################### +print_summary() { + echo -e "${BLUE}========================================${NC}" + echo -e "${BLUE}Test Summary${NC}" + echo -e "${BLUE}========================================${NC}" + + local total_failed=0 + + if [ $RUN_UNIT_TESTS -eq 1 ]; then + if [ $UNIT_TESTS_RESULT -eq 0 ]; then + echo -e "${GREEN}✓ Unit Tests: PASSED${NC}" + else + echo -e "${RED}✗ Unit Tests: FAILED${NC}" + total_failed=$((total_failed + 1)) + fi + fi + + if [ $RUN_INTEGRATION_TESTS -eq 1 ]; then + if [ $INTEGRATION_TESTS_RESULT -eq 0 ]; then + echo -e "${GREEN}✓ Integration Tests: PASSED${NC}" + else + echo -e "${RED}✗ Integration Tests: FAILED${NC}" + total_failed=$((total_failed + 1)) + fi + fi + + if [ $MUSCLE3_AVAILABLE -eq 1 ] && [ $RUN_MUSCLE3_TESTS -eq 1 ]; then + if [ $MUSCLE3_TESTS_RESULT -eq 0 ]; then + echo -e "${GREEN}✓ MUSCLE3 Tests: PASSED${NC}" + else + echo -e "${RED}✗ MUSCLE3 Tests: FAILED${NC}" + total_failed=$((total_failed + 1)) + fi + elif [ $RUN_MUSCLE3_TESTS -eq 1 ]; then + echo -e "${YELLOW}⚠ MUSCLE3 Tests: SKIPPED (not installed)${NC}" + fi + + if [ $RUN_IMAS_TESTS -eq 1 ]; then + if [ $IMAS_TESTS_RESULT -eq 0 ]; then + echo -e "${GREEN}✓ IMAS Tests: PASSED${NC}" + else + echo -e "${RED}✗ IMAS Tests: FAILED${NC}" + total_failed=$((total_failed + 1)) + fi + fi + + echo -e "${BLUE}========================================${NC}" + + if [ $total_failed -eq 0 ]; then + echo -e "${GREEN}✅ All tests PASSED!${NC}" + return 0 + else + echo -e "${RED}❌ $total_failed test suite(s) FAILED${NC}" + return 1 + fi +} + +####################################################################################################################### +# Main Execution +####################################################################################################################### +main() { + echo -e "${CYAN}" + echo "╔════════════════════════════════════════╗" + echo "║ iWrap Master Test Suite ║" + echo "╚════════════════════════════════════════╝" + echo -e "${NC}" + echo "" + + parse_args "$@" + setup_environment + + # Run tests + run_unit_tests + run_integration_tests + run_muscle3_tests + run_imas_tests + + # Print summary and exit + print_summary + exit $? +} + +# Run main +main "$@" diff --git a/tests/run-all-tests.sh b/tests/run-all-tests.sh new file mode 100755 index 0000000..448e00f --- /dev/null +++ b/tests/run-all-tests.sh @@ -0,0 +1,185 @@ +#!/bin/bash +####################################################################################################################### +# iWrap Test Runner +# Runs all tests including optional MUSCLE3 tests +####################################################################################################################### + +set -e # Exit on error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Test results +CORE_TESTS_PASSED=0 +MUSCLE3_TESTS_PASSED=0 +INTEGRATION_TESTS_PASSED=0 + +echo -e "${BLUE}========================================${NC}" +echo -e "${BLUE}iWrap Test Suite${NC}" +echo -e "${BLUE}========================================${NC}" +echo "" + +# Check if pytest is available +if ! command -v pytest &> /dev/null; then + echo -e "${RED}Error: pytest is not installed${NC}" + echo "Please install pytest: pip install pytest" + exit 1 +fi + +# Check Python path +if [ -z "$PYTHONPATH" ]; then + export PYTHONPATH=$(pwd) +else + export PYTHONPATH=$(pwd):$PYTHONPATH +fi + +echo -e "${BLUE}PYTHONPATH set to: $PYTHONPATH${NC}" +echo "" + +####################################################################################################################### +# Run Core Unit Tests +####################################################################################################################### +echo -e "${BLUE}========================================${NC}" +echo -e "${BLUE}Running Core Unit Tests${NC}" +echo -e "${BLUE}========================================${NC}" + +if pytest tests/unit/core -v -m "not muscle3" 2>&1 | tee /tmp/iwrap_core_tests.log; then + CORE_TESTS_PASSED=1 + echo -e "${GREEN}✓ Core unit tests passed${NC}" +else + echo -e "${RED}✗ Core unit tests failed${NC}" +fi +echo "" + +####################################################################################################################### +# Run Generator Unit Tests (excluding MUSCLE3) +####################################################################################################################### +echo -e "${BLUE}========================================${NC}" +echo -e "${BLUE}Running Generator Unit Tests (Core)${NC}" +echo -e "${BLUE}========================================${NC}" + +if [ -d "tests/unit/generators" ]; then + if pytest tests/unit/generators -v -m "not muscle3" 2>&1 | tee /tmp/iwrap_generator_tests.log; then + echo -e "${GREEN}✓ Core generator tests passed${NC}" + else + echo -e "${YELLOW}⚠ Some core generator tests failed${NC}" + fi +else + echo -e "${YELLOW}⚠ No generator tests directory found${NC}" +fi +echo "" + +####################################################################################################################### +# Check if MUSCLE3 is available +####################################################################################################################### +echo -e "${BLUE}========================================${NC}" +echo -e "${BLUE}Checking MUSCLE3 Availability${NC}" +echo -e "${BLUE}========================================${NC}" + +if python3 -c "import muscle3" 2>/dev/null; then + echo -e "${GREEN}✓ MUSCLE3 library is installed${NC}" + MUSCLE3_AVAILABLE=1 +else + echo -e "${YELLOW}⚠ MUSCLE3 library is NOT installed${NC}" + echo -e "${YELLOW} MUSCLE3 tests will be skipped${NC}" + echo -e "${YELLOW} To run MUSCLE3 tests, install with: pip install iwrap[muscle3]${NC}" + MUSCLE3_AVAILABLE=0 +fi +echo "" + +####################################################################################################################### +# Run MUSCLE3 Tests (if available) +####################################################################################################################### +if [ $MUSCLE3_AVAILABLE -eq 1 ]; then + echo -e "${BLUE}========================================${NC}" + echo -e "${BLUE}Running MUSCLE3 Unit Tests${NC}" + echo -e "${BLUE}========================================${NC}" + + if pytest tests/unit/generators -v -m "muscle3" 2>&1 | tee /tmp/iwrap_muscle3_tests.log; then + MUSCLE3_TESTS_PASSED=1 + echo -e "${GREEN}✓ MUSCLE3 unit tests passed${NC}" + else + echo -e "${RED}✗ MUSCLE3 unit tests failed${NC}" + fi + echo "" +fi + +####################################################################################################################### +# Run Integration Tests +####################################################################################################################### +echo -e "${BLUE}========================================${NC}" +echo -e "${BLUE}Running Integration Tests${NC}" +echo -e "${BLUE}========================================${NC}" + +if [ -d "tests/integration" ]; then + if [ $MUSCLE3_AVAILABLE -eq 1 ]; then + # Run all integration tests + if pytest tests/integration -v 2>&1 | tee /tmp/iwrap_integration_tests.log; then + INTEGRATION_TESTS_PASSED=1 + echo -e "${GREEN}✓ Integration tests passed${NC}" + else + echo -e "${RED}✗ Integration tests failed${NC}" + fi + else + # Run only non-MUSCLE3 integration tests + if pytest tests/integration -v -m "not muscle3" 2>&1 | tee /tmp/iwrap_integration_tests.log; then + INTEGRATION_TESTS_PASSED=1 + echo -e "${GREEN}✓ Core integration tests passed${NC}" + else + echo -e "${RED}✗ Core integration tests failed${NC}" + fi + fi +else + echo -e "${YELLOW}⚠ No integration tests directory found${NC}" +fi +echo "" + +####################################################################################################################### +# Test Summary +####################################################################################################################### +echo -e "${BLUE}========================================${NC}" +echo -e "${BLUE}Test Summary${NC}" +echo -e "${BLUE}========================================${NC}" + +if [ $CORE_TESTS_PASSED -eq 1 ]; then + echo -e "${GREEN}✓ Core Tests: PASSED${NC}" +else + echo -e "${RED}✗ Core Tests: FAILED${NC}" +fi + +if [ $MUSCLE3_AVAILABLE -eq 1 ]; then + if [ $MUSCLE3_TESTS_PASSED -eq 1 ]; then + echo -e "${GREEN}✓ MUSCLE3 Tests: PASSED${NC}" + else + echo -e "${RED}✗ MUSCLE3 Tests: FAILED${NC}" + fi +else + echo -e "${YELLOW}⚠ MUSCLE3 Tests: SKIPPED (MUSCLE3 not installed)${NC}" +fi + +if [ $INTEGRATION_TESTS_PASSED -eq 1 ]; then + echo -e "${GREEN}✓ Integration Tests: PASSED${NC}" +else + echo -e "${RED}✗ Integration Tests: FAILED${NC}" +fi + +echo "" +echo -e "${BLUE}========================================${NC}" + +# Exit with error if any required tests failed +if [ $CORE_TESTS_PASSED -eq 0 ]; then + echo -e "${RED}Test suite FAILED - Core tests failed${NC}" + exit 1 +fi + +if [ $MUSCLE3_AVAILABLE -eq 1 ] && [ $MUSCLE3_TESTS_PASSED -eq 0 ]; then + echo -e "${RED}Test suite FAILED - MUSCLE3 tests failed${NC}" + exit 1 +fi + +echo -e "${GREEN}✅ All tests PASSED!${NC}" +exit 0 diff --git a/tests/run-muscle3-tests.sh b/tests/run-muscle3-tests.sh new file mode 100755 index 0000000..bb2e1ea --- /dev/null +++ b/tests/run-muscle3-tests.sh @@ -0,0 +1,46 @@ +#!/bin/bash +####################################################################################################################### +# MUSCLE3 Test Runner +# Runs only MUSCLE3-specific tests +####################################################################################################################### + +set -e + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +echo -e "${BLUE}========================================${NC}" +echo -e "${BLUE}MUSCLE3 Test Suite${NC}" +echo -e "${BLUE}========================================${NC}" +echo "" + +# Check if MUSCLE3 is installed +if ! python3 -c "import muscle3" 2>/dev/null; then + echo -e "${RED}Error: MUSCLE3 is not installed${NC}" + echo -e "${YELLOW}Install with: pip install iwrap[muscle3]${NC}" + exit 1 +fi + +echo -e "${GREEN}✓ MUSCLE3 is installed${NC}" +echo "" + +# Set PYTHONPATH +export PYTHONPATH=$(pwd):$PYTHONPATH + +# Run MUSCLE3 tests +echo -e "${BLUE}Running MUSCLE3 tests...${NC}" +pytest tests/ -v -m "muscle3" --tb=short + +if [ $? -eq 0 ]; then + echo "" + echo -e "${GREEN}✅ All MUSCLE3 tests PASSED!${NC}" + exit 0 +else + echo "" + echo -e "${RED}❌ MUSCLE3 tests FAILED${NC}" + exit 1 +fi diff --git a/tests/unit/generators/test_muscle3_cpp.py b/tests/unit/generators/test_muscle3_cpp.py new file mode 100644 index 0000000..427dec3 --- /dev/null +++ b/tests/unit/generators/test_muscle3_cpp.py @@ -0,0 +1,61 @@ +""" +Unit tests for MUSCLE3 C++ actor generator +""" +import pytest +import sys +from pathlib import Path + +# Check if MUSCLE3 is available +muscle3_available = pytest.importorskip("muscle3", reason="MUSCLE3 not installed") + + +class TestMuscle3CppGenerator: + """Test suite for MUSCLE3 C++ actor generator""" + + @pytest.fixture(autouse=True) + def setup(self): + """Setup test environment""" + iwrap_root = Path(__file__).parent.parent.parent + if str(iwrap_root) not in sys.path: + sys.path.insert(0, str(iwrap_root)) + + @pytest.mark.muscle3 + @pytest.mark.unit + def test_generator_import(self): + """Test that MUSCLE3 C++ generator can be imported""" + from iwrap.generators.actor_generators.muscle3_cpp.m3_cpp_actor import CppActorGenerator + assert CppActorGenerator is not None + + @pytest.mark.muscle3 + @pytest.mark.unit + def test_generator_instantiation(self): + """Test MUSCLE3 C++ generator instantiation""" + from iwrap.generators.actor_generators.muscle3_cpp.m3_cpp_actor import CppActorGenerator + generator = CppActorGenerator() + assert generator is not None + assert generator.type == 'MUSCLE3-CPP' + assert generator.name == 'MUSCLE3 (C++)' + assert generator.actor_language == 'python' + + @pytest.mark.muscle3 + @pytest.mark.unit + def test_generator_api_version(self): + """Test that generator has correct API version""" + from iwrap.generators.actor_generators.muscle3_cpp.m3_cpp_actor import CppActorGenerator + generator = CppActorGenerator() + assert generator.COMPLIANT_API == '2.1' + + @pytest.mark.muscle3 + @pytest.mark.unit + def test_generator_properties(self): + """Test generator properties""" + from iwrap.generators.actor_generators.muscle3_cpp.m3_cpp_actor import CppActorGenerator + generator = CppActorGenerator() + + assert 'cpp' in generator.code_languages + assert 'legacy' in generator.actor_data_types + assert 'legacy' in generator.code_data_types + + +if __name__ == '__main__': + pytest.main([__file__, '-v']) diff --git a/tests/unit/generators/test_muscle3_fortran.py b/tests/unit/generators/test_muscle3_fortran.py new file mode 100644 index 0000000..f18de82 --- /dev/null +++ b/tests/unit/generators/test_muscle3_fortran.py @@ -0,0 +1,61 @@ +""" +Unit tests for MUSCLE3 Fortran actor generator +""" +import pytest +import sys +from pathlib import Path + +# Check if MUSCLE3 is available +muscle3_available = pytest.importorskip("muscle3", reason="MUSCLE3 not installed") + + +class TestMuscle3FortranGenerator: + """Test suite for MUSCLE3 Fortran actor generator""" + + @pytest.fixture(autouse=True) + def setup(self): + """Setup test environment""" + iwrap_root = Path(__file__).parent.parent.parent + if str(iwrap_root) not in sys.path: + sys.path.insert(0, str(iwrap_root)) + + @pytest.mark.muscle3 + @pytest.mark.unit + def test_generator_import(self): + """Test that MUSCLE3 Fortran generator can be imported""" + from iwrap.generators.actor_generators.muscle3_fortran.m3_fortran_actor import FortranPAFActorGenerator + assert FortranPAFActorGenerator is not None + + @pytest.mark.muscle3 + @pytest.mark.unit + def test_generator_instantiation(self): + """Test MUSCLE3 Fortran generator instantiation""" + from iwrap.generators.actor_generators.muscle3_fortran.m3_fortran_actor import FortranPAFActorGenerator + generator = FortranPAFActorGenerator() + assert generator is not None + assert generator.type == 'MUSCLE3-Fortran' + assert generator.name == 'MUSCLE3 (Fortran)' + assert generator.actor_language == 'python' + + @pytest.mark.muscle3 + @pytest.mark.unit + def test_generator_api_version(self): + """Test that generator has correct API version""" + from iwrap.generators.actor_generators.muscle3_fortran.m3_fortran_actor import FortranPAFActorGenerator + generator = FortranPAFActorGenerator() + assert generator.COMPLIANT_API == '2.1' + + @pytest.mark.muscle3 + @pytest.mark.unit + def test_generator_properties(self): + """Test generator properties""" + from iwrap.generators.actor_generators.muscle3_fortran.m3_fortran_actor import FortranPAFActorGenerator + generator = FortranPAFActorGenerator() + + assert 'fortran' in generator.code_languages + assert 'legacy' in generator.actor_data_types + assert 'legacy' in generator.code_data_types + + +if __name__ == '__main__': + pytest.main([__file__, '-v']) diff --git a/tests/unit/generators/test_muscle3_python.py b/tests/unit/generators/test_muscle3_python.py new file mode 100644 index 0000000..bdf571d --- /dev/null +++ b/tests/unit/generators/test_muscle3_python.py @@ -0,0 +1,137 @@ +""" +Unit tests for MUSCLE3 Python actor generator +""" +import pytest +import sys +from pathlib import Path + +# Check if MUSCLE3 is available +muscle3_available = pytest.importorskip("muscle3", reason="MUSCLE3 not installed") + + +class TestMuscle3PythonGenerator: + """Test suite for MUSCLE3 Python actor generator""" + + @pytest.fixture(autouse=True) + def setup(self): + """Setup test environment""" + # Add iWrap to path if needed + iwrap_root = Path(__file__).parent.parent.parent + if str(iwrap_root) not in sys.path: + sys.path.insert(0, str(iwrap_root)) + + @pytest.mark.muscle3 + @pytest.mark.unit + def test_generator_import(self): + """Test that MUSCLE3 Python generator can be imported""" + from iwrap.generators.actor_generators.muscle3_python.m3_python_actor import PythonActorGenerator + assert PythonActorGenerator is not None + + @pytest.mark.muscle3 + @pytest.mark.unit + def test_generator_instantiation(self): + """Test MUSCLE3 Python generator instantiation""" + from iwrap.generators.actor_generators.muscle3_python.m3_python_actor import PythonActorGenerator + generator = PythonActorGenerator() + assert generator is not None + assert generator.type == 'MUSCLE3-Python' + assert generator.name == 'MUSCLE3 (Python)' + assert generator.actor_language == 'python' + + @pytest.mark.muscle3 + @pytest.mark.unit + def test_generator_api_version(self): + """Test that generator has correct API version""" + from iwrap.generators.actor_generators.muscle3_python.m3_python_actor import PythonActorGenerator + generator = PythonActorGenerator() + assert generator.COMPLIANT_API == '2.1' + + @pytest.mark.muscle3 + @pytest.mark.unit + def test_generator_properties(self): + """Test generator properties""" + from iwrap.generators.actor_generators.muscle3_python.m3_python_actor import PythonActorGenerator + generator = PythonActorGenerator() + + assert 'python' in generator.code_languages + assert 'legacy' in generator.actor_data_types + assert 'legacy' in generator.code_data_types + assert generator.description == 'Wrapping Python code into MUSCLE3 micro model' + + @pytest.mark.muscle3 + @pytest.mark.unit + def test_generator_discovery(self): + """Test that MUSCLE3 generator is discovered by engine""" + from iwrap.generation_engine.engine import Engine + + engine = Engine() + engine.startup() + + generators = engine.registered_generators + generator_types = [g.type for g in generators] + + assert 'MUSCLE3-Python' in generator_types + assert 'MUSCLE3-CPP' in generator_types + assert 'MUSCLE3-Fortran' in generator_types + + +class TestMuscle3CommonUtils: + """Test suite for MUSCLE3 common utilities""" + + @pytest.mark.muscle3 + @pytest.mark.unit + def test_m3_utils_import(self): + """Test that m3_utils can be imported""" + from iwrap.generators.actor_generators.muscle3_common import m3_utils + assert m3_utils is not None + + @pytest.mark.muscle3 + @pytest.mark.unit + def test_template_filter_func(self): + """Test template filter function""" + from iwrap.generators.actor_generators.muscle3_common.m3_utils import template_filter_func + + assert template_filter_func(['normal_file.py']) == True + assert template_filter_func(['__pycache__/file.pyc']) == False + assert template_filter_func(['macros/macro.j2']) == False + + @pytest.mark.muscle3 + @pytest.mark.unit + def test_validate_function(self): + """Test validation function for MUSCLE3 constraints""" + from iwrap.generators.actor_generators.muscle3_common.m3_utils import validate + + # Valid case: no IDS arguments in init/finalize + valid_settings = { + 'code_description': { + 'implementation': { + 'subroutines': { + 'init': {'arguments': []}, + 'finalize': {'arguments': []} + } + } + } + } + + # Should not raise + validate(valid_settings) + + # Invalid case: IDS arguments in init + invalid_settings = { + 'code_description': { + 'implementation': { + 'subroutines': { + 'init': {'arguments': [{'name': 'equilibrium', 'type': 'input'}]}, + 'finalize': {'arguments': []} + } + } + } + } + + # Should raise ValueError + with pytest.raises(ValueError, match='MUSCLE3 actor generator cannot handle INIT/FINALIZE'): + validate(invalid_settings) + + +if __name__ == '__main__': + pytest.main([__file__, '-v']) diff --git a/versioneer.py b/versioneer.py deleted file mode 100644 index 84f29c6..0000000 --- a/versioneer.py +++ /dev/null @@ -1,2220 +0,0 @@ - -# Version: 0.28 - -"""The Versioneer - like a rocketeer, but for versions. - -The Versioneer -============== - -* like a rocketeer, but for versions! -* https://github.com/python-versioneer/python-versioneer -* Brian Warner -* License: Public Domain (Unlicense) -* Compatible with: Python 3.7, 3.8, 3.9, 3.10 and pypy3 -* [![Latest Version][pypi-image]][pypi-url] -* [![Build Status][travis-image]][travis-url] - -This is a tool for managing a recorded version number in setuptools-based -python projects. The goal is to remove the tedious and error-prone "update -the embedded version string" step from your release process. Making a new -release should be as easy as recording a new tag in your version-control -system, and maybe making new tarballs. - - -## Quick Install - -Versioneer provides two installation modes. The "classic" vendored mode installs -a copy of versioneer into your repository. The experimental build-time dependency mode -is intended to allow you to skip this step and simplify the process of upgrading. - -### Vendored mode - -* `pip install versioneer` to somewhere in your $PATH - * A [conda-forge recipe](https://github.com/conda-forge/versioneer-feedstock) is - available, so you can also use `conda install -c conda-forge versioneer` -* add a `[tool.versioneer]` section to your `pyproject.toml` or a - `[versioneer]` section to your `setup.cfg` (see [Install](INSTALL.md)) - * Note that you will need to add `tomli; python_version < "3.11"` to your - build-time dependencies if you use `pyproject.toml` -* run `versioneer install --vendor` in your source tree, commit the results -* verify version information with `python setup.py version` - -### Build-time dependency mode - -* `pip install versioneer` to somewhere in your $PATH - * A [conda-forge recipe](https://github.com/conda-forge/versioneer-feedstock) is - available, so you can also use `conda install -c conda-forge versioneer` -* add a `[tool.versioneer]` section to your `pyproject.toml` or a - `[versioneer]` section to your `setup.cfg` (see [Install](INSTALL.md)) -* add `versioneer` (with `[toml]` extra, if configuring in `pyproject.toml`) - to the `requires` key of the `build-system` table in `pyproject.toml`: - ```toml - [build-system] - requires = ["setuptools", "versioneer[toml]"] - build-backend = "setuptools.build_meta" - ``` -* run `versioneer install --no-vendor` in your source tree, commit the results -* verify version information with `python setup.py version` - -## Version Identifiers - -Source trees come from a variety of places: - -* a version-control system checkout (mostly used by developers) -* a nightly tarball, produced by build automation -* a snapshot tarball, produced by a web-based VCS browser, like github's - "tarball from tag" feature -* a release tarball, produced by "setup.py sdist", distributed through PyPI - -Within each source tree, the version identifier (either a string or a number, -this tool is format-agnostic) can come from a variety of places: - -* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows - about recent "tags" and an absolute revision-id -* the name of the directory into which the tarball was unpacked -* an expanded VCS keyword ($Id$, etc) -* a `_version.py` created by some earlier build step - -For released software, the version identifier is closely related to a VCS -tag. Some projects use tag names that include more than just the version -string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool -needs to strip the tag prefix to extract the version identifier. For -unreleased software (between tags), the version identifier should provide -enough information to help developers recreate the same tree, while also -giving them an idea of roughly how old the tree is (after version 1.2, before -version 1.3). Many VCS systems can report a description that captures this, -for example `git describe --tags --dirty --always` reports things like -"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the -0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has -uncommitted changes). - -The version identifier is used for multiple purposes: - -* to allow the module to self-identify its version: `myproject.__version__` -* to choose a name and prefix for a 'setup.py sdist' tarball - -## Theory of Operation - -Versioneer works by adding a special `_version.py` file into your source -tree, where your `__init__.py` can import it. This `_version.py` knows how to -dynamically ask the VCS tool for version information at import time. - -`_version.py` also contains `$Revision$` markers, and the installation -process marks `_version.py` to have this marker rewritten with a tag name -during the `git archive` command. As a result, generated tarballs will -contain enough information to get the proper version. - -To allow `setup.py` to compute a version too, a `versioneer.py` is added to -the top level of your source tree, next to `setup.py` and the `setup.cfg` -that configures it. This overrides several distutils/setuptools commands to -compute the version when invoked, and changes `setup.py build` and `setup.py -sdist` to replace `_version.py` with a small static file that contains just -the generated version data. - -## Installation - -See [INSTALL.md](./INSTALL.md) for detailed installation instructions. - -## Version-String Flavors - -Code which uses Versioneer can learn about its version string at runtime by -importing `_version` from your main `__init__.py` file and running the -`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can -import the top-level `versioneer.py` and run `get_versions()`. - -Both functions return a dictionary with different flavors of version -information: - -* `['version']`: A condensed version string, rendered using the selected - style. This is the most commonly used value for the project's version - string. The default "pep440" style yields strings like `0.11`, - `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section - below for alternative styles. - -* `['full-revisionid']`: detailed revision identifier. For Git, this is the - full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". - -* `['date']`: Date and time of the latest `HEAD` commit. For Git, it is the - commit date in ISO 8601 format. This will be None if the date is not - available. - -* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that - this is only accurate if run in a VCS checkout, otherwise it is likely to - be False or None - -* `['error']`: if the version string could not be computed, this will be set - to a string describing the problem, otherwise it will be None. It may be - useful to throw an exception in setup.py if this is set, to avoid e.g. - creating tarballs with a version string of "unknown". - -Some variants are more useful than others. Including `full-revisionid` in a -bug report should allow developers to reconstruct the exact code being tested -(or indicate the presence of local changes that should be shared with the -developers). `version` is suitable for display in an "about" box or a CLI -`--version` output: it can be easily compared against release notes and lists -of bugs fixed in various releases. - -The installer adds the following text to your `__init__.py` to place a basic -version in `YOURPROJECT.__version__`: - - from ._version import get_versions - __version__ = get_versions()['version'] - del get_versions - -## Styles - -The setup.cfg `style=` configuration controls how the VCS information is -rendered into a version string. - -The default style, "pep440", produces a PEP440-compliant string, equal to the -un-prefixed tag name for actual releases, and containing an additional "local -version" section with more detail for in-between builds. For Git, this is -TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags ---dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the -tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and -that this commit is two revisions ("+2") beyond the "0.11" tag. For released -software (exactly equal to a known tag), the identifier will only contain the -stripped tag, e.g. "0.11". - -Other styles are available. See [details.md](details.md) in the Versioneer -source tree for descriptions. - -## Debugging - -Versioneer tries to avoid fatal errors: if something goes wrong, it will tend -to return a version of "0+unknown". To investigate the problem, run `setup.py -version`, which will run the version-lookup code in a verbose mode, and will -display the full contents of `get_versions()` (including the `error` string, -which may help identify what went wrong). - -## Known Limitations - -Some situations are known to cause problems for Versioneer. This details the -most significant ones. More can be found on Github -[issues page](https://github.com/python-versioneer/python-versioneer/issues). - -### Subprojects - -Versioneer has limited support for source trees in which `setup.py` is not in -the root directory (e.g. `setup.py` and `.git/` are *not* siblings). The are -two common reasons why `setup.py` might not be in the root: - -* Source trees which contain multiple subprojects, such as - [Buildbot](https://github.com/buildbot/buildbot), which contains both - "master" and "slave" subprojects, each with their own `setup.py`, - `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI - distributions (and upload multiple independently-installable tarballs). -* Source trees whose main purpose is to contain a C library, but which also - provide bindings to Python (and perhaps other languages) in subdirectories. - -Versioneer will look for `.git` in parent directories, and most operations -should get the right version string. However `pip` and `setuptools` have bugs -and implementation details which frequently cause `pip install .` from a -subproject directory to fail to find a correct version string (so it usually -defaults to `0+unknown`). - -`pip install --editable .` should work correctly. `setup.py install` might -work too. - -Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in -some later version. - -[Bug #38](https://github.com/python-versioneer/python-versioneer/issues/38) is tracking -this issue. The discussion in -[PR #61](https://github.com/python-versioneer/python-versioneer/pull/61) describes the -issue from the Versioneer side in more detail. -[pip PR#3176](https://github.com/pypa/pip/pull/3176) and -[pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve -pip to let Versioneer work correctly. - -Versioneer-0.16 and earlier only looked for a `.git` directory next to the -`setup.cfg`, so subprojects were completely unsupported with those releases. - -### Editable installs with setuptools <= 18.5 - -`setup.py develop` and `pip install --editable .` allow you to install a -project into a virtualenv once, then continue editing the source code (and -test) without re-installing after every change. - -"Entry-point scripts" (`setup(entry_points={"console_scripts": ..})`) are a -convenient way to specify executable scripts that should be installed along -with the python package. - -These both work as expected when using modern setuptools. When using -setuptools-18.5 or earlier, however, certain operations will cause -`pkg_resources.DistributionNotFound` errors when running the entrypoint -script, which must be resolved by re-installing the package. This happens -when the install happens with one version, then the egg_info data is -regenerated while a different version is checked out. Many setup.py commands -cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into -a different virtualenv), so this can be surprising. - -[Bug #83](https://github.com/python-versioneer/python-versioneer/issues/83) describes -this one, but upgrading to a newer version of setuptools should probably -resolve it. - - -## Updating Versioneer - -To upgrade your project to a new release of Versioneer, do the following: - -* install the new Versioneer (`pip install -U versioneer` or equivalent) -* edit `setup.cfg` and `pyproject.toml`, if necessary, - to include any new configuration settings indicated by the release notes. - See [UPGRADING](./UPGRADING.md) for details. -* re-run `versioneer install --[no-]vendor` in your source tree, to replace - `SRC/_version.py` -* commit any changed files - -## Future Directions - -This tool is designed to make it easily extended to other version-control -systems: all VCS-specific components are in separate directories like -src/git/ . The top-level `versioneer.py` script is assembled from these -components by running make-versioneer.py . In the future, make-versioneer.py -will take a VCS name as an argument, and will construct a version of -`versioneer.py` that is specific to the given VCS. It might also take the -configuration arguments that are currently provided manually during -installation by editing setup.py . Alternatively, it might go the other -direction and include code from all supported VCS systems, reducing the -number of intermediate scripts. - -## Similar projects - -* [setuptools_scm](https://github.com/pypa/setuptools_scm/) - a non-vendored build-time - dependency -* [minver](https://github.com/jbweston/miniver) - a lightweight reimplementation of - versioneer -* [versioningit](https://github.com/jwodder/versioningit) - a PEP 518-based setuptools - plugin - -## License - -To make Versioneer easier to embed, all its code is dedicated to the public -domain. The `_version.py` that it creates is also in the public domain. -Specifically, both are released under the "Unlicense", as described in -https://unlicense.org/. - -[pypi-image]: https://img.shields.io/pypi/v/versioneer.svg -[pypi-url]: https://pypi.python.org/pypi/versioneer/ -[travis-image]: -https://img.shields.io/travis/com/python-versioneer/python-versioneer.svg -[travis-url]: https://travis-ci.com/github/python-versioneer/python-versioneer - -""" -# pylint:disable=invalid-name,import-outside-toplevel,missing-function-docstring -# pylint:disable=missing-class-docstring,too-many-branches,too-many-statements -# pylint:disable=raise-missing-from,too-many-lines,too-many-locals,import-error -# pylint:disable=too-few-public-methods,redefined-outer-name,consider-using-with -# pylint:disable=attribute-defined-outside-init,too-many-arguments - -import configparser -import errno -import json -import os -import re -import subprocess -import sys -from pathlib import Path -from typing import Callable, Dict -import functools - -have_tomllib = True -if sys.version_info >= (3, 11): - import tomllib -else: - try: - import tomli as tomllib - except ImportError: - have_tomllib = False - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_root(): - """Get the project root directory. - - We require that all commands are run from the project root, i.e. the - directory that contains setup.py, setup.cfg, and versioneer.py . - """ - root = os.path.realpath(os.path.abspath(os.getcwd())) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - # allow 'python path/to/setup.py COMMAND' - root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - err = ("Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND').") - raise VersioneerBadRootError(err) - try: - # Certain runtime workflows (setup.py install/develop in a setuptools - # tree) execute all dependencies in a single python process, so - # "versioneer" may be imported multiple times, and python's shared - # module-import table will cache the first one. So we can't use - # os.path.dirname(__file__), as that will find whichever - # versioneer.py was first imported, even in later projects. - my_path = os.path.realpath(os.path.abspath(__file__)) - me_dir = os.path.normcase(os.path.splitext(my_path)[0]) - vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) - if me_dir != vsr_dir and "VERSIONEER_PEP518" not in globals(): - print("Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(my_path), versioneer_py)) - except NameError: - pass - return root - - -def get_config_from_root(root): - """Read the project setup.cfg file to determine Versioneer config.""" - # This might raise OSError (if setup.cfg is missing), or - # configparser.NoSectionError (if it lacks a [versioneer] section), or - # configparser.NoOptionError (if it lacks "VCS="). See the docstring at - # the top of versioneer.py for instructions on writing your setup.cfg . - root = Path(root) - pyproject_toml = root / "pyproject.toml" - setup_cfg = root / "setup.cfg" - section = None - if pyproject_toml.exists() and have_tomllib: - try: - with open(pyproject_toml, 'rb') as fobj: - pp = tomllib.load(fobj) - section = pp['tool']['versioneer'] - except (tomllib.TOMLDecodeError, KeyError): - pass - if not section: - parser = configparser.ConfigParser() - with open(setup_cfg) as cfg_file: - parser.read_file(cfg_file) - parser.get("versioneer", "VCS") # raise error if missing - - section = parser["versioneer"] - - cfg = VersioneerConfig() - cfg.VCS = section['VCS'] - cfg.style = section.get("style", "") - cfg.versionfile_source = section.get("versionfile_source") - cfg.versionfile_build = section.get("versionfile_build") - cfg.tag_prefix = section.get("tag_prefix") - if cfg.tag_prefix in ("''", '""', None): - cfg.tag_prefix = "" - cfg.parentdir_prefix = section.get("parentdir_prefix") - cfg.verbose = section.get("verbose") - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -# these dictionaries contain VCS-specific tools -LONG_VERSION_PY: Dict[str, str] = {} -HANDLERS: Dict[str, Dict[str, Callable]] = {} - - -def register_vcs_handler(vcs, method): # decorator - """Create decorator to mark a method as the handler of a VCS.""" - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - HANDLERS.setdefault(vcs, {})[method] = f - return f - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - process = None - - popen_kwargs = {} - if sys.platform == "win32": - # This hides the console window if pythonw.exe is used - startupinfo = subprocess.STARTUPINFO() - startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW - popen_kwargs["startupinfo"] = startupinfo - - for command in commands: - try: - dispcmd = str([command] + args) - # remember shell=False, so use git.cmd on windows, not just git - process = subprocess.Popen([command] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None), **popen_kwargs) - break - except OSError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None, None - stdout = process.communicate()[0].strip().decode() - if process.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, process.returncode - return stdout, process.returncode - - -LONG_VERSION_PY['git'] = r''' -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. -# Generated by versioneer-0.28 -# https://github.com/python-versioneer/python-versioneer - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys -from typing import Callable, Dict -import functools - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" - git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" - git_date = "%(DOLLAR)sFormat:%%ci%(DOLLAR)s" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "%(STYLE)s" - cfg.tag_prefix = "%(TAG_PREFIX)s" - cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" - cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY: Dict[str, str] = {} -HANDLERS: Dict[str, Dict[str, Callable]] = {} - - -def register_vcs_handler(vcs, method): # decorator - """Create decorator to mark a method as the handler of a VCS.""" - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - process = None - - popen_kwargs = {} - if sys.platform == "win32": - # This hides the console window if pythonw.exe is used - startupinfo = subprocess.STARTUPINFO() - startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW - popen_kwargs["startupinfo"] = startupinfo - - for command in commands: - try: - dispcmd = str([command] + args) - # remember shell=False, so use git.cmd on windows, not just git - process = subprocess.Popen([command] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None), **popen_kwargs) - break - except OSError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %%s" %% dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %%s" %% (commands,)) - return None, None - stdout = process.communicate()[0].strip().decode() - if process.returncode != 0: - if verbose: - print("unable to run %%s (error)" %% dispcmd) - print("stdout was %%s" %% stdout) - return None, process.returncode - return stdout, process.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for _ in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print("Tried directories %%s but none started with prefix %%s" %% - (str(rootdirs), parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - with open(versionfile_abs, "r") as fobj: - for line in fobj: - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - except OSError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if "refnames" not in keywords: - raise NotThisMethod("Short version file found") - date = keywords.get("date") - if date is not None: - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - - # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = {r.strip() for r in refnames.strip("()").split(",")} - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %%d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = {r for r in refs if re.search(r'\d', r)} - if verbose: - print("discarding '%%s', no digits" %% ",".join(refs - tags)) - if verbose: - print("likely tags: %%s" %% ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] - # Filter out refs that exactly match prefix or that don't start - # with a number once the prefix is stripped (mostly a concern - # when prefix is '') - if not re.match(r'\d', r): - continue - if verbose: - print("picking %%s" %% r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - # GIT_DIR can interfere with correct operation of Versioneer. - # It may be intended to be passed to the Versioneer-versioned project, - # but that should not change where we get our version from. - env = os.environ.copy() - env.pop("GIT_DIR", None) - runner = functools.partial(runner, env=env) - - _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=not verbose) - if rc != 0: - if verbose: - print("Directory %%s not under git control" %% root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = runner(GITS, [ - "describe", "--tags", "--dirty", "--always", "--long", - "--match", f"{tag_prefix}[[:digit:]]*" - ], cwd=root) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], - cwd=root) - # --abbrev-ref was added in git-1.6.3 - if rc != 0 or branch_name is None: - raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") - branch_name = branch_name.strip() - - if branch_name == "HEAD": - # If we aren't exactly on a branch, pick a branch which represents - # the current commit. If all else fails, we are on a branchless - # commit. - branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) - # --contains was added in git-1.5.4 - if rc != 0 or branches is None: - raise NotThisMethod("'git branch --contains' returned error") - branches = branches.split("\n") - - # Remove the first line if we're running detached - if "(" in branches[0]: - branches.pop(0) - - # Strip off the leading "* " from the list of branches. - branches = [branch[2:] for branch in branches] - if "master" in branches: - branch_name = "master" - elif not branches: - branch_name = None - else: - # Pick the first branch that is returned. Good or bad. - branch_name = branches[0] - - pieces["branch"] = branch_name - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) - if not mo: - # unparsable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%%s'" - %% describe_out) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%%s' doesn't start with prefix '%%s'" - print(fmt %% (full_tag, tag_prefix)) - pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" - %% (full_tag, tag_prefix)) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) - pieces["distance"] = len(out.split()) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = runner(GITS, ["show", "-s", "--format=%%ci", "HEAD"], cwd=root)[0].strip() - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_branch(pieces): - """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . - - The ".dev0" means not master branch. Note that .dev0 sorts backwards - (a feature branch will appear "older" than the master branch). - - Exceptions: - 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0" - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+untagged.%%d.g%%s" %% (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def pep440_split_post(ver): - """Split pep440 version string at the post-release segment. - - Returns the release segments before the post-release and the - post-release version number (or -1 if no post-release segment is present). - """ - vc = str.split(ver, ".post") - return vc[0], int(vc[1] or 0) if len(vc) == 2 else None - - -def render_pep440_pre(pieces): - """TAG[.postN.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post0.devDISTANCE - """ - if pieces["closest-tag"]: - if pieces["distance"]: - # update the post release segment - tag_version, post_version = pep440_split_post(pieces["closest-tag"]) - rendered = tag_version - if post_version is not None: - rendered += ".post%%d.dev%%d" %% (post_version + 1, pieces["distance"]) - else: - rendered += ".post0.dev%%d" %% (pieces["distance"]) - else: - # no commits, use the tag as the version - rendered = pieces["closest-tag"] - else: - # exception #1 - rendered = "0.post0.dev%%d" %% pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%%s" %% pieces["short"] - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%%s" %% pieces["short"] - return rendered - - -def render_pep440_post_branch(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . - - The ".dev0" means not master branch. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%%s" %% pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+g%%s" %% pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-branch": - rendered = render_pep440_branch(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-post-branch": - rendered = render_pep440_post_branch(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%%s'" %% style) - - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, - verbose) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for _ in cfg.versionfile_source.split('/'): - root = os.path.dirname(root) - except NameError: - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None} - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", "date": None} -''' - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - with open(versionfile_abs, "r") as fobj: - for line in fobj: - mo_tag_in_git_refnames = None - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - mo_tag_in_git_refnames = re.search(r'(\d+\.)(\d+\.)(\d+)', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_describe_output ="): - if not mo_tag_in_git_refnames: - # print("Tag version not found, Getting from git describe method") - mo_tag_in_git_describe = re.search(r'(\d+\.)(\d+\.)(\d+)', line) - if mo_tag_in_git_describe: - # print('Version found :', mo_tag_in_git_describe.group(0)) - words = line.split("=") - version_string = "" - if len(words)==2: - version_string = words[1].replace('"','').strip() - # print('Version found :', version_string) - mo_describe = '(, tag: '+ version_string +',,'+ keywords["refnames"] + ')' - keywords["refnames"] = mo_describe - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - except OSError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if "refnames" not in keywords: - raise NotThisMethod("Short version file found") - date = keywords.get("date") - if date is not None: - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = {r.strip() for r in refnames.strip("()").split(",")} - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = {r for r in refs if re.search(r'\d', r)} - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] - # Filter out refs that exactly match prefix or that don't start - # with a number once the prefix is stripped (mostly a concern - # when prefix is '') - if not re.match(r'\d', r): - continue - if verbose: - print("picking %s" % r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - # GIT_DIR can interfere with correct operation of Versioneer. - # It may be intended to be passed to the Versioneer-versioned project, - # but that should not change where we get our version from. - env = os.environ.copy() - env.pop("GIT_DIR", None) - runner = functools.partial(runner, env=env) - - _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=not verbose) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = runner(GITS, [ - "describe", "--tags", "--dirty", "--always", "--long", - "--match", f"{tag_prefix}[[:digit:]]*" - ], cwd=root) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], - cwd=root) - # --abbrev-ref was added in git-1.6.3 - if rc != 0 or branch_name is None: - raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") - branch_name = branch_name.strip() - - if branch_name == "HEAD": - # If we aren't exactly on a branch, pick a branch which represents - # the current commit. If all else fails, we are on a branchless - # commit. - branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) - # --contains was added in git-1.5.4 - if rc != 0 or branches is None: - raise NotThisMethod("'git branch --contains' returned error") - branches = branches.split("\n") - - # Remove the first line if we're running detached - if "(" in branches[0]: - branches.pop(0) - - # Strip off the leading "* " from the list of branches. - branches = [branch[2:] for branch in branches] - if "master" in branches: - branch_name = "master" - elif not branches: - branch_name = None - else: - # Pick the first branch that is returned. Good or bad. - branch_name = branches[0] - - pieces["branch"] = branch_name - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) - if not mo: - # unparsable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%s'" - % describe_out) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" - % (full_tag, tag_prefix)) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) - pieces["distance"] = len(out.split()) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip() - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def do_vcs_install(versionfile_source, ipy): - """Git-specific installation logic for Versioneer. - - For Git, this means creating/changing .gitattributes to mark _version.py - for export-subst keyword substitution. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - files = [versionfile_source] - if ipy: - files.append(ipy) - if "VERSIONEER_PEP518" not in globals(): - try: - my_path = __file__ - if my_path.endswith((".pyc", ".pyo")): - my_path = os.path.splitext(my_path)[0] + ".py" - versioneer_file = os.path.relpath(my_path) - except NameError: - versioneer_file = "versioneer.py" - files.append(versioneer_file) - present = False - try: - with open(".gitattributes", "r") as fobj: - for line in fobj: - if line.strip().startswith(versionfile_source): - if "export-subst" in line.strip().split()[1:]: - present = True - break - except OSError: - pass - if not present: - with open(".gitattributes", "a+") as fobj: - fobj.write(f"{versionfile_source} export-subst\n") - files.append(".gitattributes") - run_command(GITS, ["add", "--"] + files) - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for _ in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print("Tried directories %s but none started with prefix %s" % - (str(rootdirs), parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -SHORT_VERSION_PY = """ -# This file was generated by 'versioneer.py' (0.28) from -# revision-control system data, or from the parent directory name of an -# unpacked source archive. Distribution tarballs contain a pre-generated copy -# of this file. - -import json - -version_json = ''' -%s -''' # END VERSION_JSON - - -def get_versions(): - return json.loads(version_json) -""" - - -def versions_from_file(filename): - """Try to determine the version from _version.py if present.""" - try: - with open(filename) as f: - contents = f.read() - except OSError: - raise NotThisMethod("unable to read _version.py") - mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) - if not mo: - mo = re.search(r"version_json = '''\r\n(.*)''' # END VERSION_JSON", - contents, re.M | re.S) - if not mo: - raise NotThisMethod("no version_json in _version.py") - return json.loads(mo.group(1)) - - -def write_to_version_file(filename, versions): - """Write the given version number to the given _version.py file.""" - os.unlink(filename) - contents = json.dumps(versions, sort_keys=True, - indent=1, separators=(",", ": ")) - with open(filename, "w") as f: - f.write(SHORT_VERSION_PY % contents) - - print("set %s to '%s'" % (filename, versions["version"])) - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_branch(pieces): - """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . - - The ".dev0" means not master branch. Note that .dev0 sorts backwards - (a feature branch will appear "older" than the master branch). - - Exceptions: - 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0" - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def pep440_split_post(ver): - """Split pep440 version string at the post-release segment. - - Returns the release segments before the post-release and the - post-release version number (or -1 if no post-release segment is present). - """ - vc = str.split(ver, ".post") - return vc[0], int(vc[1] or 0) if len(vc) == 2 else None - - -def render_pep440_pre(pieces): - """TAG[.postN.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post0.devDISTANCE - """ - if pieces["closest-tag"]: - if pieces["distance"]: - # update the post release segment - tag_version, post_version = pep440_split_post(pieces["closest-tag"]) - rendered = tag_version - if post_version is not None: - rendered += ".post%d.dev%d" % (post_version + 1, pieces["distance"]) - else: - rendered += ".post0.dev%d" % (pieces["distance"]) - else: - # no commits, use the tag as the version - rendered = pieces["closest-tag"] - else: - # exception #1 - rendered = "0.post0.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_post_branch(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . - - The ".dev0" means not master branch. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-branch": - rendered = render_pep440_branch(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-post-branch": - rendered = render_pep440_post_branch(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} - - -class VersioneerBadRootError(Exception): - """The project root directory is unknown or missing key files.""" - - -def get_versions(verbose=False): - """Get the project version from whatever source is available. - - Returns dict with two keys: 'version' and 'full'. - """ - if "versioneer" in sys.modules: - # see the discussion in cmdclass.py:get_cmdclass() - del sys.modules["versioneer"] - - root = get_root() - cfg = get_config_from_root(root) - - assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" - handlers = HANDLERS.get(cfg.VCS) - assert handlers, "unrecognized VCS '%s'" % cfg.VCS - verbose = verbose or cfg.verbose - assert cfg.versionfile_source is not None, \ - "please set versioneer.versionfile_source" - assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" - - versionfile_abs = os.path.join(root, cfg.versionfile_source) - - # extract version from first of: _version.py, VCS command (e.g. 'git - # describe'), parentdir. This is meant to work for developers using a - # source checkout, for users of a tarball created by 'setup.py sdist', - # and for users of a tarball/zipball created by 'git archive' or github's - # download-from-tag feature or the equivalent in other VCSes. - - get_keywords_f = handlers.get("get_keywords") - from_keywords_f = handlers.get("keywords") - if get_keywords_f and from_keywords_f: - try: - keywords = get_keywords_f(versionfile_abs) - ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) - if verbose: - print("got version from expanded keyword %s" % ver) - return ver - except NotThisMethod: - pass - - try: - ver = versions_from_file(versionfile_abs) - if verbose: - print("got version from file %s %s" % (versionfile_abs, ver)) - return ver - except NotThisMethod: - pass - - from_vcs_f = handlers.get("pieces_from_vcs") - if from_vcs_f: - try: - pieces = from_vcs_f(cfg.tag_prefix, root, verbose) - ver = render(pieces, cfg.style) - if verbose: - print("got version from VCS %s" % ver) - return ver - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - if verbose: - print("got version from parentdir %s" % ver) - return ver - except NotThisMethod: - pass - - if verbose: - print("unable to compute version") - - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, "error": "unable to compute version", - "date": None} - - -def get_version(): - """Get the short version string for this project.""" - return get_versions()["version"] - - -def get_cmdclass(cmdclass=None): - """Get the custom setuptools subclasses used by Versioneer. - - If the package uses a different cmdclass (e.g. one from numpy), it - should be provide as an argument. - """ - if "versioneer" in sys.modules: - del sys.modules["versioneer"] - # this fixes the "python setup.py develop" case (also 'install' and - # 'easy_install .'), in which subdependencies of the main project are - # built (using setup.py bdist_egg) in the same python process. Assume - # a main project A and a dependency B, which use different versions - # of Versioneer. A's setup.py imports A's Versioneer, leaving it in - # sys.modules by the time B's setup.py is executed, causing B to run - # with the wrong versioneer. Setuptools wraps the sub-dep builds in a - # sandbox that restores sys.modules to it's pre-build state, so the - # parent is protected against the child's "import versioneer". By - # removing ourselves from sys.modules here, before the child build - # happens, we protect the child from the parent's versioneer too. - # Also see https://github.com/python-versioneer/python-versioneer/issues/52 - - cmds = {} if cmdclass is None else cmdclass.copy() - - # we add "version" to setuptools - from setuptools import Command - - class cmd_version(Command): - description = "report generated version string" - user_options = [] - boolean_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - vers = get_versions(verbose=True) - print("Version: %s" % vers["version"]) - print(" full-revisionid: %s" % vers.get("full-revisionid")) - print(" dirty: %s" % vers.get("dirty")) - print(" date: %s" % vers.get("date")) - if vers["error"]: - print(" error: %s" % vers["error"]) - cmds["version"] = cmd_version - - # we override "build_py" in setuptools - # - # most invocation pathways end up running build_py: - # distutils/build -> build_py - # distutils/install -> distutils/build ->.. - # setuptools/bdist_wheel -> distutils/install ->.. - # setuptools/bdist_egg -> distutils/install_lib -> build_py - # setuptools/install -> bdist_egg ->.. - # setuptools/develop -> ? - # pip install: - # copies source tree to a tempdir before running egg_info/etc - # if .git isn't copied too, 'git describe' will fail - # then does setup.py bdist_wheel, or sometimes setup.py install - # setup.py egg_info -> ? - - # pip install -e . and setuptool/editable_wheel will invoke build_py - # but the build_py command is not expected to copy any files. - - # we override different "build_py" commands for both environments - if 'build_py' in cmds: - _build_py = cmds['build_py'] - else: - from setuptools.command.build_py import build_py as _build_py - - class cmd_build_py(_build_py): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - _build_py.run(self) - if getattr(self, "editable_mode", False): - # During editable installs `.py` and data files are - # not copied to build_lib - return - # now locate _version.py in the new build/ directory and replace - # it with an updated value - if cfg.versionfile_build: - target_versionfile = os.path.join(self.build_lib, - cfg.versionfile_build) - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - cmds["build_py"] = cmd_build_py - - if 'build_ext' in cmds: - _build_ext = cmds['build_ext'] - else: - from setuptools.command.build_ext import build_ext as _build_ext - - class cmd_build_ext(_build_ext): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - _build_ext.run(self) - if self.inplace: - # build_ext --inplace will only build extensions in - # build/lib<..> dir with no _version.py to write to. - # As in place builds will already have a _version.py - # in the module dir, we do not need to write one. - return - # now locate _version.py in the new build/ directory and replace - # it with an updated value - if not cfg.versionfile_build: - return - target_versionfile = os.path.join(self.build_lib, - cfg.versionfile_build) - if not os.path.exists(target_versionfile): - print(f"Warning: {target_versionfile} does not exist, skipping " - "version update. This can happen if you are running build_ext " - "without first running build_py.") - return - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - cmds["build_ext"] = cmd_build_ext - - if "cx_Freeze" in sys.modules: # cx_freeze enabled? - from cx_Freeze.dist import build_exe as _build_exe - # nczeczulin reports that py2exe won't like the pep440-style string - # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. - # setup(console=[{ - # "version": versioneer.get_version().split("+", 1)[0], # FILEVERSION - # "product_version": versioneer.get_version(), - # ... - - class cmd_build_exe(_build_exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _build_exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) - cmds["build_exe"] = cmd_build_exe - del cmds["build_py"] - - if 'py2exe' in sys.modules: # py2exe enabled? - try: - from py2exe.setuptools_buildexe import py2exe as _py2exe - except ImportError: - from py2exe.distutils_buildexe import py2exe as _py2exe - - class cmd_py2exe(_py2exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _py2exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % - {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) - cmds["py2exe"] = cmd_py2exe - - # sdist farms its file list building out to egg_info - if 'egg_info' in cmds: - _egg_info = cmds['egg_info'] - else: - from setuptools.command.egg_info import egg_info as _egg_info - - class cmd_egg_info(_egg_info): - def find_sources(self): - # egg_info.find_sources builds the manifest list and writes it - # in one shot - super().find_sources() - - # Modify the filelist and normalize it - root = get_root() - cfg = get_config_from_root(root) - self.filelist.append('versioneer.py') - if cfg.versionfile_source: - # There are rare cases where versionfile_source might not be - # included by default, so we must be explicit - self.filelist.append(cfg.versionfile_source) - self.filelist.sort() - self.filelist.remove_duplicates() - - # The write method is hidden in the manifest_maker instance that - # generated the filelist and was thrown away - # We will instead replicate their final normalization (to unicode, - # and POSIX-style paths) - from setuptools import unicode_utils - normalized = [unicode_utils.filesys_decode(f).replace(os.sep, '/') - for f in self.filelist.files] - - manifest_filename = os.path.join(self.egg_info, 'SOURCES.txt') - with open(manifest_filename, 'w') as fobj: - fobj.write('\n'.join(normalized)) - - cmds['egg_info'] = cmd_egg_info - - # we override different "sdist" commands for both environments - if 'sdist' in cmds: - _sdist = cmds['sdist'] - else: - from setuptools.command.sdist import sdist as _sdist - - class cmd_sdist(_sdist): - def run(self): - versions = get_versions() - self._versioneer_generated_versions = versions - # unless we update this, the command will keep using the old - # version - self.distribution.metadata.version = versions["version"] - return _sdist.run(self) - - def make_release_tree(self, base_dir, files): - root = get_root() - cfg = get_config_from_root(root) - _sdist.make_release_tree(self, base_dir, files) - # now locate _version.py in the new base_dir directory - # (remembering that it may be a hardlink) and replace it with an - # updated value - target_versionfile = os.path.join(base_dir, cfg.versionfile_source) - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, - self._versioneer_generated_versions) - cmds["sdist"] = cmd_sdist - - return cmds - - -CONFIG_ERROR = """ -setup.cfg is missing the necessary Versioneer configuration. You need -a section like: - - [versioneer] - VCS = git - style = pep440 - versionfile_source = src/myproject/_version.py - versionfile_build = myproject/_version.py - tag_prefix = - parentdir_prefix = myproject- - -You will also need to edit your setup.py to use the results: - - import versioneer - setup(version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), ...) - -Please read the docstring in ./versioneer.py for configuration instructions, -edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. -""" - -SAMPLE_CONFIG = """ -# See the docstring in versioneer.py for instructions. Note that you must -# re-run 'versioneer.py setup' after changing this section, and commit the -# resulting files. - -[versioneer] -#VCS = git -#style = pep440 -#versionfile_source = -#versionfile_build = -#tag_prefix = -#parentdir_prefix = - -""" - -OLD_SNIPPET = """ -from ._version import get_versions -__version__ = get_versions()['version'] -del get_versions -""" - -INIT_PY_SNIPPET = """ -from . import {0} -__version__ = {0}.get_versions()['version'] -""" - - -def do_setup(): - """Do main VCS-independent setup function for installing Versioneer.""" - root = get_root() - try: - cfg = get_config_from_root(root) - except (OSError, configparser.NoSectionError, - configparser.NoOptionError) as e: - if isinstance(e, (OSError, configparser.NoSectionError)): - print("Adding sample versioneer config to setup.cfg", - file=sys.stderr) - with open(os.path.join(root, "setup.cfg"), "a") as f: - f.write(SAMPLE_CONFIG) - print(CONFIG_ERROR, file=sys.stderr) - return 1 - - print(" creating %s" % cfg.versionfile_source) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write(LONG % {"DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - }) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), - "__init__.py") - if os.path.exists(ipy): - try: - with open(ipy, "r") as f: - old = f.read() - except OSError: - old = "" - module = os.path.splitext(os.path.basename(cfg.versionfile_source))[0] - snippet = INIT_PY_SNIPPET.format(module) - if OLD_SNIPPET in old: - print(" replacing boilerplate in %s" % ipy) - with open(ipy, "w") as f: - f.write(old.replace(OLD_SNIPPET, snippet)) - elif snippet not in old: - print(" appending to %s" % ipy) - with open(ipy, "a") as f: - f.write(snippet) - else: - print(" %s unmodified" % ipy) - else: - print(" %s doesn't exist, ok" % ipy) - ipy = None - - # Make VCS-specific changes. For git, this means creating/changing - # .gitattributes to mark _version.py for export-subst keyword - # substitution. - do_vcs_install(cfg.versionfile_source, ipy) - return 0 - - -def scan_setup_py(): - """Validate the contents of setup.py against Versioneer's expectations.""" - found = set() - setters = False - errors = 0 - with open("setup.py", "r") as f: - for line in f.readlines(): - if "import versioneer" in line: - found.add("import") - if "versioneer.get_cmdclass()" in line: - found.add("cmdclass") - if "versioneer.get_version()" in line: - found.add("get_version") - if "versioneer.VCS" in line: - setters = True - if "versioneer.versionfile_source" in line: - setters = True - if len(found) != 3: - print("") - print("Your setup.py appears to be missing some important items") - print("(but I might be wrong). Please make sure it has something") - print("roughly like the following:") - print("") - print(" import versioneer") - print(" setup( version=versioneer.get_version(),") - print(" cmdclass=versioneer.get_cmdclass(), ...)") - print("") - errors += 1 - if setters: - print("You should remove lines like 'versioneer.VCS = ' and") - print("'versioneer.versionfile_source = ' . This configuration") - print("now lives in setup.cfg, and should be removed from setup.py") - print("") - errors += 1 - return errors - - -def setup_command(): - """Set up Versioneer and exit with appropriate error code.""" - errors = do_setup() - errors += scan_setup_py() - sys.exit(1 if errors else 0) - - -if __name__ == "__main__": - cmd = sys.argv[1] - if cmd == "setup": - setup_command()