diff --git a/bindings/python/dlite-path-python.i b/bindings/python/dlite-path-python.i index db5f296eb..2d422d852 100644 --- a/bindings/python/dlite-path-python.i +++ b/bindings/python/dlite-path-python.i @@ -55,6 +55,7 @@ def _create_path(name): # Create DLite search paths objects storage_path = _create_path("storages") +template_path = _create_path("templates") storage_plugin_path = _create_path("storage-plugins") mapping_plugin_path = _create_path("mapping-plugins") python_storage_plugin_path = _create_path("python-storage-plugins") diff --git a/bindings/python/dlite-path.i b/bindings/python/dlite-path.i index 2b4f51a92..db65a2050 100644 --- a/bindings/python/dlite-path.i +++ b/bindings/python/dlite-path.i @@ -24,6 +24,7 @@ //#include "utils/compat.h" #include "utils/fileutils.h" //#include "dlite-macros.h" +#include "dlite-codegen.h" #include "pyembed/dlite-python-storage.h" #include "pyembed/dlite-python-mapping.h" #include "pyembed/dlite-python-protocol.h" @@ -49,6 +50,8 @@ Creates a _Path instance of type `pathtype`. _FUPaths(const char *pathtype) { if (strcmp(pathtype, "storages") == 0) { return dlite_storage_paths(); + } else if (strcmp(pathtype, "templates") == 0) { + return dlite_codegen_path_get(); } else if (strcmp(pathtype, "storage-plugins") == 0) { return dlite_storage_plugin_paths_get(); } else if (strcmp(pathtype, "mapping-plugins") == 0) { diff --git a/doc/user_guide/index.rst b/doc/user_guide/index.rst index c6ef982c6..66f704ff1 100644 --- a/doc/user_guide/index.rst +++ b/doc/user_guide/index.rst @@ -12,6 +12,7 @@ User Guide storage_plugins storage_plugins_mongodb protocol_plugins + search_paths mappings transactions tools diff --git a/doc/user_guide/search_paths.md b/doc/user_guide/search_paths.md new file mode 100644 index 000000000..2658c2f5d --- /dev/null +++ b/doc/user_guide/search_paths.md @@ -0,0 +1,103 @@ +Search paths +============ +It is possible to extend DLite with new datamodels, code generation templates and plugins, by appending to corresponding search path. The table below lists the different types of search paths that are available in DLite. + +| Search path type | Description | +|-------------------------|-------------------------------------------------------| +| storages | Storage URLs or directory paths to datamodels | +| templates | Directory paths to code generator templates | +| storage_plugins | Directory paths to storage plugins written in C | +| mapping-plugins | Directory paths to mapping plugins written in C | +| python-storage-plugins | Directory paths to storage plugins written in Python | +| python-mapping-plugins | Directory paths to mapping plugins written in Python | +| python-protocol-plugins | Directory paths to protocol plugins written in Python | + +Search paths can be extended in different three ways: +* setting [environment variables] +* appending to DLite path variables (from Python) +* using [entry points] (only in user-defined Python packages) + +The table below lists the lists the name of the environment variables and Python path variables corresponding to the different types of search paths. + +| Search path type | Environment variable name | Python variable name | +|-------------------------|-----------------------------------|-----------------------------------| +| storages | DLITE_STORAGES | dlite.storage_path | +| templates | DLITE_TEMPLATE_DIRS | dlite.template_path | +| storage_plugins | DLITE_STORAGE_PLUGIN_DIRS | dlite.storage_plugin_path | +| mapping-plugins | DLITE_MAPPING_PLUGIN_DIRS | dlite.mapping-plugin_path | +| python-storage-plugins | DLITE_PYTHON_STORAGE_PLUGIN_DIRS | dlite.python-storage-plugin_path | +| python-mapping-plugins | DLITE_PYTHON_MAPPING_PLUGIN_DIRS | dlite.python-mapping-plugin_path | +| python-protocol-plugins | DLITE_PYTHON_PROTOCOL_PLUGIN_DIRS | dlite.python-protocol-plugin_path | + + +Setting environment variables +----------------------------- +This is typically done where you set up your environment, like in a virtualenv activate script or the users `~/.bashrc` file. + +:::{note} +All the path variables, except for `DLITE_STORAGES`, uses (`:`) colon as path separator. +However, since colon may appears in URLs, `DLITE_STORAGES` uses instead the pipe symbol (`|`) as path separator. +::: + +See [environment variables] for more details. + + +Appending to DLite path variables +--------------------------------- +The `dlite` Python module defines the path variables listed in the table above. +A Python script or application can configure new datamodels and plugins by appending to these variables. + +:::{example} +Adding the sub-directory `datamodels` to the search path for datamodels, can be done with: + +```python +>>> from pathlib import Path +>>> import dlite +>>> dlite.storage_path.append(Path("my_package_root") / "datamodels") +``` +::: + + +Using entry points +------------------ +How to use entry points is easiest described with an example. +Lets assume you have a package with the following directory layout: + +``` +project_root +├── mypackage +│ ├── __init__.py +│ ├── mymodule.py +│ ├── data +│ │ ├── python_storage_plugins +│ │ │ ├── myplugin.py +│ │ │ └── anotherplugin.py +│ │ └── datamodels +│ │ ├── mydatamodel.yaml +│ │ └── anotherdatamodel.yaml +├── pyproject.toml +├── README.md +└── LICENSE +``` + +To make your datamodels and Python storage plugins available for users of your package, you can add the following section to your `pyproject.toml`: + +```toml +[tool.setuptools.package-data] +"mypackages.data.datamodels" = ["*.json", "*.yaml"] +"mypackages.data.python_storage_plugins" = ["*.py"] + +# Note the quotes around dlite.python_storage_plugins to escape the embedded dot +[project.entry-points."dlite.python_storage_plugins"] +storage_plugin_path = "mypackage:data/python_storage_plugins" + +[project.entry-points."dlite.storages"] +datamodel_path = "mypackage:data/datamodels" +``` + +See the [Setuptools documentation] for how to this can be done with `setup.py` or `setup.cfg`. + + +[environment variables]: https://sintef.github.io/dlite/user_guide/environment_variables.html +[entry points]: https://setuptools.pypa.io/en/latest/userguide/entry_point.html +[Setuptools documentation]: https://setuptools.pypa.io/en/latest/userguide/index.html diff --git a/doc/user_guide/storage_plugins.md b/doc/user_guide/storage_plugins.md index 1734fadf1..194406590 100644 --- a/doc/user_guide/storage_plugins.md +++ b/doc/user_guide/storage_plugins.md @@ -222,6 +222,21 @@ This could lead to confusing and hard-to-find bugs due to interference between y However, since DLite v0.5.23, plugins are evaluated in separate scopes (which are available in `dlite._plugindict). +### Distributing storage plugins +If you created a python package with new storage plugins, you can make them +available for users of your package by creating entry points with the paths to the storage plugins. + +Add the following section to your `pyproject.toml`: + +```toml +[project.entry-points.dlite] +python_storage_plugins = "path/to/python_storage_plugins/directory" + +``` + +See [Extending DLite] for more information. + + Working with storages from C and Fortran ---------------------------------------- diff --git a/src/dlite-codegen.h b/src/dlite-codegen.h index 2e704393e..6b450c660 100644 --- a/src/dlite-codegen.h +++ b/src/dlite-codegen.h @@ -10,6 +10,7 @@ typically is code that will be written to file and compiled. */ +#include "utils/tgen.h" #include "utils/fileutils.h"