diff --git a/bindings/python/dlite-path-python.i b/bindings/python/dlite-path-python.i index 8b900a62d..2d422d852 100644 --- a/bindings/python/dlite-path-python.i +++ b/bindings/python/dlite-path-python.i @@ -37,12 +37,31 @@ %pythoncode %{ -storage_path = FUPath("storages") -storage_plugin_path = FUPath("storage-plugins") -mapping_plugin_path = FUPath("mapping-plugins") -python_storage_plugin_path = FUPath("python-storage-plugins") -python_mapping_plugin_path = FUPath("python-mapping-plugins") -python_protocol_plugin_path = FUPath("python-protocol-plugins") +import sys +from importlib.metadata import entry_points + +def _create_path(name): + """Return new DLite search path object, with given name.""" + if sys.version_info < (3, 10): # Fallback for Python < 3.10 + eps = entry_points().get(f"dlite.{name}", ()) + else: # For Python 3.10+ + eps = entry_points(group=f"dlite.{name}") + + path = FUPath(name) + path.name = name + for entry_point in eps: + path.append(entry_point.value) + return path + +# Create DLite search paths objects +storage_path = _create_path("storages") +template_path = _create_path("templates") +storage_plugin_path = _create_path("storage-plugins") +mapping_plugin_path = _create_path("mapping-plugins") +python_storage_plugin_path = _create_path("python-storage-plugins") +python_mapping_plugin_path = _create_path("python-mapping-plugins") +python_protocol_plugin_path = _create_path("python-protocol-plugins") + # Update default search paths from pathlib import Path diff --git a/bindings/python/dlite-path.i b/bindings/python/dlite-path.i index 2b4f51a92..db65a2050 100644 --- a/bindings/python/dlite-path.i +++ b/bindings/python/dlite-path.i @@ -24,6 +24,7 @@ //#include "utils/compat.h" #include "utils/fileutils.h" //#include "dlite-macros.h" +#include "dlite-codegen.h" #include "pyembed/dlite-python-storage.h" #include "pyembed/dlite-python-mapping.h" #include "pyembed/dlite-python-protocol.h" @@ -49,6 +50,8 @@ Creates a _Path instance of type `pathtype`. _FUPaths(const char *pathtype) { if (strcmp(pathtype, "storages") == 0) { return dlite_storage_paths(); + } else if (strcmp(pathtype, "templates") == 0) { + return dlite_codegen_path_get(); } else if (strcmp(pathtype, "storage-plugins") == 0) { return dlite_storage_plugin_paths_get(); } else if (strcmp(pathtype, "mapping-plugins") == 0) { diff --git a/doc/user_guide/index.rst b/doc/user_guide/index.rst index 35a4fdef2..7ea392630 100644 --- a/doc/user_guide/index.rst +++ b/doc/user_guide/index.rst @@ -13,6 +13,7 @@ User Guide storage_plugins storage_plugins_mongodb protocol_plugins + search_paths mappings transactions tools diff --git a/doc/user_guide/search_paths.md b/doc/user_guide/search_paths.md new file mode 100644 index 000000000..5f5617e8f --- /dev/null +++ b/doc/user_guide/search_paths.md @@ -0,0 +1,103 @@ +Search paths +============ +It is possible to extend DLite with new datamodels, code generation templates and plugins, by appending to corresponding search path. The table below lists the different types of search paths that are available in DLite. + +| Search path type | Description | +|-------------------------|----------------------------------------------------------------| +| storages | Storage URLs or directory paths to datamodels | +| templates | Directory paths to code generator templates | +| storage_plugins | Directory paths to storage plugins (drivers) written in C | +| mapping-plugins | Directory paths to mapping plugins written in C | +| python-storage-plugins | Directory paths to storage plugins (drivers) written in Python | +| python-mapping-plugins | Directory paths to mapping plugins written in Python | +| python-protocol-plugins | Directory paths to protocol plugins written in Python | + +Search paths can be extended in different three ways: +* setting [environment variables] +* appending to DLite path variables (from Python) +* using [entry points] (only in user-defined Python packages) + +The table below lists the lists the name of the environment variables and Python path variables corresponding to the different types of search paths. + +| Search path type | Environment variable name | Python variable name | +|-------------------------|-----------------------------------|-----------------------------------| +| storages | DLITE_STORAGES | dlite.storage_path | +| templates | DLITE_TEMPLATE_DIRS | dlite.template_path | +| storage_plugins | DLITE_STORAGE_PLUGIN_DIRS | dlite.storage_plugin_path | +| mapping-plugins | DLITE_MAPPING_PLUGIN_DIRS | dlite.mapping-plugin_path | +| python-storage-plugins | DLITE_PYTHON_STORAGE_PLUGIN_DIRS | dlite.python-storage-plugin_path | +| python-mapping-plugins | DLITE_PYTHON_MAPPING_PLUGIN_DIRS | dlite.python-mapping-plugin_path | +| python-protocol-plugins | DLITE_PYTHON_PROTOCOL_PLUGIN_DIRS | dlite.python-protocol-plugin_path | + + +Setting environment variables +----------------------------- +This is typically done where you set up your environment, like in a virtualenv activate script or the users `~/.bashrc` file. + +:::{note} +All the path variables, except for `DLITE_STORAGES`, uses (`:`) colon as path separator. +However, since colon may appears in URLs, `DLITE_STORAGES` uses instead the pipe symbol (`|`) as path separator. +::: + +See [environment variables] for more details. + + +Appending to DLite path variables +--------------------------------- +The `dlite` Python module defines the path variables listed in the table above. +A Python script or application can configure new datamodels and plugins by appending to these variables. + +:::{example} +Adding the sub-directory `datamodels` to the search path for datamodels, can be done with: + +```python +>>> from pathlib import Path +>>> import dlite +>>> dlite.storage_path.append(Path("my_package_root") / "datamodels") +``` +::: + + +Using entry points +------------------ +How to use entry points is easiest described with an example. +Let us assume you have a package with the following directory layout: + +``` +project_root +├── mypackage +│ ├── __init__.py +│ ├── mymodule.py +│ ├── data +│ │ ├── python_storage_plugins +│ │ │ ├── myplugin.py +│ │ │ └── anotherplugin.py +│ │ └── datamodels +│ │ ├── mydatamodel.yaml +│ │ └── anotherdatamodel.yaml +├── pyproject.toml +├── README.md +└── LICENSE +``` + +To make your datamodels and Python storage plugins available for users of your package, you can add the following section to your `pyproject.toml`: + +```toml +[tool.setuptools.package-data] +"mypackages.data.datamodels" = ["*.json", "*.yaml"] +"mypackages.data.python_storage_plugins" = ["*.py"] + +# Note the quotes around dlite.python_storage_plugins to escape the embedded dot +[project.entry-points."dlite.python_storage_plugins"] +storage_plugin_path = "mypackage:data/python_storage_plugins" + +[project.entry-points."dlite.storages"] +datamodel_path = "mypackage:data/datamodels" +``` + +See the [Setuptools documentation] for how to this can be done with `setup.py` or `setup.cfg`. + + +[environment variables]: https://sintef.github.io/dlite/user_guide/environment_variables.html +[entry points]: https://setuptools.pypa.io/en/latest/userguide/entry_point.html +[Setuptools documentation]: https://setuptools.pypa.io/en/latest/userguide/index.html diff --git a/doc/user_guide/storage_plugins.md b/doc/user_guide/storage_plugins.md index c45d147f0..eb422c72f 100644 --- a/doc/user_guide/storage_plugins.md +++ b/doc/user_guide/storage_plugins.md @@ -45,15 +45,15 @@ import dlite dlite.python_storage_plugin_path.append("/path/to/plugins/dir") ``` -Often drivers are connected to very specific datamodel (entities). +Often drivers are connected to very specific datamodel (entities). DLite will find these datamodels if the path to their directory is set with the environment variable `DLITE_STORAGES` or added within python with `dlite.storage_path.append` similarly to described above for drivers. ```{attention} -Often, during development dlite will fail unexpectedly. This is typically either because of an error in the -datamodel or the driver. -The variable DLITE_PYDEBUG can be set as `export DLITE_PYDEBUG=` to get python debugging information. +Often, during development dlite will fail unexpectedly. This is typically either because of an error in the +datamodel or the driver. +The variable DLITE_PYDEBUG can be set as `export DLITE_PYDEBUG=` to get python debugging information. This will give information about the driver. It is advisable to first check that the datamodel is valid with the command `dlite-validate datamodelfilename`. ``` @@ -252,6 +252,21 @@ This could lead to confusing and hard-to-find bugs due to interference between y However, since DLite v0.5.23, plugins are evaluated in separate scopes (which are available in `dlite._plugindict). +### Distributing storage plugins +If you created a python package with new storage plugins, you can make them +available for users of your package by creating entry points with the paths to the storage plugins. + +Add the following section to your `pyproject.toml`: + +```toml +[project.entry-points.dlite] +python_storage_plugins = "path/to/python_storage_plugins/directory" + +``` + +See [Search paths] for more information about entry points. + + Working with storages from C and Fortran ---------------------------------------- @@ -271,3 +286,4 @@ An example is available in [ex4]. [Python storage plugin example]: https://github.com/SINTEF/dlite/tree/master/examples/storage_plugin [ex1]: https://github.com/SINTEF/dlite/tree/master/examples/ex1 [ex4]: https://github.com/SINTEF/dlite/tree/master/examples/ex4 +[Search paths]: https://sintef.github.io/dlite/user_guide/search_paths.html diff --git a/src/dlite-codegen.h b/src/dlite-codegen.h index 2e704393e..6b450c660 100644 --- a/src/dlite-codegen.h +++ b/src/dlite-codegen.h @@ -10,6 +10,7 @@ typically is code that will be written to file and compiled. */ +#include "utils/tgen.h" #include "utils/fileutils.h"