From b694128d6b57657d7e1daeb715050857ef58edd0 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Sun, 13 Aug 2023 22:51:48 +0200 Subject: [PATCH 1/5] Added a test for the http plugin. Also cleaned up the plugin and improved documentation. --- bindings/python/dlite-entity-python.i | 24 ++++++++++- bindings/python/utils.py | 3 +- .../python/python-storage-plugins/http.py | 42 +++++++++++++------ 3 files changed, 55 insertions(+), 14 deletions(-) diff --git a/bindings/python/dlite-entity-python.i b/bindings/python/dlite-entity-python.i index 8cae90479..0ede8394e 100644 --- a/bindings/python/dlite-entity-python.i +++ b/bindings/python/dlite-entity-python.i @@ -384,7 +384,29 @@ def get_instance(id: str, metaid: str = None, check_storages: bool = True) -> "I @classmethod def from_dict(cls, d, id=None, single=None, check_storages=True): - """Load the instance from dictionary.""" + """Load the instance from dictionary. + + Arguments: + d: Dict to parse. It should be of the same form as returned + by the Instance.asdict() method. + id: Identity of the returned instance. + + If `d` is in single-entity form with no explicit 'uuid' or + 'uri', its identity will be assigned by `id`. Otherwise + `id` must be consistent with the 'uuid' and/or 'uri' + fields of `d`. + + If `d` is in multi-entity form, `id` is used to select the + instance to return. + single: A boolean, None or "auto". Determines whether to + assume that the dict is in single-entity form. + If `single` is None or "auto", the form is inferred. + check_storages: Whether to check if the instance already exists + in storages specified in `dlite.storage_path`. + + Returns: + New instance. + """ from dlite.utils import instance_from_dict return instance_from_dict( d, id=id, single=single, check_storages=check_storages, diff --git a/bindings/python/utils.py b/bindings/python/utils.py index c5f45aa28..24e547301 100644 --- a/bindings/python/utils.py +++ b/bindings/python/utils.py @@ -81,7 +81,8 @@ def instance_from_dict(d, id=None, single=None, check_storages=True): Whether the dict is assumed to be in single-entity form If `single` is None or "auto", the form is inferred. check_storages: bool - Whether to check if the instance already exists in storages. + Whether to check if the instance already exists in storages + specified in `dlite.storage_path`. """ if single is None or single == 'auto': single = True if 'properties' in d else False diff --git a/storages/python/python-storage-plugins/http.py b/storages/python/python-storage-plugins/http.py index b567a0138..71c4384e0 100644 --- a/storages/python/python-storage-plugins/http.py +++ b/storages/python/python-storage-plugins/http.py @@ -1,19 +1,37 @@ +"""DLite storage plugin for fetching instances with HTTP GET requests. + +This simple plugin was initially added to support the entity service (see +https://github.com/SINTEF/dlite-entities-service), but may have other uses. +""" import json import requests import dlite +from dlite.options import Options class http(dlite.DLiteStorageBase): - """DLite storage plugin that fetches entities with http GET.""" - - def open(self, uri, options=None): - """Opens `uri`.""" - - def load(self, id): - """Returns instance retrieved from HTTP GET on `id`.""" - r = requests.get(id) - content = json.loads(r.content) - if "detail" in content: - raise dlite.DLiteError(content["detail"]) - return dlite.Instance.from_json(content) + """DLite storage plugin that fetches instances with HTTP GET.""" + + def open(self, location, options=None): + """Opens `location`. + + Arguments: + location: web address to access + options: Supported options: + - `single`: Whether the input is assumed to be in single- + entity form. The default (`"auto"`) will try to infer + it automatically. + """ + self.options = Options("single=auto") + + r = requests.get(location) + self.content = json.loads(r.content) + if "detail" in self.content: + raise dlite.DLiteStorageOpenError(content["detail"]) + + def load(self, id=None): + """Returns instance retrieved from HTTP GET.""" + s = self.options.single + single = s if s == "auto" else dlite.asbool(s) + return dlite.Instance.from_dict(self.content, id=id, single=single) From aa954d2cb8426b2f0d833b0884dd25bb2b919b85 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Sun, 13 Aug 2023 23:34:19 +0200 Subject: [PATCH 2/5] Added test_http.py --- storages/python/tests-python/CMakeLists.txt | 1 + storages/python/tests-python/test_http.py | 13 +++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 storages/python/tests-python/test_http.py diff --git a/storages/python/tests-python/CMakeLists.txt b/storages/python/tests-python/CMakeLists.txt index 144b6fd78..80d4ad473 100644 --- a/storages/python/tests-python/CMakeLists.txt +++ b/storages/python/tests-python/CMakeLists.txt @@ -11,6 +11,7 @@ set(python-tests test_minio test_template test_template-jinja + test_http ) include(FindPythonModule) diff --git a/storages/python/tests-python/test_http.py b/storages/python/tests-python/test_http.py new file mode 100644 index 000000000..8fee6ec03 --- /dev/null +++ b/storages/python/tests-python/test_http.py @@ -0,0 +1,13 @@ +try: + import requests +except ModuleNotFoundError: + print("requests not installed, skipping test") + raise SystemExit(44) # skip test + +import dlite + + +url = "https://raw.githubusercontent.com/SINTEF/dlite/master/storages/python/tests-python/input/test_meta.json" + +meta = dlite.Instance.from_location("http", url) +print(meta) From 36962c020f049c6d3027ac1799863c6cc05e079e Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 00:20:21 +0200 Subject: [PATCH 3/5] Added --package option to patch-activate.sh script. --- cmake/patch-activate.sh | 58 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/cmake/patch-activate.sh b/cmake/patch-activate.sh index 59c9409ac..c1abfaefc 100755 --- a/cmake/patch-activate.sh +++ b/cmake/patch-activate.sh @@ -1,13 +1,61 @@ -#!/bin/sh -# -# Patch virtualenv activate script to support LD_LIBRARY_PATH -# +#!/bin/bash + +help() { +cat <&2 exit 1 fi +libdirs="\$VIRTUAL_ENV/lib" + +# Parse options +while true; do + case "$1" in + '') break;; + -h|--help) help; exit 0;; + -p|--package) add_package $2; shift;; + *) echo "Invalid argument: '$1'"; exit 1;; + esac + shift +done + + cd $VIRTUAL_ENV/bin patch -u -f -F 1 < Date: Tue, 15 Aug 2023 00:40:32 +0200 Subject: [PATCH 4/5] Added requests to requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4c9bd4dec..ef86edbc5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,4 @@ openpyxl>=3.0.9,<3.2 pymongo>=4.4.0,<5 tripper>=0.2.5,<0.3 pydantic>=1.10.0,<2 - +requests>=2.10,<3 From 6191f65e00e3b3347f22580948a991568ccf07f3 Mon Sep 17 00:00:00 2001 From: Jesper Friis Date: Tue, 15 Aug 2023 00:46:40 +0200 Subject: [PATCH 5/5] added assert statement to test_http --- storages/python/tests-python/test_http.py | 40 ++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/storages/python/tests-python/test_http.py b/storages/python/tests-python/test_http.py index 8fee6ec03..d2a27d71d 100644 --- a/storages/python/tests-python/test_http.py +++ b/storages/python/tests-python/test_http.py @@ -10,4 +10,42 @@ url = "https://raw.githubusercontent.com/SINTEF/dlite/master/storages/python/tests-python/input/test_meta.json" meta = dlite.Instance.from_location("http", url) -print(meta) + +assert str(meta) == """ +{ + "uri": "http://onto-ns.com/meta/0.1/TestEntity", + "description": "test entity with explicit meta", + "dimensions": { + "L": "first dim", + "M": "second dim", + "N": "third dim" + }, + "properties": { + "myblob": { + "type": "blob3" + }, + "mydouble": { + "type": "float64", + "unit": "m", + "description": "A double following a single character..." + }, + "myfixstring": { + "type": "string3", + "description": "A fix string." + }, + "mystring": { + "type": "string", + "description": "A string pointer." + }, + "myshort": { + "type": "uint16", + "description": "An unsigned short integer." + }, + "myarray": { + "type": "int32", + "shape": ["L", "M", "N"], + "description": "An array string pointer." + } + } +} +""".strip()