From bcb8bb33efcffd97e0281d541a0da8eda1b07c54 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 6 Aug 2024 10:07:07 +0200 Subject: [PATCH 1/3] fix(splitkey): separate next and metadata --- pyfdb/processed_fdb.h | 3 ++- pyfdb/pyfdb.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pyfdb/processed_fdb.h b/pyfdb/processed_fdb.h index 06b2b1e..1b0647e 100644 --- a/pyfdb/processed_fdb.h +++ b/pyfdb/processed_fdb.h @@ -27,7 +27,8 @@ int fdb_delete_request(fdb_request_t* req); struct fdb_split_key_t; typedef struct fdb_split_key_t fdb_split_key_t; int fdb_new_splitkey(fdb_split_key_t** key); -int fdb_splitkey_next_metadata(fdb_split_key_t* it, const char** key, const char** value, size_t* level); +int fdb_splitkey_next(fdb_split_key_t* it); +int fdb_splitkey_metadata(fdb_split_key_t* it, const char** key, const char** value, size_t* level); int fdb_delete_splitkey(fdb_split_key_t* key); struct fdb_listiterator_t; diff --git a/pyfdb/pyfdb.py b/pyfdb/pyfdb.py index 25f8bd2..cba668d 100644 --- a/pyfdb/pyfdb.py +++ b/pyfdb/pyfdb.py @@ -207,8 +207,9 @@ def __iter__(self): level = ffi.new("size_t*") meta = dict() - while lib.fdb_splitkey_next_metadata(key, k, v, level) == 0: + while lib.fdb_splitkey_metadata(key, k, v, level) == 0: meta[ffi.string(k[0]).decode("utf-8")] = ffi.string(v[0]).decode("utf-8") + lib.fdb_splitkey_next(key) el["keys"] = meta yield el From 1b4067af899ce6de9a8e7e89964ce484ab325d6b Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 6 Aug 2024 14:33:45 +0200 Subject: [PATCH 2/3] feat(list): added depth option to list() --- pyfdb/processed_fdb.h | 2 +- pyfdb/pyfdb.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pyfdb/processed_fdb.h b/pyfdb/processed_fdb.h index 1b0647e..8872ad8 100644 --- a/pyfdb/processed_fdb.h +++ b/pyfdb/processed_fdb.h @@ -54,7 +54,7 @@ typedef struct fdb_handle_t fdb_handle_t; int fdb_new_handle(fdb_handle_t** fdb); int fdb_archive(fdb_handle_t* fdb, fdb_key_t* key, const char* data, size_t length); int fdb_archive_multiple(fdb_handle_t* fdb, fdb_request_t* req, const char* data, size_t length); -int fdb_list(fdb_handle_t* fdb, const fdb_request_t* req, fdb_listiterator_t** it, bool duplicates); +int fdb_list(fdb_handle_t* fdb, const fdb_request_t* req, fdb_listiterator_t** it, bool duplicates, int depth); int fdb_retrieve(fdb_handle_t* fdb, fdb_request_t* req, fdb_datareader_t* dr); int fdb_flush(fdb_handle_t* fdb); int fdb_delete_handle(fdb_handle_t* fdb); diff --git a/pyfdb/pyfdb.py b/pyfdb/pyfdb.py index cba668d..d3ad174 100644 --- a/pyfdb/pyfdb.py +++ b/pyfdb/pyfdb.py @@ -168,15 +168,15 @@ class ListIterator: __iterator = None __key = False - def __init__(self, fdb, request, duplicates, key=False, expand=True): + def __init__(self, fdb, request, duplicates, key=False, expand=True, depth=3): iterator = ffi.new("fdb_listiterator_t**") if request: req = Request(request) if expand: req.expand() - lib.fdb_list(fdb.ctype, req.ctype, iterator, duplicates) + lib.fdb_list(fdb.ctype, req.ctype, iterator, duplicates, depth) else: - lib.fdb_list(fdb.ctype, ffi.NULL, iterator, duplicates) + lib.fdb_list(fdb.ctype, ffi.NULL, iterator, duplicates, depth) self.__iterator = ffi.gc(iterator[0], lib.fdb_delete_listiterator) self.__key = key @@ -292,8 +292,8 @@ def archive(self, data, request=None): def flush(self): lib.fdb_flush(self.ctype) - def list(self, request=None, duplicates=False, keys=False): - return ListIterator(self, request, duplicates, keys) + def list(self, request=None, duplicates=False, keys=False, expand=True, depth=3): + return ListIterator(self, request, duplicates, keys, expand, depth) def retrieve(self, request): return DataRetriever(self, request) @@ -313,11 +313,11 @@ def archive(data): fdb.archive(data) -def list(request, duplicates=False, keys=False): +def list(request, duplicates=False, keys=False, expand=True, depth=3): global fdb if not fdb: fdb = FDB() - return ListIterator(fdb, request, duplicates, keys) + return ListIterator(fdb, request, duplicates, keys, expand, depth) def retrieve(request): From d4c346160aa5dcc937aeead783784df0c5d0b963 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Tue, 6 Aug 2024 18:18:51 +0200 Subject: [PATCH 3/3] test(list): depth option --- tests/test_pyfdb.py | 211 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 171 insertions(+), 40 deletions(-) diff --git a/tests/test_pyfdb.py b/tests/test_pyfdb.py index 9b4a5b3..1bdb826 100755 --- a/tests/test_pyfdb.py +++ b/tests/test_pyfdb.py @@ -8,45 +8,51 @@ # granted to it by virtue of its status as an intergovernmental organisation nor # does it submit to any jurisdiction. import os -import shutil from pyeccodes import Reader import pyfdb -fdb = pyfdb.FDB() -# Archive -key = { - "domain": "g", - "stream": "oper", - "levtype": "pl", - "levelist": "300", - "date": "20191110", - "time": "0000", - "step": "0", - "param": "138", - "class": "rd", - "type": "an", - "expver": "xxxx", -} +def test_archive(): + """ + Test the archive function + """ + key = { + "domain": "g", + "stream": "oper", + "levtype": "pl", + "levelist": "300", + "date": "20191110", + "time": "0000", + "step": "0", + "param": "138", + "class": "rd", + "type": "an", + "expver": "xxxx", + } -def test_archival_read(): + def archive_grib(grib: str): + filename = os.path.join(os.path.dirname(__file__), grib) + pyfdb.archive(open(filename, "rb").read()) - filename = os.path.join(os.path.dirname(__file__), "x138-300.grib") - pyfdb.archive(open(filename, "rb").read()) + archive_grib("x138-300.grib") key["levelist"] = "400" - filename = os.path.join(os.path.dirname(__file__), "x138-400.grib") - pyfdb.archive(open(filename, "rb").read()) + archive_grib("x138-400.grib") key["expver"] = "xxxy" - filename = os.path.join(os.path.dirname(__file__), "y138-400.grib") - pyfdb.archive(open(filename, "rb").read()) + archive_grib("y138-400.grib") + pyfdb.flush() - # List + +def test_list(): + """ + Test the list function + """ + request = { "class": "rd", "expver": "xxxx", @@ -60,6 +66,7 @@ def test_archival_read(): "levelist": [300, "500"], "param": ["138", 155, "t"], } + print("direct function, request as dictionary:", request) for el in pyfdb.list(request, True): assert el["path"] @@ -98,6 +105,8 @@ def test_archival_read(): request["levelist"] = ["400", "500", "700", "850", "1000"] print("") print("fdb object, request as dictionary:", request) + + fdb = pyfdb.FDB() for el in fdb.list(request, True, True): assert el["path"] assert el["path"].find("rd:xxxx:oper:20191110:0000:g/an:pl.") != -1 @@ -106,7 +115,139 @@ def test_archival_read(): assert keys["class"] == "rd" assert keys["levelist"] == "400" - # Retrieve + +def test_list_depth(): + """ + Test fdb list depth option + """ + + request = { + "class": "rd", + "expver": ["xxxx", "xxxy"], + "stream": "oper", + "date": "20191110", + "time": "0000", + "domain": "g", + "type": "an", + "levtype": "pl", + "step": 0, + "levelist": ["300", "400"], + "param": "138", + } + + print("test list: request={0}".format(request)) + + print("list: depth=1") + + list = [ + {"class": "rd", "date": "20191110", "domain": "g", "expver": "xxxx", "stream": "oper", "time": "0000"}, + {"class": "rd", "date": "20191110", "domain": "g", "expver": "xxxy", "stream": "oper", "time": "0000"}, + ] + + for id, el in enumerate(pyfdb.list(request, True, True, False, 1)): + assert "keys" in el + assert el["keys"] == list[id] + # print("%(keys)s" % el) + + print("list: depth=2") + + list = [ + { + "class": "rd", + "date": "20191110", + "domain": "g", + "expver": "xxxx", + "stream": "oper", + "time": "0000", + "levtype": "pl", + "type": "an", + }, + { + "class": "rd", + "date": "20191110", + "domain": "g", + "expver": "xxxy", + "stream": "oper", + "time": "0000", + "levtype": "pl", + "type": "an", + }, + ] + + for id, el in enumerate(pyfdb.list(request, True, True, False, 2)): + assert "keys" in el + assert el["keys"] == list[id] + # print("%(keys)s" % el) + + print("list: depth=3") + + list = [ + { + "class": "rd", + "date": "20191110", + "domain": "g", + "expver": "xxxx", + "stream": "oper", + "time": "0000", + "levtype": "pl", + "type": "an", + "levelist": "300", + "param": "138", + "step": "0", + }, + { + "class": "rd", + "date": "20191110", + "domain": "g", + "expver": "xxxx", + "stream": "oper", + "time": "0000", + "levtype": "pl", + "type": "an", + "levelist": "400", + "param": "138", + "step": "0", + }, + { + "class": "rd", + "date": "20191110", + "domain": "g", + "expver": "xxxy", + "stream": "oper", + "time": "0000", + "levtype": "pl", + "type": "an", + "levelist": "400", + "param": "138", + "step": "0", + }, + ] + + for id, el in enumerate(pyfdb.list(request, True, True, False, 3)): + assert "keys" in el + assert el["keys"] == list[id] + # print("%(keys)s" % el) + + # default depth is 3 + for id, el in enumerate(pyfdb.list(request, True, True, False)): + assert "keys" in el + assert el["keys"] == list[id] + # print("%(keys)s" % el) + + +def test_retrieve(): + """ + Test the retrieve function + """ + import shutil + + fdb = pyfdb.FDB() + + def retrieve_grib(grib: str): + filename = os.path.join(os.path.dirname(__file__), grib) + with open(filename, "wb") as dest, fdb.retrieve(request) as src: + shutil.copyfileobj(src, dest) + request = { "domain": "g", "stream": "oper", @@ -121,23 +262,13 @@ def test_archival_read(): "type": "an", } - filename = os.path.join(os.path.dirname(__file__), "x138-300bis.grib") - print("") - print("save to file ", filename) - with open(filename, "wb") as o, fdb.retrieve(request) as i: - shutil.copyfileobj(i, o) + retrieve_grib("x138-300bis.grib") request["levelist"] = "400" - filename = os.path.join(os.path.dirname(__file__), "x138-400bis.grib") - print("save to file ", filename) - with open(filename, "wb") as o, fdb.retrieve(request) as i: - shutil.copyfileobj(i, o) + retrieve_grib("x138-400bis.grib") request["expver"] = "xxxy" - filename = os.path.join(os.path.dirname(__file__), "y138-400bis.grib") - print("save to file ", filename) - with open(filename, "wb") as o, pyfdb.retrieve(request) as i: - shutil.copyfileobj(i, o) + retrieve_grib("y138-400bis.grib") # request = { # 'class': 'od', @@ -186,5 +317,5 @@ def test_archival_read(): print("") print("save to file ", filename) - with open(filename, "wb") as o, fdb.retrieve(request) as i: - shutil.copyfileobj(i, o) + with open(filename, "wb") as dest, fdb.retrieve(request) as src: + shutil.copyfileobj(src, dest)