Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Feature list depth level #21

Closed
wants to merge 3 commits into from
Closed

Feature list depth level #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bcb8bb3
fix(splitkey): separate next and metadata
mcakircali Aug 6, 2024
1b4067a
feat(list): added depth option to list()
mcakircali Aug 6, 2024
d4c3461
test(list): depth option
mcakircali Aug 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions pyfdb/processed_fdb.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@ int fdb_delete_request(fdb_request_t* req);
struct fdb_split_key_t;
typedef struct fdb_split_key_t fdb_split_key_t;
int fdb_new_splitkey(fdb_split_key_t** key);
int fdb_splitkey_next_metadata(fdb_split_key_t* it, const char** key, const char** value, size_t* level);
int fdb_splitkey_next(fdb_split_key_t* it);
int fdb_splitkey_metadata(fdb_split_key_t* it, const char** key, const char** value, size_t* level);
int fdb_delete_splitkey(fdb_split_key_t* key);

struct fdb_listiterator_t;
Expand All @@ -53,7 +54,7 @@ typedef struct fdb_handle_t fdb_handle_t;
int fdb_new_handle(fdb_handle_t** fdb);
int fdb_archive(fdb_handle_t* fdb, fdb_key_t* key, const char* data, size_t length);
int fdb_archive_multiple(fdb_handle_t* fdb, fdb_request_t* req, const char* data, size_t length);
int fdb_list(fdb_handle_t* fdb, const fdb_request_t* req, fdb_listiterator_t** it, bool duplicates);
int fdb_list(fdb_handle_t* fdb, const fdb_request_t* req, fdb_listiterator_t** it, bool duplicates, int depth);
int fdb_retrieve(fdb_handle_t* fdb, fdb_request_t* req, fdb_datareader_t* dr);
int fdb_flush(fdb_handle_t* fdb);
int fdb_delete_handle(fdb_handle_t* fdb);
17 changes: 9 additions & 8 deletions pyfdb/pyfdb.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,15 +168,15 @@ class ListIterator:
__iterator = None
__key = False

def __init__(self, fdb, request, duplicates, key=False, expand=True):
def __init__(self, fdb, request, duplicates, key=False, expand=True, depth=3):
iterator = ffi.new("fdb_listiterator_t**")
if request:
req = Request(request)
if expand:
req.expand()
lib.fdb_list(fdb.ctype, req.ctype, iterator, duplicates)
lib.fdb_list(fdb.ctype, req.ctype, iterator, duplicates, depth)
else:
lib.fdb_list(fdb.ctype, ffi.NULL, iterator, duplicates)
lib.fdb_list(fdb.ctype, ffi.NULL, iterator, duplicates, depth)

self.__iterator = ffi.gc(iterator[0], lib.fdb_delete_listiterator)
self.__key = key
Expand Down Expand Up @@ -207,8 +207,9 @@ def __iter__(self):
level = ffi.new("size_t*")

meta = dict()
while lib.fdb_splitkey_next_metadata(key, k, v, level) == 0:
while lib.fdb_splitkey_metadata(key, k, v, level) == 0:
meta[ffi.string(k[0]).decode("utf-8")] = ffi.string(v[0]).decode("utf-8")
lib.fdb_splitkey_next(key)
el["keys"] = meta

yield el
Expand Down Expand Up @@ -291,8 +292,8 @@ def archive(self, data, request=None):
def flush(self):
lib.fdb_flush(self.ctype)

def list(self, request=None, duplicates=False, keys=False):
return ListIterator(self, request, duplicates, keys)
def list(self, request=None, duplicates=False, keys=False, expand=True, depth=3):
return ListIterator(self, request, duplicates, keys, expand, depth)

def retrieve(self, request):
return DataRetriever(self, request)
Expand All @@ -312,11 +313,11 @@ def archive(data):
fdb.archive(data)


def list(request, duplicates=False, keys=False):
def list(request, duplicates=False, keys=False, expand=True, depth=3):
global fdb
if not fdb:
fdb = FDB()
return ListIterator(fdb, request, duplicates, keys)
return ListIterator(fdb, request, duplicates, keys, expand, depth)


def retrieve(request):
Expand Down
211 changes: 171 additions & 40 deletions tests/test_pyfdb.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,45 +8,51 @@
# granted to it by virtue of its status as an intergovernmental organisation nor
# does it submit to any jurisdiction.
import os
import shutil

from pyeccodes import Reader

import pyfdb

fdb = pyfdb.FDB()

# Archive
key = {
"domain": "g",
"stream": "oper",
"levtype": "pl",
"levelist": "300",
"date": "20191110",
"time": "0000",
"step": "0",
"param": "138",
"class": "rd",
"type": "an",
"expver": "xxxx",
}
def test_archive():
"""
Test the archive function
"""

key = {
"domain": "g",
"stream": "oper",
"levtype": "pl",
"levelist": "300",
"date": "20191110",
"time": "0000",
"step": "0",
"param": "138",
"class": "rd",
"type": "an",
"expver": "xxxx",
}

def test_archival_read():
def archive_grib(grib: str):
filename = os.path.join(os.path.dirname(__file__), grib)
pyfdb.archive(open(filename, "rb").read())

filename = os.path.join(os.path.dirname(__file__), "x138-300.grib")
pyfdb.archive(open(filename, "rb").read())
archive_grib("x138-300.grib")

key["levelist"] = "400"
filename = os.path.join(os.path.dirname(__file__), "x138-400.grib")
pyfdb.archive(open(filename, "rb").read())
archive_grib("x138-400.grib")

key["expver"] = "xxxy"
filename = os.path.join(os.path.dirname(__file__), "y138-400.grib")
pyfdb.archive(open(filename, "rb").read())
archive_grib("y138-400.grib")

pyfdb.flush()

# List

def test_list():
"""
Test the list function
"""

request = {
"class": "rd",
"expver": "xxxx",
Expand All @@ -60,6 +66,7 @@ def test_archival_read():
"levelist": [300, "500"],
"param": ["138", 155, "t"],
}

print("direct function, request as dictionary:", request)
for el in pyfdb.list(request, True):
assert el["path"]
Expand Down Expand Up @@ -98,6 +105,8 @@ def test_archival_read():
request["levelist"] = ["400", "500", "700", "850", "1000"]
print("")
print("fdb object, request as dictionary:", request)

fdb = pyfdb.FDB()
for el in fdb.list(request, True, True):
assert el["path"]
assert el["path"].find("rd:xxxx:oper:20191110:0000:g/an:pl.") != -1
Expand All @@ -106,7 +115,139 @@ def test_archival_read():
assert keys["class"] == "rd"
assert keys["levelist"] == "400"

# Retrieve

def test_list_depth():
"""
Test fdb list depth option
"""

request = {
"class": "rd",
"expver": ["xxxx", "xxxy"],
"stream": "oper",
"date": "20191110",
"time": "0000",
"domain": "g",
"type": "an",
"levtype": "pl",
"step": 0,
"levelist": ["300", "400"],
"param": "138",
}

print("test list: request={0}".format(request))

print("list: depth=1")

list = [
{"class": "rd", "date": "20191110", "domain": "g", "expver": "xxxx", "stream": "oper", "time": "0000"},
{"class": "rd", "date": "20191110", "domain": "g", "expver": "xxxy", "stream": "oper", "time": "0000"},
]

for id, el in enumerate(pyfdb.list(request, True, True, False, 1)):
assert "keys" in el
assert el["keys"] == list[id]
# print("%(keys)s" % el)

print("list: depth=2")

list = [
{
"class": "rd",
"date": "20191110",
"domain": "g",
"expver": "xxxx",
"stream": "oper",
"time": "0000",
"levtype": "pl",
"type": "an",
},
{
"class": "rd",
"date": "20191110",
"domain": "g",
"expver": "xxxy",
"stream": "oper",
"time": "0000",
"levtype": "pl",
"type": "an",
},
]

for id, el in enumerate(pyfdb.list(request, True, True, False, 2)):
assert "keys" in el
assert el["keys"] == list[id]
# print("%(keys)s" % el)

print("list: depth=3")

list = [
{
"class": "rd",
"date": "20191110",
"domain": "g",
"expver": "xxxx",
"stream": "oper",
"time": "0000",
"levtype": "pl",
"type": "an",
"levelist": "300",
"param": "138",
"step": "0",
},
{
"class": "rd",
"date": "20191110",
"domain": "g",
"expver": "xxxx",
"stream": "oper",
"time": "0000",
"levtype": "pl",
"type": "an",
"levelist": "400",
"param": "138",
"step": "0",
},
{
"class": "rd",
"date": "20191110",
"domain": "g",
"expver": "xxxy",
"stream": "oper",
"time": "0000",
"levtype": "pl",
"type": "an",
"levelist": "400",
"param": "138",
"step": "0",
},
]

for id, el in enumerate(pyfdb.list(request, True, True, False, 3)):
assert "keys" in el
assert el["keys"] == list[id]
# print("%(keys)s" % el)

# default depth is 3
for id, el in enumerate(pyfdb.list(request, True, True, False)):
assert "keys" in el
assert el["keys"] == list[id]
# print("%(keys)s" % el)


def test_retrieve():
"""
Test the retrieve function
"""
import shutil

fdb = pyfdb.FDB()

def retrieve_grib(grib: str):
filename = os.path.join(os.path.dirname(__file__), grib)
with open(filename, "wb") as dest, fdb.retrieve(request) as src:
shutil.copyfileobj(src, dest)

request = {
"domain": "g",
"stream": "oper",
Expand All @@ -121,23 +262,13 @@ def test_archival_read():
"type": "an",
}

filename = os.path.join(os.path.dirname(__file__), "x138-300bis.grib")
print("")
print("save to file ", filename)
with open(filename, "wb") as o, fdb.retrieve(request) as i:
shutil.copyfileobj(i, o)
retrieve_grib("x138-300bis.grib")

request["levelist"] = "400"
filename = os.path.join(os.path.dirname(__file__), "x138-400bis.grib")
print("save to file ", filename)
with open(filename, "wb") as o, fdb.retrieve(request) as i:
shutil.copyfileobj(i, o)
retrieve_grib("x138-400bis.grib")

request["expver"] = "xxxy"
filename = os.path.join(os.path.dirname(__file__), "y138-400bis.grib")
print("save to file ", filename)
with open(filename, "wb") as o, pyfdb.retrieve(request) as i:
shutil.copyfileobj(i, o)
retrieve_grib("y138-400bis.grib")

# request = {
# 'class': 'od',
Expand Down Expand Up @@ -186,5 +317,5 @@ def test_archival_read():

print("")
print("save to file ", filename)
with open(filename, "wb") as o, fdb.retrieve(request) as i:
shutil.copyfileobj(i, o)
with open(filename, "wb") as dest, fdb.retrieve(request) as src:
shutil.copyfileobj(src, dest)
Loading