diff --git a/pyfdb/fdb_api.py b/pyfdb/fdb_api.py new file mode 100644 index 0000000..0179df1 --- /dev/null +++ b/pyfdb/fdb_api.py @@ -0,0 +1,167 @@ +import os +from typing import List + +import cffi +import findlibs +from packaging import version + +__version__ = "0.0.4" +__fdb_version__ = "5.12.1" + +ffi = cffi.FFI() + + +class FDBException(RuntimeError): + pass + + +class PatchedLib: + """ + Patch a CFFI library with error handling + + Finds the header file associated with the FDB C API and parses it, loads the shared library, + and patches the accessors with automatic python-C error handling. + """ + + def __init__(self): + self.path = findlibs.find("fdb5") + + if self.path is None: + raise RuntimeError("FDB5 library not found") + + ffi.cdef(self.__read_header()) + self.__lib = ffi.dlopen(self.path) + + # All of the executable members of the CFFI-loaded library are functions in the FDB + # C API. These should be wrapped with the correct error handling. Otherwise forward + # these on directly. + + for f in dir(self.__lib): + try: + attr = getattr(self.__lib, f) + setattr(self, f, self.__check_error(attr, f) if callable(attr) else attr) + except Exception as e: + print(e) + print("Error retrieving attribute", f, "from library") + + # Initialise the library, and set it up for python-appropriate behaviour + + self.fdb_initialise() + + # Check the library version + + tmp_str = ffi.new("char**") + self.fdb_version(tmp_str) + self.version = ffi.string(tmp_str[0]).decode("utf-8") + + if version.parse(self.version) < version.parse(__fdb_version__): + raise RuntimeError( + f"This version of pyfdb ({__version__}) requires fdb version {__fdb_version__} or greater." + f"You have fdb version {self.version} loaded from {self.path}" + ) + + def __read_header(self): + with open(os.path.join(os.path.dirname(__file__), "processed_fdb.h"), "r") as f: + return f.read() + + def __check_error(self, fn, name): + """ + If calls into the FDB library return errors, ensure that they get detected and reported + by throwing an appropriate python exception. + """ + + def wrapped_fn(*args, **kwargs): + retval = fn(*args, **kwargs) + if retval != self.__lib.FDB_SUCCESS and retval != self.__lib.FDB_ITERATION_COMPLETE: + error_str = "Error in function {}: {}".format( + name, ffi.string(self.__lib.fdb_error_string(retval)).decode("utf-8", "backslashreplace") + ) + raise FDBException(error_str) + return retval + + return wrapped_fn + + def __repr__(self): + return f"" + + +class FDBApi: + + @classmethod + def get_gc_fdb_keys(cls): + """ + Create a new `fdb_key_t**` object and return a garbage collected version + of it. + """ + key = ffi.new("fdb_key_t**") + lib.fdb_new_key(key) + # Set free function + return fdb_api.ffi.gc(key[0], fdb_api.lib.fdb_delete_key) + + @classmethod + def add_fdb_key(cls, fdb_keys, param: str, value: str): + """ + Takes a new `fdb_key_t**` object, a parameter and a value and adds it + as its keys. + + param and value are ascii encode during the function call + ----------------------- + e.g. + self.__key = FDBApi.get_gc_fdb_keys() + FDBApi.add_fdb_key(self.__key, "param", "value") + + """ + lib.fdb_key_add( + fdb_keys, + fdb_api.ffi.new("const char[]", param.encode("ascii")), + fdb_api.ffi.new("const char[]", value.encode("ascii")), + ) + + @classmethod + def get_gc_fdb_request(cls): + """ + Create a new `fdb_key_t**` object and return a garbage collected version + of it. + """ + newrequest = ffi.new("fdb_request_t**") + + # we assume a retrieve request represented as a dictionary + lib.fdb_new_request(newrequest) + return ffi.gc(newrequest[0], lib.fdb_delete_request) + + @classmethod + def add_fdb_request(cls, requests, name: str, cvals: List, num_vals: int): + """ + Takes a new `fdb_request_t**` object, a name and a cvals and adds it + as its keys. + + param and value are ascii encode during the function call + ----------------------- + e.g. + + self.__request = FDBApi.get_gc_fdb_request() + FDBApi.add_fdb_request(self.__request, name, cvals, len(values)) + """ + lib.fdb_request_add( + requests, + ffi.new("const char[]", name.encode("ascii")), + ffi.new("const char*[]", cvals), + num_vals, + ) + + @classmethod + def create_ffi_string_ascii_encoded(cls, value): + """ + Create a new const char[] for a given value, encoded as ascii + """ + return ffi.new("const char[]", value.encode("ascii")) + + @classmethod + def fdb_expand_request(cls, request): + """ + Forwards call to `PatchedLib::fdb_expand_request` method. + """ + lib.fdb_expand_request(request) + + +lib = PatchedLib() diff --git a/pyfdb/pyfdb.py b/pyfdb/pyfdb.py index e62a9fa..8f32863 100644 --- a/pyfdb/pyfdb.py +++ b/pyfdb/pyfdb.py @@ -14,117 +14,23 @@ # limitations under the License. import io import json -import os from functools import wraps -import cffi -import findlibs -from packaging import version - -__version__ = "0.0.4" - -__fdb_version__ = "5.12.1" - -ffi = cffi.FFI() - - -class FDBException(RuntimeError): - pass - - -class PatchedLib: - """ - Patch a CFFI library with error handling - - Finds the header file associated with the FDB C API and parses it, loads the shared library, - and patches the accessors with automatic python-C error handling. - """ - - def __init__(self): - self.path = findlibs.find("fdb5") - - if self.path is None: - raise RuntimeError("FDB5 library not found") - - ffi.cdef(self.__read_header()) - self.__lib = ffi.dlopen(self.path) - - # All of the executable members of the CFFI-loaded library are functions in the FDB - # C API. These should be wrapped with the correct error handling. Otherwise forward - # these on directly. - - for f in dir(self.__lib): - try: - attr = getattr(self.__lib, f) - setattr(self, f, self.__check_error(attr, f) if callable(attr) else attr) - except Exception as e: - print(e) - print("Error retrieving attribute", f, "from library") - - # Initialise the library, and set it up for python-appropriate behaviour - - self.fdb_initialise() - - # Check the library version - - tmp_str = ffi.new("char**") - self.fdb_version(tmp_str) - self.version = ffi.string(tmp_str[0]).decode("utf-8") - - if version.parse(self.version) < version.parse(__fdb_version__): - raise RuntimeError( - f"This version of pyfdb ({__version__}) requires fdb version {__fdb_version__} or greater." - f"You have fdb version {self.version} loaded from {self.path}" - ) - - def __read_header(self): - with open(os.path.join(os.path.dirname(__file__), "processed_fdb.h"), "r") as f: - return f.read() - - def __check_error(self, fn, name): - """ - If calls into the FDB library return errors, ensure that they get detected and reported - by throwing an appropriate python exception. - """ - - def wrapped_fn(*args, **kwargs): - retval = fn(*args, **kwargs) - if retval != self.__lib.FDB_SUCCESS and retval != self.__lib.FDB_ITERATION_COMPLETE: - error_str = "Error in function {}: {}".format( - name, ffi.string(self.__lib.fdb_error_string(retval)).decode("utf-8", "backslashreplace") - ) - raise FDBException(error_str) - return retval - - return wrapped_fn - - def __repr__(self): - return f"" - - -# Bootstrap the library - -lib = PatchedLib() +from pyfdb import fdb_api +from pyfdb.fdb_api import FDBApi class Key: __key = None def __init__(self, keys): - key = ffi.new("fdb_key_t**") - lib.fdb_new_key(key) - # Set free function - self.__key = ffi.gc(key[0], lib.fdb_delete_key) + self.__key = FDBApi.get_gc_fdb_keys() for k, v in keys.items(): self.set(k, v) def set(self, param, value): - lib.fdb_key_add( - self.__key, - ffi.new("const char[]", param.encode("ascii")), - ffi.new("const char[]", value.encode("ascii")), - ) + FDBApi.add_fdb_key(self.__key, param, value) @property def ctype(self): @@ -135,11 +41,7 @@ class Request: __request = None def __init__(self, request): - newrequest = ffi.new("fdb_request_t**") - - # we assume a retrieve request represented as a dictionary - lib.fdb_new_request(newrequest) - self.__request = ffi.gc(newrequest[0], lib.fdb_delete_request) + self.__request = FDBApi.get_gc_fdb_request() for name, values in request.items(): self.value(name, values) @@ -152,18 +54,13 @@ def value(self, name, values): for value in values: if isinstance(value, int): value = str(value) - cval = ffi.new("const char[]", value.encode("ascii")) + cval = FDBApi.create_ffi_string_ascii_encoded(value) cvals.append(cval) - lib.fdb_request_add( - self.__request, - ffi.new("const char[]", name.encode("ascii")), - ffi.new("const char*[]", cvals), - len(values), - ) + FDBApi.add_fdb_request(self.__request, name, cvals, len(values)) def expand(self): - lib.fdb_expand_request(self.__request) + FDBApi.fdb_expand_request(self.__request) @property def ctype(self): @@ -175,45 +72,45 @@ class ListIterator: __key = False def __init__(self, fdb, request, duplicates, key=False, expand=True): - iterator = ffi.new("fdb_listiterator_t**") + iterator = fdb_api.ffi.new("fdb_listiterator_t**") if request: req = Request(request) if expand: req.expand() - lib.fdb_list(fdb.ctype, req.ctype, iterator, duplicates) + fdb_api.lib.fdb_list(fdb.ctype, req.ctype, iterator, duplicates) else: - lib.fdb_list(fdb.ctype, ffi.NULL, iterator, duplicates) + fdb_api.lib.fdb_list(fdb.ctype, fdb_api.ffi.NULL, iterator, duplicates) - self.__iterator = ffi.gc(iterator[0], lib.fdb_delete_listiterator) + self.__iterator = fdb_api.ffi.gc(iterator[0], fdb_api.lib.fdb_delete_listiterator) self.__key = key - self.path = ffi.new("const char**") - self.off = ffi.new("size_t*") - self.len = ffi.new("size_t*") + self.path = fdb_api.ffi.new("const char**") + self.off = fdb_api.ffi.new("size_t*") + self.len = fdb_api.ffi.new("size_t*") def __next__(self) -> dict: - err = lib.fdb_listiterator_next(self.__iterator) + err = fdb_api.lib.fdb_listiterator_next(self.__iterator) if err != 0: raise StopIteration - lib.fdb_listiterator_attrs(self.__iterator, self.path, self.off, self.len) - el = dict(path=ffi.string(self.path[0]).decode("utf-8"), offset=self.off[0], length=self.len[0]) + fdb_api.lib.fdb_listiterator_attrs(self.__iterator, self.path, self.off, self.len) + el = dict(path=fdb_api.ffi.string(self.path[0]).decode("utf-8"), offset=self.off[0], length=self.len[0]) if self.__key: - splitkey = ffi.new("fdb_split_key_t**") - lib.fdb_new_splitkey(splitkey) - key = ffi.gc(splitkey[0], lib.fdb_delete_splitkey) + splitkey = fdb_api.ffi.new("fdb_split_key_t**") + fdb_api.lib.fdb_new_splitkey(splitkey) + key = fdb_api.ffi.gc(splitkey[0], fdb_api.lib.fdb_delete_splitkey) - lib.fdb_listiterator_splitkey(self.__iterator, key) + fdb_api.lib.fdb_listiterator_splitkey(self.__iterator, key) - k = ffi.new("const char**") - v = ffi.new("const char**") - level = ffi.new("size_t*") + k = fdb_api.ffi.new("const char**") + v = fdb_api.ffi.new("const char**") + level = fdb_api.ffi.new("size_t*") meta = dict() - while lib.fdb_splitkey_next_metadata(key, k, v, level) == 0: - meta[ffi.string(k[0]).decode("utf-8")] = ffi.string(v[0]).decode("utf-8") + while fdb_api.lib.fdb_splitkey_next_metadata(key, k, v, level) == 0: + meta[fdb_api.ffi.string(k[0]).decode("utf-8")] = fdb_api.ffi.string(v[0]).decode("utf-8") el["keys"] = meta return el @@ -227,30 +124,30 @@ class DataRetriever(io.RawIOBase): __opened = False def __init__(self, fdb, request, expand=True): - dataread = ffi.new("fdb_datareader_t **") - lib.fdb_new_datareader(dataread) - self.__dataread = ffi.gc(dataread[0], lib.fdb_delete_datareader) + dataread = fdb_api.ffi.new("fdb_datareader_t **") + fdb_api.lib.fdb_new_datareader(dataread) + self.__dataread = fdb_api.ffi.gc(dataread[0], fdb_api.lib.fdb_delete_datareader) req = Request(request) if expand: req.expand() - lib.fdb_retrieve(fdb.ctype, req.ctype, self.__dataread) + fdb_api.lib.fdb_retrieve(fdb.ctype, req.ctype, self.__dataread) mode = "rb" def open(self): if not self.__opened: self.__opened = True - lib.fdb_datareader_open(self.__dataread, ffi.NULL) + fdb_api.lib.fdb_datareader_open(self.__dataread, fdb_api.ffi.NULL) def close(self): if self.__opened: self.__opened = False - lib.fdb_datareader_close(self.__dataread) + fdb_api.lib.fdb_datareader_close(self.__dataread) def skip(self, count): self.open() if isinstance(count, int): - lib.fdb_datareader_skip(self.__dataread, count) + fdb_api.lib.fdb_datareader_skip(self.__dataread, count) def seek(self, where, whence=io.SEEK_SET): if whence != io.SEEK_SET: @@ -259,20 +156,20 @@ def seek(self, where, whence=io.SEEK_SET): ) self.open() if isinstance(where, int): - lib.fdb_datareader_seek(self.__dataread, where) + fdb_api.lib.fdb_datareader_seek(self.__dataread, where) def tell(self): self.open() - where = ffi.new("long*") - lib.fdb_datareader_tell(self.__dataread, where) + where = fdb_api.ffi.new("long*") + fdb_api.lib.fdb_datareader_tell(self.__dataread, where) return where[0] def read(self, count=-1) -> bytes: self.open() if isinstance(count, int): buf = bytearray(count) - read = ffi.new("long*") - lib.fdb_datareader_read(self.__dataread, ffi.from_buffer(buf), count, read) + read = fdb_api.ffi.new("long*") + fdb_api.lib.fdb_datareader_read(self.__dataread, fdb_api.ffi.from_buffer(buf), count, read) return buf[0 : read[0]] return bytearray() @@ -297,7 +194,7 @@ class FDB: __fdb = None def __init__(self, config=None, user_config=None): - fdb = ffi.new("fdb_handle_t**") + fdb = fdb_api.ffi.new("fdb_handle_t**") if config is not None or user_config is not None: @@ -311,16 +208,16 @@ def prepare_config(c): config = prepare_config(config) user_config = prepare_config(user_config) - lib.fdb_new_handle_from_yaml( + fdb_api.lib.fdb_new_handle_from_yaml( fdb, - ffi.new("const char[]", config.encode("utf-8")), - ffi.new("const char[]", user_config.encode("utf-8")), + fdb_api.ffi.new("const char[]", config.encode("utf-8")), + fdb_api.ffi.new("const char[]", user_config.encode("utf-8")), ) else: - lib.fdb_new_handle(fdb) + fdb_api.lib.fdb_new_handle(fdb) # Set free function - self.__fdb = ffi.gc(fdb[0], lib.fdb_delete_handle) + self.__fdb = fdb_api.ffi.gc(fdb[0], fdb_api.lib.fdb_delete_handle) def archive(self, data, request=None) -> None: """Archive data into the FDB5 database @@ -331,13 +228,15 @@ def archive(self, data, request=None) -> None: if not provided the key will be constructed from the data. """ if request is None: - lib.fdb_archive_multiple(self.ctype, ffi.NULL, ffi.from_buffer(data), len(data)) + fdb_api.lib.fdb_archive_multiple(self.ctype, fdb_api.ffi.NULL, fdb_api.ffi.from_buffer(data), len(data)) else: - lib.fdb_archive_multiple(self.ctype, Request(request).ctype, ffi.from_buffer(data), len(data)) + fdb_api.lib.fdb_archive_multiple( + self.ctype, Request(request).ctype, fdb_api.ffi.from_buffer(data), len(data) + ) def flush(self) -> None: """Flush any archived data to disk""" - lib.fdb_flush(self.ctype) + fdb_api.lib.fdb_flush(self.ctype) def list(self, request=None, duplicates=False, keys=False) -> ListIterator: """List entries in the FDB5 database