diff --git a/.clang-format b/.clang-format index d79db2a84..b8b37afb7 100644 --- a/.clang-format +++ b/.clang-format @@ -1,12 +1,6 @@ --- Language: Cpp AccessModifierOffset: -4 -AlignAfterOpenBracket: Align -AlignConsecutiveAssignments: true -AlignConsecutiveDeclarations: false -AlignEscapedNewlines: Left -AlignOperands: true -AlignTrailingComments: true AllowAllParametersOfDeclarationOnNextLine: true AllowShortBlocksOnASingleLine: true AllowShortCaseLabelsOnASingleLine: false @@ -19,88 +13,56 @@ AlwaysBreakBeforeMultilineStrings: true AlwaysBreakTemplateDeclarations: true BinPackArguments: true BinPackParameters: true -BraceWrapping: +BraceWrapping: AfterClass: false AfterControlStatement: false - AfterEnum: true + AfterEnum: false AfterFunction: false AfterNamespace: false - AfterObjCDeclaration: false AfterStruct: false AfterUnion: false - BeforeCatch: true - BeforeElse: true + BeforeCatch: false + BeforeElse: false IndentBraces: false - SplitEmptyFunction: true - SplitEmptyRecord: true - SplitEmptyNamespace: true + SplitEmptyFunction: false + SplitEmptyRecord: false + SplitEmptyNamespace: false BreakBeforeBinaryOperators: None BreakBeforeBraces: Custom BreakBeforeInheritanceComma: false BreakBeforeTernaryOperators: true -BreakConstructorInitializersBeforeComma: false -BreakConstructorInitializers: AfterColon -BreakAfterJavaFieldAnnotations: false BreakStringLiterals: true ColumnLimit: 120 CommentPragmas: '^ IWYU pragma:' CompactNamespaces: false -ConstructorInitializerAllOnOneLineOrOnePerLine: true +PackConstructorInitializers: CurrentLine +BreakConstructorInitializers: BeforeComma ConstructorInitializerIndentWidth: 4 ContinuationIndentWidth: 4 Cpp11BracedListStyle: true DerivePointerAlignment: false -DisableFormat: false -ExperimentalAutoDetectBinPacking: false FixNamespaceComments: true -ForEachMacros: - - foreach - - Q_FOREACH - - BOOST_FOREACH -IncludeCategories: - - Regex: '^<.*\.h>' - Priority: 1 - - Regex: '^<.*' - Priority: 2 - - Regex: '.*' - Priority: 3 -IncludeIsMainRegex: '([-_](test|unittest))?$' IndentCaseLabels: true IndentWidth: 4 IndentWrappedFunctionNames: false -JavaScriptQuotes: Leave -JavaScriptWrapImports: true KeepEmptyLinesAtTheStartOfBlocks: true MacroBlockBegin: '' MacroBlockEnd: '' -MaxEmptyLinesToKeep: 2 +MaxEmptyLinesToKeep: 1 NamespaceIndentation: None -ObjCBlockIndentWidth: 2 -ObjCSpaceAfterProperty: false -ObjCSpaceBeforeProtocolList: false -PenaltyBreakAssignment: 2 -PenaltyBreakBeforeFirstCallParameter: 1 -PenaltyBreakComment: 300 -PenaltyBreakFirstLessLess: 120 -PenaltyBreakString: 1000 -PenaltyExcessCharacter: 1000000 -PenaltyReturnTypeOnItsOwnLine: 200 PointerAlignment: Left ReflowComments: true SortIncludes: true SortUsingDeclarations: true SpaceAfterCStyleCast: false -SpaceAfterTemplateKeyword: true SpaceBeforeAssignmentOperators: true SpaceBeforeParens: ControlStatements SpaceInEmptyParentheses: false -SpacesBeforeTrailingComments: 2 SpacesInAngles: false SpacesInContainerLiterals: true SpacesInCStyleCastParentheses: false SpacesInParentheses: false SpacesInSquareBrackets: false -Standard: Auto TabWidth: 4 UseTab: Never ... diff --git a/src/dummy_daos/daos.cc b/src/dummy_daos/daos.cc index 11fde5816..c8d367767 100644 --- a/src/dummy_daos/daos.cc +++ b/src/dummy_daos/daos.cc @@ -15,19 +15,19 @@ */ #include -#include #include -#include #include +#include +#include -#include "eckit/runtime/Main.h" -#include "eckit/filesystem/PathName.h" #include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" -#include "eckit/io/Length.h" +#include "eckit/filesystem/PathName.h" #include "eckit/io/FileHandle.h" -#include "eckit/types/UUID.h" +#include "eckit/io/Length.h" #include "eckit/log/TimeStamp.h" +#include "eckit/runtime/Main.h" +#include "eckit/types/UUID.h" #include "eckit/utils/MD5.h" #include "daos.h" @@ -36,25 +36,25 @@ using eckit::PathName; namespace { - void deldir(eckit::PathName& p) { - if (!p.exists()) { - return; - } +void deldir(eckit::PathName& p) { + if (!p.exists()) { + return; + } - std::vector files; - std::vector dirs; - p.children(files, dirs); + std::vector files; + std::vector dirs; + p.children(files, dirs); - for (auto& f : files) { - f.unlink(); - } - for (auto& d : dirs) { - deldir(d); - } + for (auto& f : files) { + f.unlink(); + } + for (auto& d : dirs) { + deldir(d); + } - p.rmdir(); - }; -} + p.rmdir(); +}; +} // namespace extern "C" { @@ -78,15 +78,19 @@ int daos_fini() { return 0; } -int daos_pool_connect(const char *pool, const char *sys, unsigned int flags, - daos_handle_t *poh, daos_pool_info_t *info, daos_event_t *ev) { +int daos_pool_connect(const char* pool, const char* sys, unsigned int flags, daos_handle_t* poh, daos_pool_info_t* info, + daos_event_t* ev) { poh->impl = nullptr; - if (sys != NULL) NOTIMP; - if (flags != DAOS_PC_RW) NOTIMP; - if (info != NULL) NOTIMP; - if (ev != NULL) NOTIMP; + if (sys != NULL) + NOTIMP; + if (flags != DAOS_PC_RW) + NOTIMP; + if (info != NULL) + NOTIMP; + if (ev != NULL) + NOTIMP; eckit::PathName path = dummy_daos_root() / pool; eckit::PathName realpath{dummy_daos_root()}; @@ -105,39 +109,38 @@ int daos_pool_connect(const char *pool, const char *sys, unsigned int flags, } catch (eckit::FailedSystemCall& e) { - if (path.exists()) throw; + if (path.exists()) + throw; return -1; - } - } - if (!realpath.exists()) return -1; + if (!realpath.exists()) + return -1; std::unique_ptr impl(new daos_handle_internal_t); impl->path = realpath; poh->impl = impl.release(); return 0; - } -int daos_pool_disconnect(daos_handle_t poh, daos_event_t *ev) { +int daos_pool_disconnect(daos_handle_t poh, daos_event_t* ev) { ASSERT(poh.impl); delete poh.impl; - if (ev != NULL) NOTIMP; + if (ev != NULL) + NOTIMP; return 0; - } -int daos_pool_list_cont(daos_handle_t poh, daos_size_t *ncont, - struct daos_pool_cont_info *cbuf, daos_event_t *ev) { +int daos_pool_list_cont(daos_handle_t poh, daos_size_t* ncont, struct daos_pool_cont_info* cbuf, daos_event_t* ev) { ASSERT(poh.impl); - if (ev != NULL) NOTIMP; + if (ev != NULL) + NOTIMP; daos_size_t n(*ncont); @@ -148,15 +151,17 @@ int daos_pool_list_cont(daos_handle_t poh, daos_size_t *ncont, *ncont = files.size(); - if (cbuf == NULL) return 0; + if (cbuf == NULL) + return 0; - if (files.size() > n) return -1; + if (files.size() > n) + return -1; daos_size_t nfound = 0; for (auto& f : files) { - if (f.exists()) { /// @todo: is the check in this line really necessary, given the try-catch below? + if (f.exists()) { /// @todo: is the check in this line really necessary, given the try-catch below? ++nfound; @@ -179,25 +184,23 @@ int daos_pool_list_cont(daos_handle_t poh, daos_size_t *ncont, } catch (eckit::FailedSystemCall& e) { - if (f.exists()) throw; + if (f.exists()) + throw; --nfound; - } } - } *ncont = nfound; return 0; - } -int daos_cont_create_internal(daos_handle_t poh, uuid_t *uuid) { +int daos_cont_create_internal(daos_handle_t poh, uuid_t* uuid) { ASSERT(poh.impl); - /// @note: name generation copied from LocalPathName::unique. Ditched StaticMutex + /// @note: name generation copied from LocalPathName::unique. Ditched StaticMutex /// as dummy DAOS is not thread safe. std::string hostname = eckit::Main::hostname(); @@ -228,44 +231,48 @@ int daos_cont_create_internal(daos_handle_t poh, uuid_t *uuid) { eckit::PathName cont_path = poh.impl->path / cont_uuid_cstr; - if (cont_path.exists()) throw eckit::SeriousBug("UUID clash in cont create"); + if (cont_path.exists()) + throw eckit::SeriousBug("UUID clash in cont create"); cont_path.mkdir(); - if (uuid != NULL) uuid_copy(*uuid, new_uuid); + if (uuid != NULL) + uuid_copy(*uuid, new_uuid); return 0; - } -/// @note: containers are implemented as directories within pool directories. Upon creation, a +/// @note: containers are implemented as directories within pool directories. Upon creation, a /// container directory is named with a newly generated UUID. If a label is specified, a /// symlink is created with the label as origin file name and the UUID as target directory. -/// If no label is specified, a similr symlink is created with the UUID with the +/// If no label is specified, a similr symlink is created with the UUID with the /// "__dummy_daos_uuid_" prefix as origin file name and the UUID as target directory. -/// This mechanism is necessary for listing and removing containers under concurrent, +/// This mechanism is necessary for listing and removing containers under concurrent, /// potentially racing container operations. -int daos_cont_create(daos_handle_t poh, uuid_t *uuid, daos_prop_t *cont_prop, daos_event_t *ev) { +int daos_cont_create(daos_handle_t poh, uuid_t* uuid, daos_prop_t* cont_prop, daos_event_t* ev) { ASSERT(poh.impl); if (cont_prop != NULL && cont_prop->dpp_entries) { - if (cont_prop->dpp_nr != 1) NOTIMP; - if (cont_prop->dpp_entries[0].dpe_type != DAOS_PROP_CO_LABEL) NOTIMP; + if (cont_prop->dpp_nr != 1) + NOTIMP; + if (cont_prop->dpp_entries[0].dpe_type != DAOS_PROP_CO_LABEL) + NOTIMP; - struct daos_prop_entry *entry = &cont_prop->dpp_entries[0]; + struct daos_prop_entry* entry = &cont_prop->dpp_entries[0]; - if (entry == NULL) NOTIMP; + if (entry == NULL) + NOTIMP; std::string cont_name{entry->dpe_str}; return daos_cont_create_with_label(poh, cont_name.c_str(), NULL, uuid, ev); - } - if (ev != NULL) NOTIMP; + if (ev != NULL) + NOTIMP; uuid_t new_uuid = {0}; @@ -280,38 +287,38 @@ int daos_cont_create(daos_handle_t poh, uuid_t *uuid, daos_prop_t *cont_prop, da if (::symlink(cont_path.path().c_str(), label_symlink_path.path().c_str()) < 0) { - if (errno == EEXIST) { // link path already exists due to race condition - + if (errno == EEXIST) { // link path already exists due to race condition + throw eckit::SeriousBug("unexpected race condition in unnamed container symlink creation"); - } else { // symlink fails for unknown reason + } else { // symlink fails for unknown reason throw eckit::FailedSystemCall(std::string("symlink ") + cont_path.path() + " " + label_symlink_path.path()); - } - } - if (uuid != NULL) uuid_copy(*uuid, new_uuid); + if (uuid != NULL) + uuid_copy(*uuid, new_uuid); return 0; - } -int daos_cont_create_with_label(daos_handle_t poh, const char *label, - daos_prop_t *cont_prop, uuid_t *uuid, - daos_event_t *ev) { +int daos_cont_create_with_label(daos_handle_t poh, const char* label, daos_prop_t* cont_prop, uuid_t* uuid, + daos_event_t* ev) { ASSERT(poh.impl); - if (cont_prop != NULL) NOTIMP; - if (ev != NULL) NOTIMP; + if (cont_prop != NULL) + NOTIMP; + if (ev != NULL) + NOTIMP; ASSERT(std::string{label}.rfind("__dummy_daos_uuid_", 0) != 0); ASSERT(strlen(label) <= DAOS_PROP_LABEL_MAX_LEN); eckit::PathName label_symlink_path = poh.impl->path / label; - if (label_symlink_path.exists()) return 0; + if (label_symlink_path.exists()) + return 0; uuid_t new_uuid = {0}; @@ -324,7 +331,7 @@ int daos_cont_create_with_label(daos_handle_t poh, const char *label, if (::symlink(cont_path.path().c_str(), label_symlink_path.path().c_str()) < 0) { - if (errno == EEXIST) { // link path already exists due to race condition + if (errno == EEXIST) { // link path already exists due to race condition if (uuid != NULL) { /// @todo: again might find race condition here: @@ -336,38 +343,38 @@ int daos_cont_create_with_label(daos_handle_t poh, const char *label, return 0; - } else { // symlink fails for unknown reason + } else { // symlink fails for unknown reason throw eckit::FailedSystemCall(std::string("symlink ") + cont_path.path() + " " + label_symlink_path.path()); - } - } - if (uuid != NULL) uuid_copy(*uuid, new_uuid); + if (uuid != NULL) + uuid_copy(*uuid, new_uuid); return 0; - } /// @note: in DAOS, an attempt to destroy a container with open handles results -/// in error by default. This behavior is not implemented in dummy DAOS. -/// In DAOS, an attempt to destroy a container with open handles with the -/// force flag enabled closes open handles, and therefore ongoing/future -/// operations on these handles fail. The contained objects and the -/// container are immediately destroyed. In dummy DAOS the open handles -/// are implemented with file descriptors, and these are left open. A -/// remove operation is called on the corresponding file names but the -/// descriptors remain open. Therefore, in contrast to DAOS, -/// ongoing/future operations on these handles succeed, and the files -/// are destroyed after the descriptors are closed. The folder +/// in error by default. This behavior is not implemented in dummy DAOS. +/// In DAOS, an attempt to destroy a container with open handles with the +/// force flag enabled closes open handles, and therefore ongoing/future +/// operations on these handles fail. The contained objects and the +/// container are immediately destroyed. In dummy DAOS the open handles +/// are implemented with file descriptors, and these are left open. A +/// remove operation is called on the corresponding file names but the +/// descriptors remain open. Therefore, in contrast to DAOS, +/// ongoing/future operations on these handles succeed, and the files +/// are destroyed after the descriptors are closed. The folder /// implementing the container, however, is immediately removed. -int daos_cont_destroy(daos_handle_t poh, const char *cont, int force, daos_event_t *ev) { +int daos_cont_destroy(daos_handle_t poh, const char* cont, int force, daos_event_t* ev) { ASSERT(poh.impl); - if (force != 1) NOTIMP; - if (ev != NULL) NOTIMP; + if (force != 1) + NOTIMP; + if (ev != NULL) + NOTIMP; ASSERT(std::string{cont}.rfind("__dummy_daos_uuid_", 0) != 0); @@ -388,9 +395,9 @@ int daos_cont_destroy(daos_handle_t poh, const char *cont, int force, daos_event } catch (eckit::FailedSystemCall& e) { - if (path.exists()) throw; + if (path.exists()) + throw; return -DER_NONEXIST; - } try { @@ -399,22 +406,24 @@ int daos_cont_destroy(daos_handle_t poh, const char *cont, int force, daos_event } catch (eckit::FailedSystemCall& e) { - if (realpath.exists()) throw; + if (realpath.exists()) + throw; return -DER_NONEXIST; - } return 0; - } -int daos_cont_open(daos_handle_t poh, const char *cont, unsigned int flags, daos_handle_t *coh, - daos_cont_info_t *info, daos_event_t *ev) { +int daos_cont_open(daos_handle_t poh, const char* cont, unsigned int flags, daos_handle_t* coh, daos_cont_info_t* info, + daos_event_t* ev) { ASSERT(poh.impl); - if (flags != DAOS_COO_RW) NOTIMP; - if (info != NULL) NOTIMP; - if (ev != NULL) NOTIMP; + if (flags != DAOS_COO_RW) + NOTIMP; + if (info != NULL) + NOTIMP; + if (ev != NULL) + NOTIMP; ASSERT(std::string{cont}.rfind("__dummy_daos_uuid_", 0) != 0); @@ -426,7 +435,8 @@ int daos_cont_open(daos_handle_t poh, const char *cont, unsigned int flags, daos path /= cont; } - if (!path.exists()) return -DER_NONEXIST; + if (!path.exists()) + return -DER_NONEXIST; eckit::PathName realpath{poh.impl->path}; try { @@ -436,42 +446,42 @@ int daos_cont_open(daos_handle_t poh, const char *cont, unsigned int flags, daos } catch (eckit::FailedSystemCall& e) { - if (path.exists()) throw; + if (path.exists()) + throw; return -DER_NONEXIST; - } - if (!realpath.exists()) return -DER_NONEXIST; + if (!realpath.exists()) + return -DER_NONEXIST; std::unique_ptr impl(new daos_handle_internal_t); impl->path = realpath; coh->impl = impl.release(); return 0; - } -int daos_cont_close(daos_handle_t coh, daos_event_t *ev) { +int daos_cont_close(daos_handle_t coh, daos_event_t* ev) { ASSERT(coh.impl); delete coh.impl; - if (ev != NULL) NOTIMP; + if (ev != NULL) + NOTIMP; return 0; - } -int daos_cont_alloc_oids(daos_handle_t coh, daos_size_t num_oids, uint64_t *oid, - daos_event_t *ev) { +int daos_cont_alloc_oids(daos_handle_t coh, daos_size_t num_oids, uint64_t* oid, daos_event_t* ev) { static uint64_t next_oid = 0; ASSERT(coh.impl); - if (ev != NULL) NOTIMP; - ASSERT(num_oids > (uint64_t) 0); + if (ev != NULL) + NOTIMP; + ASSERT(num_oids > (uint64_t)0); - // support for multi-node clients running dummy DAOS backed by a + // support for multi-node clients running dummy DAOS backed by a // distributed file system std::string host = eckit::Main::instance().hostname(); @@ -495,42 +505,44 @@ int daos_cont_alloc_oids(daos_handle_t coh, daos_size_t num_oids, uint64_t *oid, ASSERT((next_oid + num_oids) <= oid_mask); *oid = next_oid; - *oid |= (((uint64_t) pid) & pid_mask) << 28; - *oid |= (((uint64_t) *(((unsigned char *) uuid) + 1)) << 48); - *oid |= (((uint64_t) *((unsigned char *) uuid)) << 56); + *oid |= (((uint64_t)pid) & pid_mask) << 28; + *oid |= (((uint64_t)*(((unsigned char*)uuid) + 1)) << 48); + *oid |= (((uint64_t)*((unsigned char*)uuid)) << 56); next_oid += num_oids; return 0; - } -int daos_obj_generate_oid(daos_handle_t coh, daos_obj_id_t *oid, - enum daos_otype_t type, daos_oclass_id_t cid, +int daos_obj_generate_oid(daos_handle_t coh, daos_obj_id_t* oid, enum daos_otype_t type, daos_oclass_id_t cid, daos_oclass_hints_t hints, uint32_t args) { ASSERT(coh.impl); - if (type != DAOS_OT_KV_HASHED && type != DAOS_OT_ARRAY && type != DAOS_OT_ARRAY_BYTE) NOTIMP; - if (cid != OC_S1) NOTIMP; - if (hints != 0) NOTIMP; - if (args != 0) NOTIMP; - - oid->hi &= (uint64_t) 0x00000000FFFFFFFF; - oid->hi |= ((((uint64_t) type) & OID_FMT_TYPE_MAX) << OID_FMT_TYPE_SHIFT); - oid->hi |= ((((uint64_t) cid) & OID_FMT_CLASS_MAX) << OID_FMT_CLASS_SHIFT); + if (type != DAOS_OT_KV_HASHED && type != DAOS_OT_ARRAY && type != DAOS_OT_ARRAY_BYTE) + NOTIMP; + if (cid != OC_S1) + NOTIMP; + if (hints != 0) + NOTIMP; + if (args != 0) + NOTIMP; + + oid->hi &= (uint64_t)0x00000000FFFFFFFF; + oid->hi |= ((((uint64_t)type) & OID_FMT_TYPE_MAX) << OID_FMT_TYPE_SHIFT); + oid->hi |= ((((uint64_t)cid) & OID_FMT_CLASS_MAX) << OID_FMT_CLASS_SHIFT); return 0; - } /// @note: real daos_kv_open does not involve RPC whereas dummy daos requires /// interaction with the file system -int daos_kv_open(daos_handle_t coh, daos_obj_id_t oid, unsigned int mode, - daos_handle_t *oh, daos_event_t *ev) { +int daos_kv_open(daos_handle_t coh, daos_obj_id_t oid, unsigned int mode, daos_handle_t* oh, daos_event_t* ev) { ASSERT(coh.impl); - if (mode != DAOS_OO_RW) NOTIMP; - if (ev != NULL) NOTIMP; + if (mode != DAOS_OO_RW) + NOTIMP; + if (ev != NULL) + NOTIMP; std::stringstream os; os << std::setw(16) << std::setfill('0') << std::hex << oid.hi; @@ -544,17 +556,18 @@ int daos_kv_open(daos_handle_t coh, daos_obj_id_t oid, unsigned int mode, oh->impl = impl.release(); return 0; - } /// @note: destruction of KVs with open handles may not be consistent. Notes /// in daos_cont_destroy apply here too. -int daos_kv_destroy(daos_handle_t oh, daos_handle_t th, daos_event_t *ev) { +int daos_kv_destroy(daos_handle_t oh, daos_handle_t th, daos_event_t* ev) { ASSERT(oh.impl); - if (th.impl != DAOS_TX_NONE.impl) NOTIMP; - if (ev != NULL) NOTIMP; + if (th.impl != DAOS_TX_NONE.impl) + NOTIMP; + if (ev != NULL) + NOTIMP; try { @@ -562,76 +575,84 @@ int daos_kv_destroy(daos_handle_t oh, daos_handle_t th, daos_event_t *ev) { } catch (eckit::FailedSystemCall& e) { - if (oh.impl->path.exists()) throw; + if (oh.impl->path.exists()) + throw; return -DER_NONEXIST; - } return 0; - } -int daos_obj_close(daos_handle_t oh, daos_event_t *ev) { +int daos_obj_close(daos_handle_t oh, daos_event_t* ev) { ASSERT(oh.impl); delete oh.impl; - if (ev != NULL) NOTIMP; + if (ev != NULL) + NOTIMP; return 0; - } // daos_kv_put and get are only guaranteed to work if values of a same fixed size are put/get. -// e.g. two racing processes could both openForWrite as part of daos_kv_put (and wipe file -// content), then one writes (puts) content of length 2*x, the other writes content of +// e.g. two racing processes could both openForWrite as part of daos_kv_put (and wipe file +// content), then one writes (puts) content of length 2*x, the other writes content of // length x, but old content remains at the end from the first kv_put. -// e.g. a process could openForRead as part of daos_kv_get (which retrieves content length), -// then another raching process could daos_kv_put of some content with different length, +// e.g. a process could openForRead as part of daos_kv_get (which retrieves content length), +// then another raching process could daos_kv_put of some content with different length, // and then the first process would resume retrieval and obtain content of unexpected length. // if so, daos_kv_put and get are transactional -int daos_kv_put(daos_handle_t oh, daos_handle_t th, uint64_t flags, const char *key, - daos_size_t size, const void *buf, daos_event_t *ev) { +int daos_kv_put(daos_handle_t oh, daos_handle_t th, uint64_t flags, const char* key, daos_size_t size, const void* buf, + daos_event_t* ev) { ASSERT(oh.impl); - if (th.impl != DAOS_TX_NONE.impl) NOTIMP; - if (flags != 0) NOTIMP; - if (ev != NULL) NOTIMP; + if (th.impl != DAOS_TX_NONE.impl) + NOTIMP; + if (flags != 0) + NOTIMP; + if (ev != NULL) + NOTIMP; eckit::FileHandle fh(oh.impl->path / key, true); fh.openForWrite(eckit::Length(size)); eckit::AutoClose closer(fh); - long res = fh.write(buf, (long) size); - ASSERT(res == (long) size); + long res = fh.write(buf, (long)size); + ASSERT(res == (long)size); return 0; - } -int daos_kv_get(daos_handle_t oh, daos_handle_t th, uint64_t flags, const char *key, - daos_size_t *size, void *buf, daos_event_t *ev) { +int daos_kv_get(daos_handle_t oh, daos_handle_t th, uint64_t flags, const char* key, daos_size_t* size, void* buf, + daos_event_t* ev) { ASSERT(oh.impl); - if (th.impl != DAOS_TX_NONE.impl) NOTIMP; - if (flags != 0) NOTIMP; - if (ev != NULL) NOTIMP; + if (th.impl != DAOS_TX_NONE.impl) + NOTIMP; + if (flags != 0) + NOTIMP; + if (ev != NULL) + NOTIMP; bool exists = (oh.impl->path / key).exists(); - if (!exists && buf != NULL) return -DER_NONEXIST; + if (!exists && buf != NULL) + return -DER_NONEXIST; daos_size_t dest_size = *size; *size = 0; - if (!exists) return 0; + if (!exists) + return 0; eckit::FileHandle fh(oh.impl->path / key); eckit::Length len = fh.size(); *size = len; - if (buf == NULL) return 0; + if (buf == NULL) + return 0; - if (len > dest_size) return -1; + if (len > dest_size) + return -1; fh.openForRead(); eckit::AutoClose closer(fh); @@ -639,61 +660,75 @@ int daos_kv_get(daos_handle_t oh, daos_handle_t th, uint64_t flags, const char * ASSERT(eckit::Length(res) == len); return 0; - } -int daos_kv_remove(daos_handle_t oh, daos_handle_t th, uint64_t flags, - const char *key, daos_event_t *ev) { +int daos_kv_remove(daos_handle_t oh, daos_handle_t th, uint64_t flags, const char* key, daos_event_t* ev) { ASSERT(oh.impl); - if (th.impl != DAOS_TX_NONE.impl) NOTIMP; - if (flags != 0) NOTIMP; - if (ev != NULL) NOTIMP; - - if (!oh.impl->path.exists()) return -1; + if (th.impl != DAOS_TX_NONE.impl) + NOTIMP; + if (flags != 0) + NOTIMP; + if (ev != NULL) + NOTIMP; + + if (!oh.impl->path.exists()) + return -1; /// @todo: should removal of a non-existing key fail? /// @todo: if not, can the exist check be avoided and unlink be called directly? - if ((oh.impl->path / key).exists()) (oh.impl->path / key).unlink(); + if ((oh.impl->path / key).exists()) + (oh.impl->path / key).unlink(); return 0; - } -int daos_kv_list(daos_handle_t oh, daos_handle_t th, uint32_t *nr, - daos_key_desc_t *kds, d_sg_list_t *sgl, daos_anchor_t *anchor, - daos_event_t *ev) { +int daos_kv_list(daos_handle_t oh, daos_handle_t th, uint32_t* nr, daos_key_desc_t* kds, d_sg_list_t* sgl, + daos_anchor_t* anchor, daos_event_t* ev) { ASSERT(oh.impl); static std::vector ongoing_req; static std::string req_hash; static unsigned long long n = (((unsigned long long)::getpid()) << 32); - if (th.impl != DAOS_TX_NONE.impl) NOTIMP; - if (ev != NULL) NOTIMP; + if (th.impl != DAOS_TX_NONE.impl) + NOTIMP; + if (ev != NULL) + NOTIMP; + + if (nr == NULL) + return -1; + if (kds == NULL) + return -1; + if (sgl == NULL) + return -1; + if (sgl->sg_nr != 1) + NOTIMP; + if (sgl->sg_iovs == NULL) + return -1; + if (anchor == NULL) + return -1; - if (nr == NULL) return -1; - if (kds == NULL) return -1; - if (sgl == NULL) return -1; - if (sgl->sg_nr != 1) NOTIMP; - if (sgl->sg_iovs == NULL) return -1; - if (anchor == NULL) return -1; + if (!oh.impl->path.exists()) + return -1; - if (!oh.impl->path.exists()) return -1; + if (anchor->da_type == DAOS_ANCHOR_TYPE_EOF) + return -1; + if (anchor->da_type == DAOS_ANCHOR_TYPE_HKEY) + NOTIMP; - if (anchor->da_type == DAOS_ANCHOR_TYPE_EOF) return -1; - if (anchor->da_type == DAOS_ANCHOR_TYPE_HKEY) NOTIMP; - if (anchor->da_type == DAOS_ANCHOR_TYPE_ZERO) { /// client process must consume all key names before starting a new request - if (ongoing_req.size() != 0) NOTIMP; + if (ongoing_req.size() != 0) + NOTIMP; std::vector files; std::vector dirs; oh.impl->path.children(files, dirs); - for (auto& f : files) ongoing_req.push_back(f.baseName()); + for (auto& f : files) + ongoing_req.push_back(f.baseName()); anchor->da_type = DAOS_ANCHOR_TYPE_KEY; @@ -711,18 +746,18 @@ int daos_kv_list(daos_handle_t oh, daos_handle_t th, uint32_t *nr, eckit::MD5 md5(name); req_hash = md5.digest(); - ::memcpy((char*) &(anchor->da_buf[0]), req_hash.c_str(), req_hash.size()); - anchor->da_shard = (uint16_t) req_hash.size(); + ::memcpy((char*)&(anchor->da_buf[0]), req_hash.c_str(), req_hash.size()); + anchor->da_shard = (uint16_t)req_hash.size(); } else { if (anchor->da_type != DAOS_ANCHOR_TYPE_KEY) throw eckit::SeriousBug("Unexpected anchor type"); - /// different processes cannot collaborate on consuming a same kv_list + /// different processes cannot collaborate on consuming a same kv_list /// request (i.e. cannot share a hash) - if (std::string((char*) &(anchor->da_buf[0]), anchor->da_shard) != req_hash) NOTIMP; - + if (std::string((char*)&(anchor->da_buf[0]), anchor->da_shard) != req_hash) + NOTIMP; } size_t remain_size = sgl->sg_iovs[0].iov_buf_len; @@ -732,11 +767,12 @@ int daos_kv_list(daos_handle_t oh, daos_handle_t th, uint32_t *nr, while (remain_kds > 0 && remain_size > 0 && ongoing_req.size() > 0) { size_t next_size = ongoing_req.back().size(); if (next_size > remain_size) { - if (*nr == 0) return -1; + if (*nr == 0) + return -1; remain_size = 0; continue; } - ::memcpy((char*) sgl->sg_iovs[0].iov_buf + sgl_pos, ongoing_req.back().c_str(), next_size); + ::memcpy((char*)sgl->sg_iovs[0].iov_buf + sgl_pos, ongoing_req.back().c_str(), next_size); ongoing_req.pop_back(); kds[*nr].kd_key_len = next_size; remain_size -= next_size; @@ -745,13 +781,13 @@ int daos_kv_list(daos_handle_t oh, daos_handle_t th, uint32_t *nr, *nr += 1; } - if (ongoing_req.size() == 0) anchor->da_type = DAOS_ANCHOR_TYPE_EOF; + if (ongoing_req.size() == 0) + anchor->da_type = DAOS_ANCHOR_TYPE_EOF; return 0; - } -int daos_array_generate_oid(daos_handle_t coh, daos_obj_id_t *oid, bool add_attr, daos_oclass_id_t cid, +int daos_array_generate_oid(daos_handle_t coh, daos_obj_id_t* oid, bool add_attr, daos_oclass_id_t cid, daos_oclass_hints_t hints, uint32_t args) { if (add_attr) { @@ -761,18 +797,17 @@ int daos_array_generate_oid(daos_handle_t coh, daos_obj_id_t *oid, bool add_attr } else { return daos_obj_generate_oid(coh, oid, DAOS_OT_ARRAY_BYTE, cid, hints, args); - } - } -int daos_array_create(daos_handle_t coh, daos_obj_id_t oid, daos_handle_t th, - daos_size_t cell_size, daos_size_t chunk_size, - daos_handle_t *oh, daos_event_t *ev) { +int daos_array_create(daos_handle_t coh, daos_obj_id_t oid, daos_handle_t th, daos_size_t cell_size, + daos_size_t chunk_size, daos_handle_t* oh, daos_event_t* ev) { ASSERT(coh.impl); - if (th.impl != DAOS_TX_NONE.impl) NOTIMP; - if (ev != NULL) NOTIMP; + if (th.impl != DAOS_TX_NONE.impl) + NOTIMP; + if (ev != NULL) + NOTIMP; std::stringstream os; os << std::setw(16) << std::setfill('0') << std::hex << oid.hi; @@ -786,14 +821,15 @@ int daos_array_create(daos_handle_t coh, daos_obj_id_t oid, daos_handle_t th, oh->impl = impl.release(); return 0; - } -int daos_array_destroy(daos_handle_t oh, daos_handle_t th, daos_event_t *ev) { +int daos_array_destroy(daos_handle_t oh, daos_handle_t th, daos_event_t* ev) { ASSERT(oh.impl); - if (th.impl != DAOS_TX_NONE.impl) NOTIMP; - if (ev != NULL) NOTIMP; + if (th.impl != DAOS_TX_NONE.impl) + NOTIMP; + if (ev != NULL) + NOTIMP; try { @@ -801,26 +837,27 @@ int daos_array_destroy(daos_handle_t oh, daos_handle_t th, daos_event_t *ev) { } catch (eckit::FailedSystemCall& e) { - if (oh.impl->path.exists()) throw; + if (oh.impl->path.exists()) + throw; return -DER_NONEXIST; - } return 0; - } -int daos_array_open(daos_handle_t coh, daos_obj_id_t oid, daos_handle_t th, - unsigned int mode, daos_size_t *cell_size, - daos_size_t *chunk_size, daos_handle_t *oh, daos_event_t *ev) { +int daos_array_open(daos_handle_t coh, daos_obj_id_t oid, daos_handle_t th, unsigned int mode, daos_size_t* cell_size, + daos_size_t* chunk_size, daos_handle_t* oh, daos_event_t* ev) { ASSERT(coh.impl); - if (th.impl != DAOS_TX_NONE.impl) NOTIMP; - if (mode != DAOS_OO_RW) NOTIMP; - if (ev != NULL) NOTIMP; + if (th.impl != DAOS_TX_NONE.impl) + NOTIMP; + if (mode != DAOS_OO_RW) + NOTIMP; + if (ev != NULL) + NOTIMP; *cell_size = 1; - *chunk_size = (uint64_t) 1048576; + *chunk_size = (uint64_t)1048576; std::stringstream os; os << std::setw(16) << std::setfill('0') << std::hex << oid.hi; @@ -836,61 +873,68 @@ int daos_array_open(daos_handle_t coh, daos_obj_id_t oid, daos_handle_t th, oh->impl = impl.release(); return 0; - } -int daos_array_open_with_attr(daos_handle_t coh, daos_obj_id_t oid, daos_handle_t th, - unsigned int mode, daos_size_t cell_size, daos_size_t chunk_size, - daos_handle_t *oh, daos_event_t *ev) { +int daos_array_open_with_attr(daos_handle_t coh, daos_obj_id_t oid, daos_handle_t th, unsigned int mode, + daos_size_t cell_size, daos_size_t chunk_size, daos_handle_t* oh, daos_event_t* ev) { ASSERT(coh.impl); - if (th.impl != DAOS_TX_NONE.impl) NOTIMP; - if (mode != DAOS_OO_RW) NOTIMP; - if (ev != NULL) NOTIMP; - - if (cell_size != 1) NOTIMP; - if (chunk_size != (uint64_t) 1048576) NOTIMP; + if (th.impl != DAOS_TX_NONE.impl) + NOTIMP; + if (mode != DAOS_OO_RW) + NOTIMP; + if (ev != NULL) + NOTIMP; + + if (cell_size != 1) + NOTIMP; + if (chunk_size != (uint64_t)1048576) + NOTIMP; return daos_array_create(coh, oid, th, cell_size, chunk_size, oh, ev); - } -int daos_array_close(daos_handle_t oh, daos_event_t *ev) { +int daos_array_close(daos_handle_t oh, daos_event_t* ev) { ASSERT(oh.impl); delete oh.impl; - if (ev != NULL) NOTIMP; + if (ev != NULL) + NOTIMP; return 0; - } -// daos_array_write and read have the same limitations and transactional behavior as +// daos_array_write and read have the same limitations and transactional behavior as // daos_kv_put and get -int daos_array_write(daos_handle_t oh, daos_handle_t th, daos_array_iod_t *iod, - d_sg_list_t *sgl, daos_event_t *ev) { +int daos_array_write(daos_handle_t oh, daos_handle_t th, daos_array_iod_t* iod, d_sg_list_t* sgl, daos_event_t* ev) { ASSERT(oh.impl); - if (th.impl != DAOS_TX_NONE.impl) NOTIMP; - if (ev != NULL) NOTIMP; + if (th.impl != DAOS_TX_NONE.impl) + NOTIMP; + if (ev != NULL) + NOTIMP; - if (iod->arr_nr != 1) NOTIMP; + if (iod->arr_nr != 1) + NOTIMP; - if (sgl->sg_nr != 1) NOTIMP; + if (sgl->sg_nr != 1) + NOTIMP; // source memory len - if (sgl->sg_iovs[0].iov_buf_len != sgl->sg_iovs[0].iov_len) NOTIMP; + if (sgl->sg_iovs[0].iov_buf_len != sgl->sg_iovs[0].iov_len) + NOTIMP; // source memory vs. target object len - if (sgl->sg_iovs[0].iov_buf_len != iod->arr_rgs[0].rg_len) NOTIMP; + if (sgl->sg_iovs[0].iov_buf_len != iod->arr_rgs[0].rg_len) + NOTIMP; - //sgl->sg_iovs[0].iov_buf is a void * with the data to write - //sgl->sg_iovs[0].iov_buf_len is a size_t with the source len - //iod->arr_rgs[0].rg_idx is a uint64_t with the offset to write from + // sgl->sg_iovs[0].iov_buf is a void * with the data to write + // sgl->sg_iovs[0].iov_buf_len is a size_t with the source len + // iod->arr_rgs[0].rg_idx is a uint64_t with the offset to write from eckit::FileHandle fh(oh.impl->path, true); - //eckit::Length existing_len = fh.size(); + // eckit::Length existing_len = fh.size(); // if writing data to an already existing and populated file, if the data to write // is smaller than the file or the data has an offset, the holes will be left with @@ -899,48 +943,53 @@ int daos_array_write(daos_handle_t oh, daos_handle_t th, daos_array_iod_t *iod, fh.openForAppend(eckit::Length(sgl->sg_iovs[0].iov_buf_len)); eckit::AutoClose closer(fh); fh.seek(iod->arr_rgs[0].rg_idx); - long res = fh.write(sgl->sg_iovs[0].iov_buf, (long) sgl->sg_iovs[0].iov_buf_len); - ASSERT(res == (long) sgl->sg_iovs[0].iov_buf_len); + long res = fh.write(sgl->sg_iovs[0].iov_buf, (long)sgl->sg_iovs[0].iov_buf_len); + ASSERT(res == (long)sgl->sg_iovs[0].iov_buf_len); return 0; - } -int daos_array_get_size(daos_handle_t oh, daos_handle_t th, daos_size_t *size, - daos_event_t *ev) { +int daos_array_get_size(daos_handle_t oh, daos_handle_t th, daos_size_t* size, daos_event_t* ev) { ASSERT(oh.impl); - if (th.impl != DAOS_TX_NONE.impl) NOTIMP; - if (ev != NULL) NOTIMP; + if (th.impl != DAOS_TX_NONE.impl) + NOTIMP; + if (ev != NULL) + NOTIMP; - *size = (daos_size_t) oh.impl->path.size(); + *size = (daos_size_t)oh.impl->path.size(); return 0; - } -int daos_array_read(daos_handle_t oh, daos_handle_t th, daos_array_iod_t *iod, - d_sg_list_t *sgl, daos_event_t *ev) { +int daos_array_read(daos_handle_t oh, daos_handle_t th, daos_array_iod_t* iod, d_sg_list_t* sgl, daos_event_t* ev) { ASSERT(oh.impl); - if (th.impl != DAOS_TX_NONE.impl) NOTIMP; - if (ev != NULL) NOTIMP; + if (th.impl != DAOS_TX_NONE.impl) + NOTIMP; + if (ev != NULL) + NOTIMP; - if (iod->arr_nr != 1) NOTIMP; + if (iod->arr_nr != 1) + NOTIMP; - if (sgl->sg_nr != 1) NOTIMP; + if (sgl->sg_nr != 1) + NOTIMP; // target memory len - if (sgl->sg_iovs[0].iov_buf_len != sgl->sg_iovs[0].iov_len) NOTIMP; + if (sgl->sg_iovs[0].iov_buf_len != sgl->sg_iovs[0].iov_len) + NOTIMP; // target memory vs. source object len - if (sgl->sg_iovs[0].iov_buf_len != iod->arr_rgs[0].rg_len) NOTIMP; + if (sgl->sg_iovs[0].iov_buf_len != iod->arr_rgs[0].rg_len) + NOTIMP; - //sgl->sg_iovs[0].iov_buf is a void * where to read the data into - //iod->arr_rgs[0].rg_len is a size_t with the source size - //iod->arr_rgs[0].rg_idx is a uint64_t with the offset to read from + // sgl->sg_iovs[0].iov_buf is a void * where to read the data into + // iod->arr_rgs[0].rg_len is a size_t with the source size + // iod->arr_rgs[0].rg_idx is a uint64_t with the offset to read from eckit::FileHandle fh(oh.impl->path); eckit::Length len = fh.size(); - if (iod->arr_rgs[0].rg_len + iod->arr_rgs[0].rg_idx > len) return -1; + if (iod->arr_rgs[0].rg_len + iod->arr_rgs[0].rg_idx > len) + return -1; fh.openForRead(); eckit::AutoClose closer(fh); fh.seek(iod->arr_rgs[0].rg_idx); @@ -948,16 +997,18 @@ int daos_array_read(daos_handle_t oh, daos_handle_t th, daos_array_iod_t *iod, ASSERT(eckit::Length(res) == eckit::Length(iod->arr_rgs[0].rg_len)); return 0; - } -int daos_cont_create_snap_opt(daos_handle_t coh, daos_epoch_t *epoch, char *name, - enum daos_snapshot_opts opts, daos_event_t *ev) { +int daos_cont_create_snap_opt(daos_handle_t coh, daos_epoch_t* epoch, char* name, enum daos_snapshot_opts opts, + daos_event_t* ev) { ASSERT(coh.impl); - if (name != NULL) NOTIMP; - if (opts != (DAOS_SNAP_OPT_CR | DAOS_SNAP_OPT_OIT)) NOTIMP; - if (ev != NULL) NOTIMP; + if (name != NULL) + NOTIMP; + if (opts != (DAOS_SNAP_OPT_CR | DAOS_SNAP_OPT_OIT)) + NOTIMP; + if (ev != NULL) + NOTIMP; std::vector files; std::vector dirs; @@ -972,13 +1023,13 @@ int daos_cont_create_snap_opt(daos_handle_t coh, daos_epoch_t *epoch, char *name std::string oid = it->baseName(); - if (strstr(oid.c_str(), ".snap")) continue; + if (strstr(oid.c_str(), ".snap")) + continue; ASSERT(oid.length() == 33); oids += sep + oid; sep = ","; - } } @@ -1001,38 +1052,38 @@ int daos_cont_create_snap_opt(daos_handle_t coh, daos_epoch_t *epoch, char *name *epoch = std::stoull(ts, nullptr, 16); return 0; - } -int daos_cont_destroy_snap(daos_handle_t coh, daos_epoch_range_t epr, - daos_event_t *ev) { +int daos_cont_destroy_snap(daos_handle_t coh, daos_epoch_range_t epr, daos_event_t* ev) { ASSERT(coh.impl); - if (epr.epr_hi != epr.epr_lo) NOTIMP; - if (ev != NULL) NOTIMP; + if (epr.epr_hi != epr.epr_lo) + NOTIMP; + if (ev != NULL) + NOTIMP; std::stringstream os; os << std::setw(16) << std::setfill('0') << std::hex << epr.epr_hi; - + eckit::PathName snap = coh.impl->path / os.str() + ".snap"; - - if (!snap.exists()) return -1; + + if (!snap.exists()) + return -1; snap.unlink(); return 0; - } -int daos_oit_open(daos_handle_t coh, daos_epoch_t epoch, - daos_handle_t *oh, daos_event_t *ev) { +int daos_oit_open(daos_handle_t coh, daos_epoch_t epoch, daos_handle_t* oh, daos_event_t* ev) { ASSERT(coh.impl); - if (ev != NULL) NOTIMP; + if (ev != NULL) + NOTIMP; std::stringstream os; os << std::setw(16) << std::setfill('0') << std::hex << epoch; - + std::string ts = os.str(); std::unique_ptr impl(new daos_handle_internal_t); @@ -1044,43 +1095,49 @@ int daos_oit_open(daos_handle_t coh, daos_epoch_t epoch, oh->impl = impl.release(); return 0; - } -int daos_oit_close(daos_handle_t oh, daos_event_t *ev) { +int daos_oit_close(daos_handle_t oh, daos_event_t* ev) { ASSERT(oh.impl); delete oh.impl; - if (ev != NULL) NOTIMP; + if (ev != NULL) + NOTIMP; return 0; - } -int daos_oit_list(daos_handle_t oh, daos_obj_id_t *oids, uint32_t *oids_nr, - daos_anchor_t *anchor, daos_event_t *ev) { +int daos_oit_list(daos_handle_t oh, daos_obj_id_t* oids, uint32_t* oids_nr, daos_anchor_t* anchor, daos_event_t* ev) { static std::vector ongoing_req; static std::string req_hash; static unsigned long long n = (((unsigned long long)::getpid()) << 32); ASSERT(oh.impl); - if (ev != NULL) NOTIMP; + if (ev != NULL) + NOTIMP; - if (oids_nr == NULL) return -1; - if (oids == NULL) return -1; - if (anchor == NULL) return -1; + if (oids_nr == NULL) + return -1; + if (oids == NULL) + return -1; + if (anchor == NULL) + return -1; - if (!oh.impl->path.exists()) return -1; + if (!oh.impl->path.exists()) + return -1; + + if (anchor->da_type == DAOS_ANCHOR_TYPE_EOF) + return -1; + if (anchor->da_type == DAOS_ANCHOR_TYPE_HKEY) + NOTIMP; - if (anchor->da_type == DAOS_ANCHOR_TYPE_EOF) return -1; - if (anchor->da_type == DAOS_ANCHOR_TYPE_HKEY) NOTIMP; - if (anchor->da_type == DAOS_ANCHOR_TYPE_ZERO) { /// client process must consume all key names before starting a new request - if (ongoing_req.size() != 0) NOTIMP; + if (ongoing_req.size() != 0) + NOTIMP; eckit::Length size(oh.impl->path.size()); std::vector v(size); @@ -1112,18 +1169,18 @@ int daos_oit_list(daos_handle_t oh, daos_obj_id_t *oids, uint32_t *oids_nr, eckit::MD5 md5(name); req_hash = md5.digest(); - ::memcpy((char*) &(anchor->da_buf[0]), req_hash.c_str(), req_hash.size()); - anchor->da_shard = (uint16_t) req_hash.size(); + ::memcpy((char*)&(anchor->da_buf[0]), req_hash.c_str(), req_hash.size()); + anchor->da_shard = (uint16_t)req_hash.size(); } else { if (anchor->da_type != DAOS_ANCHOR_TYPE_KEY) throw eckit::SeriousBug("Unexpected anchor type"); - /// different processes cannot collaborate on consuming a same kv_list + /// different processes cannot collaborate on consuming a same kv_list /// request (i.e. cannot share a hash) - if (std::string((char*) &(anchor->da_buf[0]), anchor->da_shard) != req_hash) NOTIMP; - + if (std::string((char*)&(anchor->da_buf[0]), anchor->da_shard) != req_hash) + NOTIMP; } uint32_t remain_oids = *oids_nr; @@ -1132,23 +1189,21 @@ int daos_oit_list(daos_handle_t oh, daos_obj_id_t *oids, uint32_t *oids_nr, std::string next_oid = ongoing_req.back(); - oids[*oids_nr] = daos_obj_id_t{ - std::stoull(next_oid.substr(17, 16), nullptr, 16), - std::stoull(next_oid.substr(0, 16), nullptr, 16) - }; + oids[*oids_nr] = daos_obj_id_t{std::stoull(next_oid.substr(17, 16), nullptr, 16), + std::stoull(next_oid.substr(0, 16), nullptr, 16)}; ongoing_req.pop_back(); - + remain_oids--; *oids_nr += 1; } - if (ongoing_req.size() == 0) anchor->da_type = DAOS_ANCHOR_TYPE_EOF; + if (ongoing_req.size() == 0) + anchor->da_type = DAOS_ANCHOR_TYPE_EOF; return 0; - } //---------------------------------------------------------------------------------------------------------------------- -} // extern "C" +} // extern "C" diff --git a/src/dummy_daos/daos.h b/src/dummy_daos/daos.h index 201c42d80..b8f7a8f75 100644 --- a/src/dummy_daos/daos.h +++ b/src/dummy_daos/daos.h @@ -16,11 +16,11 @@ #pragma once -#include #include +#include +#include #include #include -#include //---------------------------------------------------------------------------------------------------------------------- @@ -55,17 +55,16 @@ #define OBJ_CLASS_DEF(redun, grp_nr) ((redun << OC_REDUN_SHIFT) | grp_nr) #define MAX_NUM_GROUPS ((1 << 16UL) - 1) -#define DAOS_TX_NONE (daos_handle_t){NULL} +#define DAOS_TX_NONE \ + (daos_handle_t) { \ + NULL \ + } #define DAOS_PROP_LABEL_MAX_LEN (127) #define DAOS_PROP_ENTRIES_MAX_NR (128) #define DAOS_ANCHOR_BUF_MAX 104 -#define DAOS_ANCHOR_INIT { \ - .da_type = DAOS_ANCHOR_TYPE_ZERO, \ - .da_shard = 0, \ - .da_flags = 0, \ - .da_sub_anchors = 0, \ - .da_buf = { 0 } } +#define DAOS_ANCHOR_INIT \ + {.da_type = DAOS_ANCHOR_TYPE_ZERO, .da_shard = 0, .da_flags = 0, .da_sub_anchors = 0, .da_buf = {0}} #define DER_EXIST 1004 #define DER_NONEXIST 1005 @@ -78,19 +77,11 @@ enum daos_otype_t { DAOS_OT_ARRAY_BYTE = 13, }; -enum daos_pool_props { - DAOS_PROP_PO_LABEL, - DAOS_PROP_CO_LABEL -}; +enum daos_pool_props { DAOS_PROP_PO_LABEL, DAOS_PROP_CO_LABEL }; -enum daos_snapshot_opts { - DAOS_SNAP_OPT_CR = (1 << 0), - DAOS_SNAP_OPT_OIT = (1 << 1) -}; +enum daos_snapshot_opts { DAOS_SNAP_OPT_CR = (1 << 0), DAOS_SNAP_OPT_OIT = (1 << 1) }; -enum daos_obj_redun { - OR_RP_1 = 1 -}; +enum daos_obj_redun { OR_RP_1 = 1 }; enum { OC_S1 = OBJ_CLASS_DEF(OR_RP_1, 1ULL), @@ -133,21 +124,21 @@ typedef struct { } daos_range_t; typedef struct { daos_size_t arr_nr; - daos_range_t *arr_rgs; + daos_range_t* arr_rgs; daos_size_t arr_nr_short_read; daos_size_t arr_nr_read; } daos_array_iod_t; /* describe memory-space target region */ typedef struct { - void *iov_buf; + void* iov_buf; size_t iov_buf_len; size_t iov_len; } d_iov_t; typedef struct { uint32_t sg_nr; uint32_t sg_nr_out; - d_iov_t *sg_iovs; + d_iov_t* sg_iovs; } d_sg_list_t; /* pool properties */ @@ -160,21 +151,21 @@ struct daos_prop_entry { union { uint64_t dpe_val; d_string_t dpe_str; - void *dpe_val_ptr; + void* dpe_val_ptr; }; }; typedef struct { uint32_t dpp_nr; uint32_t dpp_reserv; - struct daos_prop_entry *dpp_entries; + struct daos_prop_entry* dpp_entries; } daos_prop_t; /* cont info */ struct daos_pool_cont_info { uuid_t pci_uuid; - char pci_label[DAOS_PROP_LABEL_MAX_LEN+1]; + char pci_label[DAOS_PROP_LABEL_MAX_LEN + 1]; }; /* kv list */ @@ -204,8 +195,8 @@ typedef enum { typedef uint64_t daos_epoch_t; typedef struct { - daos_epoch_t epr_lo; - daos_epoch_t epr_hi; + daos_epoch_t epr_lo; + daos_epoch_t epr_hi; } daos_epoch_range_t; //---------------------------------------------------------------------------------------------------------------------- @@ -216,113 +207,96 @@ int daos_init(void); int daos_fini(void); -int daos_pool_connect(const char *pool, const char *sys, unsigned int flags, - daos_handle_t *poh, daos_pool_info_t *info, daos_event_t *ev); +int daos_pool_connect(const char* pool, const char* sys, unsigned int flags, daos_handle_t* poh, daos_pool_info_t* info, + daos_event_t* ev); -int daos_pool_disconnect(daos_handle_t poh, daos_event_t *ev); +int daos_pool_disconnect(daos_handle_t poh, daos_event_t* ev); /* * warning: the daos_pool_list_cont API call is not fully implemented. Only the pci_label - * member of the info structs is populated in all cases. The pci_uuid member is only + * member of the info structs is populated in all cases. The pci_uuid member is only * populated for existing containers which were created without a label. */ -int daos_pool_list_cont(daos_handle_t poh, daos_size_t *ncont, - struct daos_pool_cont_info *cbuf, daos_event_t *ev); +int daos_pool_list_cont(daos_handle_t poh, daos_size_t* ncont, struct daos_pool_cont_info* cbuf, daos_event_t* ev); -int daos_cont_create(daos_handle_t poh, uuid_t *uuid, daos_prop_t *cont_prop, daos_event_t *ev); +int daos_cont_create(daos_handle_t poh, uuid_t* uuid, daos_prop_t* cont_prop, daos_event_t* ev); -int daos_cont_create_with_label(daos_handle_t poh, const char *label, - daos_prop_t *cont_prop, uuid_t *uuid, - daos_event_t *ev); +int daos_cont_create_with_label(daos_handle_t poh, const char* label, daos_prop_t* cont_prop, uuid_t* uuid, + daos_event_t* ev); -int daos_cont_destroy(daos_handle_t poh, const char *cont, int force, daos_event_t *ev); +int daos_cont_destroy(daos_handle_t poh, const char* cont, int force, daos_event_t* ev); -int daos_cont_open(daos_handle_t poh, const char *cont, unsigned int flags, daos_handle_t *coh, - daos_cont_info_t *info, daos_event_t *ev); +int daos_cont_open(daos_handle_t poh, const char* cont, unsigned int flags, daos_handle_t* coh, daos_cont_info_t* info, + daos_event_t* ev); -int daos_cont_close(daos_handle_t coh, daos_event_t *ev); +int daos_cont_close(daos_handle_t coh, daos_event_t* ev); -int daos_cont_alloc_oids(daos_handle_t coh, daos_size_t num_oids, uint64_t *oid, - daos_event_t *ev); +int daos_cont_alloc_oids(daos_handle_t coh, daos_size_t num_oids, uint64_t* oid, daos_event_t* ev); -int daos_obj_generate_oid(daos_handle_t coh, daos_obj_id_t *oid, - enum daos_otype_t type, daos_oclass_id_t cid, +int daos_obj_generate_oid(daos_handle_t coh, daos_obj_id_t* oid, enum daos_otype_t type, daos_oclass_id_t cid, daos_oclass_hints_t hints, uint32_t args); -int daos_kv_open(daos_handle_t coh, daos_obj_id_t oid, unsigned int mode, - daos_handle_t *oh, daos_event_t *ev); +int daos_kv_open(daos_handle_t coh, daos_obj_id_t oid, unsigned int mode, daos_handle_t* oh, daos_event_t* ev); -int daos_kv_destroy(daos_handle_t oh, daos_handle_t th, daos_event_t *ev); +int daos_kv_destroy(daos_handle_t oh, daos_handle_t th, daos_event_t* ev); -int daos_obj_close(daos_handle_t oh, daos_event_t *ev); +int daos_obj_close(daos_handle_t oh, daos_event_t* ev); -int daos_kv_put(daos_handle_t oh, daos_handle_t th, uint64_t flags, const char *key, - daos_size_t size, const void *buf, daos_event_t *ev); +int daos_kv_put(daos_handle_t oh, daos_handle_t th, uint64_t flags, const char* key, daos_size_t size, const void* buf, + daos_event_t* ev); -int daos_kv_get(daos_handle_t oh, daos_handle_t th, uint64_t flags, const char *key, - daos_size_t *size, void *buf, daos_event_t *ev); +int daos_kv_get(daos_handle_t oh, daos_handle_t th, uint64_t flags, const char* key, daos_size_t* size, void* buf, + daos_event_t* ev); -int daos_kv_remove(daos_handle_t oh, daos_handle_t th, uint64_t flags, - const char *key, daos_event_t *ev); +int daos_kv_remove(daos_handle_t oh, daos_handle_t th, uint64_t flags, const char* key, daos_event_t* ev); -int daos_kv_list(daos_handle_t oh, daos_handle_t th, uint32_t *nr, - daos_key_desc_t *kds, d_sg_list_t *sgl, daos_anchor_t *anchor, - daos_event_t *ev); +int daos_kv_list(daos_handle_t oh, daos_handle_t th, uint32_t* nr, daos_key_desc_t* kds, d_sg_list_t* sgl, + daos_anchor_t* anchor, daos_event_t* ev); -static inline bool daos_anchor_is_eof(daos_anchor_t *anchor) { +static inline bool daos_anchor_is_eof(daos_anchor_t* anchor) { return anchor->da_type == DAOS_ANCHOR_TYPE_EOF; } -int daos_array_generate_oid(daos_handle_t coh, daos_obj_id_t *oid, bool add_attr, daos_oclass_id_t cid, +int daos_array_generate_oid(daos_handle_t coh, daos_obj_id_t* oid, bool add_attr, daos_oclass_id_t cid, daos_oclass_hints_t hints, uint32_t args); -int daos_array_create(daos_handle_t coh, daos_obj_id_t oid, daos_handle_t th, - daos_size_t cell_size, daos_size_t chunk_size, - daos_handle_t *oh, daos_event_t *ev); +int daos_array_create(daos_handle_t coh, daos_obj_id_t oid, daos_handle_t th, daos_size_t cell_size, + daos_size_t chunk_size, daos_handle_t* oh, daos_event_t* ev); -int daos_array_destroy(daos_handle_t oh, daos_handle_t th, daos_event_t *ev); +int daos_array_destroy(daos_handle_t oh, daos_handle_t th, daos_event_t* ev); -int daos_array_open(daos_handle_t coh, daos_obj_id_t oid, daos_handle_t th, - unsigned int mode, daos_size_t *cell_size, - daos_size_t *chunk_size, daos_handle_t *oh, daos_event_t *ev); +int daos_array_open(daos_handle_t coh, daos_obj_id_t oid, daos_handle_t th, unsigned int mode, daos_size_t* cell_size, + daos_size_t* chunk_size, daos_handle_t* oh, daos_event_t* ev); -int daos_array_open_with_attr(daos_handle_t coh, daos_obj_id_t oid, daos_handle_t th, - unsigned int mode, daos_size_t cell_size, daos_size_t chunk_size, - daos_handle_t *oh, daos_event_t *ev); +int daos_array_open_with_attr(daos_handle_t coh, daos_obj_id_t oid, daos_handle_t th, unsigned int mode, + daos_size_t cell_size, daos_size_t chunk_size, daos_handle_t* oh, daos_event_t* ev); -int daos_array_close(daos_handle_t oh, daos_event_t *ev); +int daos_array_close(daos_handle_t oh, daos_event_t* ev); -static inline void d_iov_set(d_iov_t *iov, void *buf, size_t size) { +static inline void d_iov_set(d_iov_t* iov, void* buf, size_t size) { iov->iov_buf = buf; iov->iov_len = iov->iov_buf_len = size; } -int daos_array_write(daos_handle_t oh, daos_handle_t th, daos_array_iod_t *iod, - d_sg_list_t *sgl, daos_event_t *ev); +int daos_array_write(daos_handle_t oh, daos_handle_t th, daos_array_iod_t* iod, d_sg_list_t* sgl, daos_event_t* ev); -int daos_array_get_size(daos_handle_t oh, daos_handle_t th, daos_size_t *size, - daos_event_t *ev); +int daos_array_get_size(daos_handle_t oh, daos_handle_t th, daos_size_t* size, daos_event_t* ev); -int daos_array_read(daos_handle_t oh, daos_handle_t th, daos_array_iod_t *iod, - d_sg_list_t *sgl, daos_event_t *ev); +int daos_array_read(daos_handle_t oh, daos_handle_t th, daos_array_iod_t* iod, d_sg_list_t* sgl, daos_event_t* ev); -int daos_cont_create_snap_opt(daos_handle_t coh, daos_epoch_t *epoch, char *name, - enum daos_snapshot_opts opts, daos_event_t *ev); +int daos_cont_create_snap_opt(daos_handle_t coh, daos_epoch_t* epoch, char* name, enum daos_snapshot_opts opts, + daos_event_t* ev); -int daos_cont_destroy_snap(daos_handle_t coh, daos_epoch_range_t epr, - daos_event_t *ev); +int daos_cont_destroy_snap(daos_handle_t coh, daos_epoch_range_t epr, daos_event_t* ev); -int daos_oit_open(daos_handle_t coh, daos_epoch_t epoch, - daos_handle_t *oh, daos_event_t *ev); +int daos_oit_open(daos_handle_t coh, daos_epoch_t epoch, daos_handle_t* oh, daos_event_t* ev); -int daos_oit_close(daos_handle_t oh, daos_event_t *ev); +int daos_oit_close(daos_handle_t oh, daos_event_t* ev); -int daos_oit_list(daos_handle_t oh, daos_obj_id_t *oids, uint32_t *oids_nr, - daos_anchor_t *anchor, daos_event_t *ev); +int daos_oit_list(daos_handle_t oh, daos_obj_id_t* oids, uint32_t* oids_nr, daos_anchor_t* anchor, daos_event_t* ev); //---------------------------------------------------------------------------------------------------------------------- #ifdef __cplusplus -} // extern "C" +} // extern "C" #endif - diff --git a/src/dummy_daos/daos/tests_lib.cc b/src/dummy_daos/daos/tests_lib.cc index 89595a99f..4019d79a8 100644 --- a/src/dummy_daos/daos/tests_lib.cc +++ b/src/dummy_daos/daos/tests_lib.cc @@ -14,38 +14,38 @@ * @date Jun 2022 */ -#include #include +#include -#include "eckit/filesystem/TmpDir.h" #include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/TmpDir.h" #include "eckit/log/TimeStamp.h" #include "eckit/runtime/Main.h" #include "eckit/utils/MD5.h" -#include "tests_lib.h" #include "../dummy_daos.h" +#include "tests_lib.h" -namespace{ - void deldir(eckit::PathName& p) { - if (!p.exists()) { - return; - } +namespace { +void deldir(eckit::PathName& p) { + if (!p.exists()) { + return; + } - std::vector files; - std::vector dirs; - p.children(files, dirs); + std::vector files; + std::vector dirs; + p.children(files, dirs); - for (auto& f : files) { - f.unlink(); - } - for (auto& d : dirs) { - deldir(d); - } + for (auto& f : files) { + f.unlink(); + } + for (auto& d : dirs) { + deldir(d); + } - p.rmdir(); - }; -} + p.rmdir(); +}; +} // namespace //---------------------------------------------------------------------------------------------------------------------- @@ -53,7 +53,7 @@ extern "C" { daos_prop_t* daos_prop_alloc(uint32_t entries_nr) { - daos_prop_t *prop; + daos_prop_t* prop; if (entries_nr > DAOS_PROP_ENTRIES_MAX_NR) { eckit::Log::error() << "Too many property entries requested."; @@ -73,20 +73,20 @@ daos_prop_t* daos_prop_alloc(uint32_t entries_nr) { } prop->dpp_nr = entries_nr; return prop; - } -void daos_prop_fini(daos_prop_t *prop) { +void daos_prop_fini(daos_prop_t* prop) { int i; if (!prop->dpp_entries) goto out; for (i = 0; i < prop->dpp_nr; i++) { - struct daos_prop_entry *entry; + struct daos_prop_entry* entry; entry = &prop->dpp_entries[i]; - if (entry->dpe_type != DAOS_PROP_PO_LABEL) NOTIMP; + if (entry->dpe_type != DAOS_PROP_PO_LABEL) + NOTIMP; D_FREE(entry->dpe_str); } @@ -95,7 +95,7 @@ void daos_prop_fini(daos_prop_t *prop) { prop->dpp_nr = 0; } -void daos_prop_free(daos_prop_t *prop) { +void daos_prop_free(daos_prop_t* prop) { if (prop == NULL) return; @@ -103,37 +103,36 @@ void daos_prop_free(daos_prop_t *prop) { D_FREE(prop); } -/// @note: pools are implemented as directories. Upon creation, a new random and unique string +/// @note: pools are implemented as directories. Upon creation, a new random and unique string /// is generated, a pool UUID is generated from that string, and a directory is created -/// with the UUID as directory name. If a label is specified, a symlink is created with +/// with the UUID as directory name. If a label is specified, a symlink is created with /// the label as origin file name and the UUID directory as destination. If no label is /// specified, no symlink is created. -int dmg_pool_create(const char *dmg_config_file, - uid_t uid, gid_t gid, const char *grp, - const d_rank_list_t *tgts, - daos_size_t scm_size, daos_size_t nvme_size, - daos_prop_t *prop, - d_rank_list_t *svc, uuid_t uuid) { +int dmg_pool_create(const char* dmg_config_file, uid_t uid, gid_t gid, const char* grp, const d_rank_list_t* tgts, + daos_size_t scm_size, daos_size_t nvme_size, daos_prop_t* prop, d_rank_list_t* svc, uuid_t uuid) { std::string pool_name; eckit::PathName label_symlink_path; if (prop != NULL) { - if (prop->dpp_nr != 1) NOTIMP; - if (prop->dpp_entries[0].dpe_type != DAOS_PROP_PO_LABEL) NOTIMP; + if (prop->dpp_nr != 1) + NOTIMP; + if (prop->dpp_entries[0].dpe_type != DAOS_PROP_PO_LABEL) + NOTIMP; - struct daos_prop_entry *entry = &prop->dpp_entries[0]; + struct daos_prop_entry* entry = &prop->dpp_entries[0]; - if (entry == NULL) NOTIMP; + if (entry == NULL) + NOTIMP; pool_name = std::string(entry->dpe_str); label_symlink_path = dummy_daos_root() / pool_name; - if (label_symlink_path.exists()) return -1; - + if (label_symlink_path.exists()) + return -1; } /// @note: copied from LocalPathName::unique. Ditched StaticMutex as dummy DAOS is not thread safe @@ -165,67 +164,66 @@ int dmg_pool_create(const char *dmg_config_file, eckit::PathName pool_path = dummy_daos_root() / pool_uuid_cstr; - if (pool_path.exists()) throw eckit::SeriousBug("UUID clash in pool create"); + if (pool_path.exists()) + throw eckit::SeriousBug("UUID clash in pool create"); pool_path.mkdir(); - if (prop == NULL) return 0; - + if (prop == NULL) + return 0; + if (::symlink(pool_path.path().c_str(), label_symlink_path.path().c_str()) < 0) { - if (errno == EEXIST) { // link path already exists due to race condition + if (errno == EEXIST) { // link path already exists due to race condition deldir(pool_path); return -1; - } else { // symlink fails for unknown reason + } else { // symlink fails for unknown reason throw eckit::FailedSystemCall(std::string("symlink ") + pool_path.path() + " " + label_symlink_path.path()); - } - } return 0; - } -int dmg_pool_destroy(const char *dmg_config_file, - const uuid_t uuid, const char *grp, int force) { - +int dmg_pool_destroy(const char* dmg_config_file, const uuid_t uuid, const char* grp, int force) { + char uuid_str[37] = ""; uuid_unparse(uuid, uuid_str); eckit::PathName pool_path = dummy_daos_root() / uuid_str; - if (!pool_path.exists()) return -1; + if (!pool_path.exists()) + return -1; std::vector files; std::vector dirs; dummy_daos_root().children(files, dirs); - /// @note: all pool labels (symlinks) are visited and deleted if they point to the + /// @note: all pool labels (symlinks) are visited and deleted if they point to the /// specified pool UUID. None will match if the pool was not labelled. for (auto& f : files) { try { - if (f.isLink() && f.realName().baseName() == pool_path.baseName()) f.unlink(); + if (f.isLink() && f.realName().baseName() == pool_path.baseName()) + f.unlink(); } catch (eckit::FailedSystemCall& e) { - if (f.exists()) throw; - + if (f.exists()) + throw; } } deldir(pool_path); return 0; - } //---------------------------------------------------------------------------------------------------------------------- -} // extern "C" +} // extern "C" diff --git a/src/dummy_daos/daos/tests_lib.h b/src/dummy_daos/daos/tests_lib.h index a6a1680f6..561e58939 100644 --- a/src/dummy_daos/daos/tests_lib.h +++ b/src/dummy_daos/daos/tests_lib.h @@ -24,7 +24,11 @@ #define D_ALLOC_ARRAY(ptr, count) (ptr) = (__typeof__(ptr))calloc((count), (sizeof(*ptr))); #define D_ALLOC_PTR(ptr) D_ALLOC_ARRAY(ptr, 1) -#define D_FREE(ptr) ({ free(ptr); (ptr) = NULL; }) +#define D_FREE(ptr) \ + ({ \ + free(ptr); \ + (ptr) = NULL; \ + }) #define D_STRNDUP(ptr, s, n) (ptr) = strndup(s, n); //---------------------------------------------------------------------------------------------------------------------- @@ -36,7 +40,7 @@ extern "C" { typedef uint32_t d_rank_t; typedef struct { - d_rank_t *rl_ranks; + d_rank_t* rl_ranks; uint32_t rl_nr; } d_rank_list_t; @@ -44,20 +48,15 @@ typedef struct { daos_prop_t* daos_prop_alloc(uint32_t entries_nr); -void daos_prop_free(daos_prop_t *prop); +void daos_prop_free(daos_prop_t* prop); -int dmg_pool_create(const char *dmg_config_file, - uid_t uid, gid_t gid, const char *grp, - const d_rank_list_t *tgts, - daos_size_t scm_size, daos_size_t nvme_size, - daos_prop_t *prop, - d_rank_list_t *svc, uuid_t uuid); +int dmg_pool_create(const char* dmg_config_file, uid_t uid, gid_t gid, const char* grp, const d_rank_list_t* tgts, + daos_size_t scm_size, daos_size_t nvme_size, daos_prop_t* prop, d_rank_list_t* svc, uuid_t uuid); -int dmg_pool_destroy(const char *dmg_config_file, - const uuid_t uuid, const char *grp, int force); +int dmg_pool_destroy(const char* dmg_config_file, const uuid_t uuid, const char* grp, int force); //---------------------------------------------------------------------------------------------------------------------- #ifdef __cplusplus -} // extern "C" +} // extern "C" #endif /* end ifdef __cplusplus */ diff --git a/src/dummy_daos/dummy_daos.cc b/src/dummy_daos/dummy_daos.cc index cdf8a915a..6dd572857 100644 --- a/src/dummy_daos/dummy_daos.cc +++ b/src/dummy_daos/dummy_daos.cc @@ -34,14 +34,12 @@ const PathName& dummy_daos_root() { static PathName tmpdir = eckit::Resource("$TMPDIR", "/tmp"); static PathName daos_root = eckit::Resource("$DUMMY_DAOS_DATA_ROOT", tmpdir / "fdb5_dummy_daos"); return daos_root; - } const PathName& dummy_daos_get_handle_path(daos_handle_t handle) { ASSERT(handle.impl); return handle.impl->path; - } //---------------------------------------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/src/dummy_daos/dummy_daos.h b/src/dummy_daos/dummy_daos.h index 51c5d69df..cb9e2c4dd 100644 --- a/src/dummy_daos/dummy_daos.h +++ b/src/dummy_daos/dummy_daos.h @@ -8,7 +8,7 @@ * does it submit to any jurisdiction. */ -/* +/* * @file dummy_daos.h * @author Nicolau Manubens * @date Jun 2022 diff --git a/src/fdb5/LibFdb5.cc b/src/fdb5/LibFdb5.cc index dd5cdea57..e7007ee77 100644 --- a/src/fdb5/LibFdb5.cc +++ b/src/fdb5/LibFdb5.cc @@ -16,10 +16,10 @@ #include "fdb5/LibFdb5.h" -#include "eckit/eckit_version.h" #include "eckit/config/LibEcKit.h" #include "eckit/config/Resource.h" #include "eckit/config/YAMLConfiguration.h" +#include "eckit/eckit_version.h" #include "eckit/exception/Exceptions.h" #include "eckit/log/Log.h" @@ -40,10 +40,10 @@ LibFdb5& LibFdb5::instance() { return libfdb; } -const Config& LibFdb5::defaultConfig(const eckit::Configuration& userConfig) { - if(!config_) { +const Config& LibFdb5::defaultConfig(const eckit::Configuration& userConfig) { + if (!config_) { Config cfg; - config_.reset( new Config( std::move(cfg.expandConfig()), userConfig ) ); + config_.reset(new Config(std::move(cfg.expandConfig()), userConfig)); } return *config_; } @@ -57,7 +57,8 @@ void LibFdb5::registerConstructorCallback(ConstructorCallback cb) { } bool LibFdb5::dontDeregisterFactories() const { -#if eckit_VERSION_MAJOR > 1 || (eckit_VERSION_MAJOR == 1 && (eckit_VERSION_MINOR > 17 || (eckit_VERSION_MINOR == 17 && eckit_VERSION_PATCH >0))) +#if eckit_VERSION_MAJOR > 1 || \ + (eckit_VERSION_MAJOR == 1 && (eckit_VERSION_MINOR > 17 || (eckit_VERSION_MINOR == 17 && eckit_VERSION_PATCH > 0))) return eckit::LibEcKit::instance().dontDeregisterFactories(); #else return false; @@ -81,9 +82,9 @@ RemoteProtocolVersion LibFdb5::remoteProtocolVersion() const { return RemoteProtocolVersion{}; } - const std::set& LibFdb5::auxiliaryRegistry() { - static std::set auxiliaryRegistry(eckit::Resource>("$FDB_AUX_EXTENSIONS;fdbAuxExtensions", {"gribjump"})); + static std::set auxiliaryRegistry( + eckit::Resource>("$FDB_AUX_EXTENSIONS;fdbAuxExtensions", {"gribjump"})); return auxiliaryRegistry; } //---------------------------------------------------------------------------------------------------------------------- @@ -94,9 +95,9 @@ static unsigned getUserEnvRemoteProtocol() { eckit::Resource("fdbRemoteProtocolVersion;$FDB5_REMOTE_PROTOCOL_VERSION", 0); if (fdbRemoteProtocolVersion) { LOG_DEBUG_LIB(LibFdb5) << "fdbRemoteProtocolVersion overidde to version: " << fdbRemoteProtocolVersion - << std::endl; + << std::endl; } - return 0; // no version override + return 0; // no version override } static bool getUserEnvSkipSanityCheck() { @@ -104,10 +105,10 @@ static bool getUserEnvSkipSanityCheck() { } RemoteProtocolVersion::RemoteProtocolVersion() { - static unsigned user = getUserEnvRemoteProtocol(); + static unsigned user = getUserEnvRemoteProtocol(); static bool skipcheck = getUserEnvSkipSanityCheck(); - if(not user) { + if (not user) { used_ = defaulted(); return; } @@ -168,4 +169,4 @@ bool RemoteProtocolVersion::check(unsigned int version, bool throwOnFail) { //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/LibFdb5.h b/src/fdb5/LibFdb5.h index 9ff63202b..7366114d2 100644 --- a/src/fdb5/LibFdb5.h +++ b/src/fdb5/LibFdb5.h @@ -19,9 +19,9 @@ #include "eckit/system/Library.h" +#include "fdb5/api/helpers/Callback.h" #include "fdb5/database/Catalogue.h" #include "fdb5/types/TypesRegistry.h" -#include "fdb5/api/helpers/Callback.h" namespace fdb5 { @@ -94,4 +94,4 @@ class LibFdb5 : public eckit::system::Library { //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/api/DistFDB.cc b/src/fdb5/api/DistFDB.cc index fbbe524b5..420928404 100644 --- a/src/fdb5/api/DistFDB.cc +++ b/src/fdb5/api/DistFDB.cc @@ -13,9 +13,9 @@ * (Project ID: 671951) www.nextgenio.eu */ -#include -#include #include +#include +#include #include "eckit/config/LocalConfiguration.h" #include "eckit/exception/Exceptions.h" @@ -25,40 +25,38 @@ #include "metkit/hypercube/HyperCube.h" +#include "fdb5/LibFdb5.h" #include "fdb5/api/DistFDB.h" -#include "fdb5/database/Notifier.h" #include "fdb5/api/helpers/FDBToolRequest.h" #include "fdb5/api/helpers/ListIterator.h" +#include "fdb5/database/Notifier.h" #include "fdb5/io/HandleGatherer.h" -#include "fdb5/LibFdb5.h" using eckit::Log; - namespace fdb5 { static FDBBuilder distFdbBuilder("dist"); //---------------------------------------------------------------------------------------------------------------------- - struct DistributionError : public eckit::Exception { - DistributionError(const std::string& r, const eckit::CodeLocation& loc) : - Exception(std::string(" DistributionError: " + r, loc)) {} + DistributionError(const std::string& r, const eckit::CodeLocation& loc) + : Exception(std::string(" DistributionError: " + r, loc)) {} }; //---------------------------------------------------------------------------------------------------------------------- -DistFDB::DistFDB(const Config& config, const std::string& name) : - FDBBase(config, name) { +DistFDB::DistFDB(const Config& config, const std::string& name) : FDBBase(config, name) { ASSERT(config.getString("type", "") == "dist"); // Configure the available lanes. - if (!config.has("lanes")) throw eckit::UserError("No lanes configured for pool", Here()); + if (!config.has("lanes")) + throw eckit::UserError("No lanes configured for pool", Here()); - for(const auto& laneCfg : config.getSubConfigs("lanes")) { + for (const auto& laneCfg : config.getSubConfigs("lanes")) { lanes_.push_back(FDB(laneCfg)); if (!hash_.addNode(lanes_.back().id())) { std::stringstream ss; @@ -74,17 +72,17 @@ void DistFDB::archive(const Key& key, const void* data, size_t length) { std::vector laneIndices; - //LOG_DEBUG_LIB(LibFdb5) << "Number of lanes: " << lanes_.size() << std::endl; - //LOG_DEBUG_LIB(LibFdb5) << "Lane indices: "; - //for (const auto& i : laneIndices) LOG_DEBUG_LIB(LibFdb5) << i << ", "; - //LOG_DEBUG_LIB(LibFdb5) << std::endl; + // LOG_DEBUG_LIB(LibFdb5) << "Number of lanes: " << lanes_.size() << std::endl; + // LOG_DEBUG_LIB(LibFdb5) << "Lane indices: "; + // for (const auto& i : laneIndices) LOG_DEBUG_LIB(LibFdb5) << i << ", "; + // LOG_DEBUG_LIB(LibFdb5) << std::endl; hash_.hashOrder(key.keyDict(), laneIndices); - //LOG_DEBUG_LIB(LibFdb5) << "Number of lanes: " << lanes_.size() << std::endl; - //LOG_DEBUG_LIB(LibFdb5) << "Lane indices: "; - //for (const auto& i : laneIndices) LOG_DEBUG_LIB(LibFdb5) << i << ", "; - //LOG_DEBUG_LIB(LibFdb5) << std::endl; + // LOG_DEBUG_LIB(LibFdb5) << "Number of lanes: " << lanes_.size() << std::endl; + // LOG_DEBUG_LIB(LibFdb5) << "Lane indices: "; + // for (const auto& i : laneIndices) LOG_DEBUG_LIB(LibFdb5) << i << ", "; + // LOG_DEBUG_LIB(LibFdb5) << std::endl; // Given an order supplied by the Rendezvous hash, try the FDB in order until // one works. n.b. Errors are unacceptable once the FDB is dirty. @@ -97,7 +95,7 @@ void DistFDB::archive(const Key& key, const void* data, size_t length) { FDB& lane = lanes_[idx]; - if(!lane.enabled(ControlIdentifier::Archive)) { + if (!lane.enabled(ControlIdentifier::Archive)) { continue; } if (lane.disabled()) { @@ -155,7 +153,8 @@ void DistFDB::archive(const Key& key, const void* data, size_t length) { */ template -auto DistFDB::queryInternal(const FDBToolRequest& request, const QueryFN& fn) -> decltype(fn(*(FDB*)(nullptr), request)) { +auto DistFDB::queryInternal(const FDBToolRequest& request, const QueryFN& fn) + -> decltype(fn(*(FDB*)(nullptr), request)) { using QueryIterator = decltype(fn(*(FDB*)(nullptr), request)); using ValueType = typename QueryIterator::value_type; @@ -165,9 +164,7 @@ auto DistFDB::queryInternal(const FDBToolRequest& request, const QueryFN& fn) -> for (FDB& lane : lanes_) { if (lane.enabled(ControlIdentifier::Retrieve)) { - futures.emplace_back(std::async(std::launch::async, [&lane, &fn, &request] { - return fn(lane, request); - })); + futures.emplace_back(std::async(std::launch::async, [&lane, &fn, &request] { return fn(lane, request); })); } } @@ -178,80 +175,57 @@ auto DistFDB::queryInternal(const FDBToolRequest& request, const QueryFN& fn) -> return QueryIterator(new APIAggregateIterator(std::move(iterQueue))); } - ListIterator DistFDB::list(const FDBToolRequest& request) { LOG_DEBUG_LIB(LibFdb5) << "DistFDB::list() : " << request << std::endl; - return queryInternal(request, - [](FDB& fdb, const FDBToolRequest& request) { - return fdb.list(request); - }); + return queryInternal(request, [](FDB& fdb, const FDBToolRequest& request) { return fdb.list(request); }); } ListIterator DistFDB::inspect(const metkit::mars::MarsRequest& request) { LOG_DEBUG_LIB(LibFdb5) << "DistFDB::inspect() : " << request << std::endl; return queryInternal(request, - [](FDB& fdb, const FDBToolRequest& request) { - return fdb.inspect(request.request()); - }); + [](FDB& fdb, const FDBToolRequest& request) { return fdb.inspect(request.request()); }); } DumpIterator DistFDB::dump(const FDBToolRequest& request, bool simple) { LOG_DEBUG_LIB(LibFdb5) << "DistFDB::dump() : " << request << std::endl; return queryInternal(request, - [simple](FDB& fdb, const FDBToolRequest& request) { - return fdb.dump(request, simple); - }); + [simple](FDB& fdb, const FDBToolRequest& request) { return fdb.dump(request, simple); }); } StatusIterator DistFDB::status(const FDBToolRequest& request) { LOG_DEBUG_LIB(LibFdb5) << "DistFDB::status() : " << request << std::endl; - return queryInternal(request, - [](FDB& fdb, const FDBToolRequest& request) { - return fdb.status(request); - }); + return queryInternal(request, [](FDB& fdb, const FDBToolRequest& request) { return fdb.status(request); }); } WipeIterator DistFDB::wipe(const FDBToolRequest& request, bool doit, bool porcelain, bool unsafeWipeAll) { LOG_DEBUG_LIB(LibFdb5) << "DistFDB::wipe() : " << request << std::endl; - return queryInternal(request, - [doit, porcelain, unsafeWipeAll](FDB& fdb, const FDBToolRequest& request) { - return fdb.wipe(request, doit, porcelain, unsafeWipeAll); + return queryInternal(request, [doit, porcelain, unsafeWipeAll](FDB& fdb, const FDBToolRequest& request) { + return fdb.wipe(request, doit, porcelain, unsafeWipeAll); }); } PurgeIterator DistFDB::purge(const FDBToolRequest& request, bool doit, bool porcelain) { LOG_DEBUG_LIB(LibFdb5) << "DistFDB::purge() : " << request << std::endl; - return queryInternal(request, - [doit, porcelain](FDB& fdb, const FDBToolRequest& request) { - return fdb.purge(request, doit, porcelain); + return queryInternal(request, [doit, porcelain](FDB& fdb, const FDBToolRequest& request) { + return fdb.purge(request, doit, porcelain); }); } -StatsIterator DistFDB::stats(const FDBToolRequest &request) { +StatsIterator DistFDB::stats(const FDBToolRequest& request) { LOG_DEBUG_LIB(LibFdb5) << "DistFDB::stats() : " << request << std::endl; - return queryInternal(request, - [](FDB& fdb, const FDBToolRequest& request) { - return fdb.stats(request); - }); + return queryInternal(request, [](FDB& fdb, const FDBToolRequest& request) { return fdb.stats(request); }); } -ControlIterator DistFDB::control(const FDBToolRequest& request, - ControlAction action, - ControlIdentifiers identifiers) { +ControlIterator DistFDB::control(const FDBToolRequest& request, ControlAction action, ControlIdentifiers identifiers) { LOG_DEBUG_LIB(LibFdb5) << "DistFDB::control() : " << request << std::endl; - return queryInternal(request, - [action, identifiers](FDB& fdb, const FDBToolRequest& request) { - return fdb.control(request, action, identifiers); + return queryInternal(request, [action, identifiers](FDB& fdb, const FDBToolRequest& request) { + return fdb.control(request, action, identifiers); }); } - MoveIterator DistFDB::move(const FDBToolRequest& request, const eckit::URI& dest) { LOG_DEBUG_LIB(LibFdb5) << "DistFDB::move() : " << request << std::endl; - return queryInternal(request, - [dest](FDB& fdb, const FDBToolRequest& request) { - return fdb.move(request, dest); - }); + return queryInternal(request, [dest](FDB& fdb, const FDBToolRequest& request) { return fdb.move(request, dest); }); } void DistFDB::flush() { @@ -259,9 +233,7 @@ void DistFDB::flush() { std::vector> futures; for (FDB& lane : lanes_) { - futures.emplace_back(std::async(std::launch::async, [&lane] { - lane.flush(); - })); + futures.emplace_back(std::async(std::launch::async, [&lane] { lane.flush(); })); } } @@ -273,8 +245,7 @@ FDBStats DistFDB::stats() const { return s; } - -void DistFDB::print(std::ostream &s) const { +void DistFDB::print(std::ostream& s) const { s << "DistFDB(home=" << config_.expandPath("~fdb") << ")"; } diff --git a/src/fdb5/api/DistFDB.h b/src/fdb5/api/DistFDB.h index 43e177f3f..e176e537a 100644 --- a/src/fdb5/api/DistFDB.h +++ b/src/fdb5/api/DistFDB.h @@ -24,18 +24,15 @@ #include "eckit/utils/RendezvousHash.h" - namespace fdb5 { class FDB; //---------------------------------------------------------------------------------------------------------------------- - class DistFDB : public FDBBase { public: // method - using FDBBase::stats; DistFDB(const Config& config, const std::string& name); @@ -47,7 +44,7 @@ class DistFDB : public FDBBase { ListIterator list(const FDBToolRequest& request) override; - AxesIterator axesIterator(const FDBToolRequest& request, int level=3) override { NOTIMP; } + AxesIterator axesIterator(const FDBToolRequest& request, int level = 3) override { NOTIMP; } DumpIterator dump(const FDBToolRequest& request, bool simple) override; @@ -59,8 +56,7 @@ class DistFDB : public FDBBase { StatsIterator stats(const FDBToolRequest& request) override; - ControlIterator control(const FDBToolRequest& request, - ControlAction action, + ControlIterator control(const FDBToolRequest& request, ControlAction action, ControlIdentifiers identifiers) override; MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) override; @@ -70,14 +66,12 @@ class DistFDB : public FDBBase { FDBStats stats() const override; private: // methods - void print(std::ostream& s) const override; template auto queryInternal(const FDBToolRequest& request, const QueryFN& fn) -> decltype(fn(*(FDB*)(nullptr), request)); private: - eckit::RendezvousHash hash_; std::vector lanes_; diff --git a/src/fdb5/api/FDB.cc b/src/fdb5/api/FDB.cc index 8e06c18b8..13b7848ab 100644 --- a/src/fdb5/api/FDB.cc +++ b/src/fdb5/api/FDB.cc @@ -37,14 +37,13 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -FDB::FDB(const Config &config) : - internal_(FDBFactory::instance().build(config)), - dirty_(false), - reportStats_(config.getBool("statistics", false)) { +FDB::FDB(const Config& config) + : internal_(FDBFactory::instance().build(config)) + , dirty_(false) + , reportStats_(config.getBool("statistics", false)) { LibFdb5::instance().constructorCallback()(*internal_); } - FDB::~FDB() { flush(); if (reportStats_ && internal_) { @@ -61,7 +60,7 @@ void FDB::archive(eckit::DataHandle& handle) { eckit::message::Message msg; eckit::message::Reader reader(handle); - while ( (msg = reader.next()) ) { + while ((msg = reader.next())) { archive(msg); } } @@ -76,12 +75,12 @@ void FDB::archive(const metkit::mars::MarsRequest& request, eckit::DataHandle& h metkit::hypercube::HyperCube cube(request); - while ( (msg = reader.next()) ) { + while ((msg = reader.next())) { fdb5::Key key = MessageDecoder::messageToKey(msg); if (!cube.clear(key.request())) { std::stringstream ss; ss << "FDB archive - found unexpected message" << std::endl; - ss << " user request:" << std::endl << " " << request << std::endl; + ss << " user request:" << std::endl << " " << request << std::endl; ss << " unexpected message:" << std::endl << " " << key << std::endl; LOG_DEBUG_LIB(LibFdb5) << ss.str(); throw eckit::UserError(ss.str(), Here()); @@ -91,7 +90,7 @@ void FDB::archive(const metkit::mars::MarsRequest& request, eckit::DataHandle& h if (cube.countVacant()) { std::stringstream ss; ss << "FDB archive - missing " << cube.countVacant() << " messages" << std::endl; - ss << " user request:" << std::endl << " " << request << std::endl; + ss << " user request:" << std::endl << " " << request << std::endl; ss << " missing messages:" << std::endl; for (auto vacantRequest : cube.vacantRequests()) { ss << " " << vacantRequest << std::endl; @@ -112,10 +111,11 @@ void FDB::archive(const Key& key, const void* data, size_t length) { // step in archival requests from the model is just an integer. We need to include the stepunit auto stepunit = keyInternal.find("stepunits"); if (stepunit != keyInternal.end()) { - if (stepunit->second.size()>0 && static_cast(tolower(stepunit->second[0])) != 'h') { + if (stepunit->second.size() > 0 && static_cast(tolower(stepunit->second[0])) != 'h') { auto step = keyInternal.find("step"); if (step != keyInternal.end()) { - std::string canonicalStep = config().schema().registry().lookupType("step").toKey(step->second + static_cast(tolower(stepunit->second[0]))); + std::string canonicalStep = config().schema().registry().lookupType("step").toKey( + step->second + static_cast(tolower(stepunit->second[0]))); keyInternal.set("step", canonicalStep); } } @@ -129,7 +129,7 @@ void FDB::archive(const Key& key, const void* data, size_t length) { stats_.addArchive(length, timer); } -bool FDB::sorted(const metkit::mars::MarsRequest &request) { +bool FDB::sorted(const metkit::mars::MarsRequest& request) { bool sorted = false; @@ -190,28 +190,27 @@ eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) { // checking all retrieved fields against the hypercube, to remove duplicates ListElementDeduplicator dedup; metkit::hypercube::HyperCubePayloaded cube(cubeRequest, dedup); - for(auto el: elements) { + for (auto el : elements) { cube.add(el.combinedKey().request(), el); } if (cube.countVacant() > 0) { std::stringstream ss; ss << "No matching data for requests:" << std::endl; - for (auto req: cube.vacantRequests()) { + for (auto req : cube.vacantRequests()) { ss << " " << req << std::endl; } eckit::Log::warning() << ss.str() << std::endl; } - for (size_t i=0; i< cube.size(); i++) { + for (size_t i = 0; i < cube.size(); i++) { ListElement element; if (cube.find(i, element)) { result.add(element.location().dataHandle()); } } } - } - else { + } else { while (it.next(el)) { result.add(el.location().dataHandle()); } @@ -244,11 +243,11 @@ WipeIterator FDB::wipe(const FDBToolRequest& request, bool doit, bool porcelain, return internal_->wipe(request, doit, porcelain, unsafeWipeAll); } -PurgeIterator FDB::purge(const FDBToolRequest &request, bool doit, bool porcelain) { +PurgeIterator FDB::purge(const FDBToolRequest& request, bool doit, bool porcelain) { return internal_->purge(request, doit, porcelain); } -StatsIterator FDB::stats(const FDBToolRequest &request) { +StatsIterator FDB::stats(const FDBToolRequest& request) { return internal_->stats(request); } @@ -276,7 +275,7 @@ const std::string& FDB::name() const { return internal_->name(); } -const Config &FDB::config() const { +const Config& FDB::config() const { return internal_->config(); } diff --git a/src/fdb5/api/FDB.h b/src/fdb5/api/FDB.h index 75c3b5de4..6d438516b 100644 --- a/src/fdb5/api/FDB.h +++ b/src/fdb5/api/FDB.h @@ -19,32 +19,34 @@ #ifndef fdb5_api_FDB_H #define fdb5_api_FDB_H -#include #include +#include #include "eckit/distributed/Transport.h" #include "fdb5/api/FDBStats.h" +#include "fdb5/api/helpers/AxesIterator.h" +#include "fdb5/api/helpers/Callback.h" #include "fdb5/api/helpers/ControlIterator.h" #include "fdb5/api/helpers/DumpIterator.h" #include "fdb5/api/helpers/ListIterator.h" +#include "fdb5/api/helpers/MoveIterator.h" #include "fdb5/api/helpers/PurgeIterator.h" #include "fdb5/api/helpers/StatsIterator.h" #include "fdb5/api/helpers/StatusIterator.h" #include "fdb5/api/helpers/WipeIterator.h" -#include "fdb5/api/helpers/MoveIterator.h" -#include "fdb5/api/helpers/AxesIterator.h" #include "fdb5/config/Config.h" -#include "fdb5/api/helpers/Callback.h" namespace eckit { namespace message { class Message; } class DataHandle; -} // namespace eckit +} // namespace eckit -namespace metkit { class MarsRequest; } +namespace metkit { +class MarsRequest; +} namespace fdb5 { @@ -59,7 +61,6 @@ class Key; class FDB { public: // methods - FDB(const Config& config = Config().expandConfig()); ~FDB(); @@ -74,7 +75,8 @@ class FDB { void archive(eckit::message::Message msg); void archive(eckit::DataHandle& handle); void archive(const void* data, size_t length); - // warning: not high-perf API - makes sure that all the requested fields are archived and there are no data exceeding the request + // warning: not high-perf API - makes sure that all the requested fields are archived and there are no data + // exceeding the request void archive(const metkit::mars::MarsRequest& request, eckit::DataHandle& handle); // disclaimer: this is a low-level API. The provided key and the corresponding data are not checked for consistency @@ -95,28 +97,27 @@ class FDB { ListIterator inspect(const metkit::mars::MarsRequest& request); - ListIterator list(const FDBToolRequest& request, bool deduplicate=false); + ListIterator list(const FDBToolRequest& request, bool deduplicate = false); - DumpIterator dump(const FDBToolRequest& request, bool simple=false); + DumpIterator dump(const FDBToolRequest& request, bool simple = false); /// TODO: Is this function superfluous given the control() function? StatusIterator status(const FDBToolRequest& request); - WipeIterator wipe(const FDBToolRequest& request, bool doit=false, bool porcelain=false, bool unsafeWipeAll=false); + WipeIterator wipe(const FDBToolRequest& request, bool doit = false, bool porcelain = false, + bool unsafeWipeAll = false); MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest); - PurgeIterator purge(const FDBToolRequest& request, bool doit=false, bool porcelain=false); + PurgeIterator purge(const FDBToolRequest& request, bool doit = false, bool porcelain = false); StatsIterator stats(const FDBToolRequest& request); - ControlIterator control(const FDBToolRequest& request, - ControlAction action, - ControlIdentifiers identifiers); + ControlIterator control(const FDBToolRequest& request, ControlAction action, ControlIdentifiers identifiers); - IndexAxis axes(const FDBToolRequest& request, int level=3); + IndexAxis axes(const FDBToolRequest& request, int level = 3); - AxesIterator axesIterator(const FDBToolRequest& request, int level=3); + AxesIterator axesIterator(const FDBToolRequest& request, int level = 3); bool enabled(const ControlIdentifier& controlIdentifier) const; @@ -140,7 +141,6 @@ class FDB { bool disabled() const; private: // methods - void print(std::ostream&) const; friend std::ostream& operator<<(std::ostream& s, const FDB& f) { @@ -148,10 +148,9 @@ class FDB { return s; } - bool sorted(const metkit::mars::MarsRequest &request); + bool sorted(const metkit::mars::MarsRequest& request); private: // members - std::unique_ptr internal_; bool dirty_; diff --git a/src/fdb5/api/FDBFactory.cc b/src/fdb5/api/FDBFactory.cc index 6c48026ad..219aad6f0 100644 --- a/src/fdb5/api/FDBFactory.cc +++ b/src/fdb5/api/FDBFactory.cc @@ -13,27 +13,21 @@ * (Project ID: 671951) www.nextgenio.eu */ - #include "eckit/config/YAMLConfiguration.h" #include "eckit/log/Log.h" #include "eckit/message/Message.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/Mutex.h" +#include "fdb5/LibFdb5.h" #include "fdb5/api/FDBFactory.h" #include "fdb5/api/helpers/FDBToolRequest.h" -#include "fdb5/LibFdb5.h" - namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- - -FDBBase::FDBBase(const Config& config, const std::string& name) : - name_(name), - config_(config), - disabled_(false) { +FDBBase::FDBBase(const Config& config, const std::string& name) : name_(name), config_(config), disabled_(false) { bool writable = config.getBool("writable", true); bool visitable = config.getBool("visitable", true); @@ -53,7 +47,6 @@ FDBBase::FDBBase(const Config& config, const std::string& name) : LOG_DEBUG_LIB(LibFdb5) << "FDBBase: " << config << std::endl; } - FDBBase::~FDBBase() {} std::string FDBBase::id() const { @@ -67,7 +60,7 @@ FDBStats FDBBase::stats() const { return FDBStats(); } -const std::string &FDBBase::name() const { +const std::string& FDBBase::name() const { return name_; } @@ -88,14 +81,12 @@ bool FDBBase::disabled() { return disabled_; } -FDBFactory& FDBFactory::instance() -{ +FDBFactory& FDBFactory::instance() { static FDBFactory fdbfactory; return fdbfactory; } -void FDBFactory::add(const std::string& name, const FDBBuilderBase* b) -{ +void FDBFactory::add(const std::string& name, const FDBBuilderBase* b) { eckit::AutoLock lock(mutex_); ASSERT(registry_.find(name) == registry_.end()); @@ -131,14 +122,12 @@ std::unique_ptr FDBFactory::build(const Config& config) { return ret; } -FDBBuilderBase::FDBBuilderBase(const std::string &name) : - name_(name) { +FDBBuilderBase::FDBBuilderBase(const std::string& name) : name_(name) { FDBFactory::instance().add(name, this); } -FDBBuilderBase::~FDBBuilderBase() { -} +FDBBuilderBase::~FDBBuilderBase() {} //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/api/FDBFactory.h b/src/fdb5/api/FDBFactory.h index 6ec6e7b99..55844ddcb 100644 --- a/src/fdb5/api/FDBFactory.h +++ b/src/fdb5/api/FDBFactory.h @@ -22,28 +22,28 @@ #include #include "eckit/distributed/Transport.h" -#include "eckit/utils/Regex.h" #include "eckit/memory/NonCopyable.h" +#include "eckit/utils/Regex.h" -#include "fdb5/database/Catalogue.h" -#include "fdb5/config/Config.h" #include "fdb5/api/FDBStats.h" #include "fdb5/api/helpers/AxesIterator.h" -#include "fdb5/api/helpers/ListIterator.h" +#include "fdb5/api/helpers/Callback.h" #include "fdb5/api/helpers/ControlIterator.h" #include "fdb5/api/helpers/DumpIterator.h" -#include "fdb5/api/helpers/WipeIterator.h" +#include "fdb5/api/helpers/ListIterator.h" #include "fdb5/api/helpers/MoveIterator.h" #include "fdb5/api/helpers/PurgeIterator.h" #include "fdb5/api/helpers/StatsIterator.h" #include "fdb5/api/helpers/StatusIterator.h" -#include "fdb5/api/helpers/Callback.h" +#include "fdb5/api/helpers/WipeIterator.h" +#include "fdb5/config/Config.h" +#include "fdb5/database/Catalogue.h" namespace eckit::message { class Message; -} // namespace eckit::message +} // namespace eckit::message namespace metkit { @@ -63,14 +63,13 @@ class FDBToolRequest; class FDBBase : private eckit::NonCopyable, public CallbackRegistry { public: // methods - FDBBase(const Config& config, const std::string& name); virtual ~FDBBase(); // -------------- Primary API functions ---------------------------- virtual void archive(const Key& key, const void* data, size_t length) = 0; - + virtual void flush() = 0; virtual ListIterator inspect(const metkit::mars::MarsRequest& request) = 0; @@ -87,8 +86,7 @@ class FDBBase : private eckit::NonCopyable, public CallbackRegistry { virtual StatsIterator stats(const FDBToolRequest& request) = 0; - virtual ControlIterator control(const FDBToolRequest& request, - ControlAction action, + virtual ControlIterator control(const FDBToolRequest& request, ControlAction action, ControlIdentifiers identifier) = 0; virtual MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) = 0; @@ -113,7 +111,6 @@ class FDBBase : private eckit::NonCopyable, public CallbackRegistry { bool enabled(const ControlIdentifier& controlIdentifier) const; private: // methods - virtual void print(std::ostream& s) const = 0; friend std::ostream& operator<<(std::ostream& s, const FDBBase& f) { @@ -122,7 +119,6 @@ class FDBBase : private eckit::NonCopyable, public CallbackRegistry { } protected: // members - const std::string name_; Config config_; @@ -130,7 +126,6 @@ class FDBBase : private eckit::NonCopyable, public CallbackRegistry { ControlIdentifiers controlIdentifiers_; bool disabled_; - }; //---------------------------------------------------------------------------------------------------------------------- @@ -139,7 +134,6 @@ class FDBBuilderBase; class FDBFactory { public: - static FDBFactory& instance(); void add(const std::string& name, const FDBBuilderBase*); @@ -147,45 +141,36 @@ class FDBFactory { std::unique_ptr build(const Config& config); private: - FDBFactory() {} ///< private constructor only used by singleton eckit::Mutex mutex_; std::map registry_; - }; - class FDBBuilderBase { public: // methods - virtual std::unique_ptr make(const Config& config) const = 0; protected: // methods - FDBBuilderBase(const std::string& name); virtual ~FDBBuilderBase(); protected: // members - std::string name_; }; - - template class FDBBuilder : public FDBBuilderBase { - static_assert(std::is_base_of::value, "FDB Factorys can only build implementations of the FDB interface"); + static_assert(std::is_base_of::value, + "FDB Factorys can only build implementations of the FDB interface"); public: // methods - FDBBuilder(const std::string& name) : FDBBuilderBase(name) {} private: // methods - std::unique_ptr make(const Config& config) const override { return std::unique_ptr(new T(config, name_)); } diff --git a/src/fdb5/api/FDBStats.cc b/src/fdb5/api/FDBStats.cc index b1cafc670..74f4e6ca7 100644 --- a/src/fdb5/api/FDBStats.cc +++ b/src/fdb5/api/FDBStats.cc @@ -14,39 +14,36 @@ */ #include "eckit/filesystem/PathName.h" +#include "eckit/log/Bytes.h" #include "eckit/log/Log.h" -#include "eckit/log/Timer.h" #include "eckit/log/Seconds.h" -#include "eckit/log/Bytes.h" +#include "eckit/log/Timer.h" -#include "fdb5/api/FDBStats.h" #include "fdb5/LibFdb5.h" - +#include "fdb5/api/FDBStats.h" using namespace eckit; namespace fdb5 { -FDBStats::FDBStats() : - numArchive_(0), - numLocation_(0), - numFlush_(0), - numRetrieve_(0), - bytesArchive_(0), - bytesRetrieve_(0), - sumBytesArchiveSquared_(0), - sumBytesRetrieveSquared_(0), - elapsedArchive_(0), - elapsedFlush_(0), - elapsedRetrieve_(0), - sumArchiveTimingSquared_(0), - sumRetrieveTimingSquared_(0), - sumFlushTimingSquared_(0) {} - +FDBStats::FDBStats() + : numArchive_(0) + , numLocation_(0) + , numFlush_(0) + , numRetrieve_(0) + , bytesArchive_(0) + , bytesRetrieve_(0) + , sumBytesArchiveSquared_(0) + , sumBytesRetrieveSquared_(0) + , elapsedArchive_(0) + , elapsedFlush_(0) + , elapsedRetrieve_(0) + , sumArchiveTimingSquared_(0) + , sumRetrieveTimingSquared_(0) + , sumFlushTimingSquared_(0) {} FDBStats::~FDBStats() {} - FDBStats& FDBStats::operator+=(const FDBStats& rhs) { numArchive_ += rhs.numArchive_; numLocation_ += rhs.numLocation_; @@ -65,7 +62,6 @@ FDBStats& FDBStats::operator+=(const FDBStats& rhs) { return *this; } - void FDBStats::addArchive(size_t length, eckit::Timer& timer, size_t nfields) { numArchive_ += nfields; @@ -76,11 +72,9 @@ void FDBStats::addArchive(size_t length, eckit::Timer& timer, size_t nfields) { elapsedArchive_ += elapsed; sumArchiveTimingSquared_ += elapsed * elapsed; - LOG_DEBUG_LIB(LibFdb5) << "Archive count: " << numArchive_ - << ", size: " << Bytes(length) - << ", total: " << Bytes(bytesArchive_) - << ", time: " << Seconds(elapsed) - << ", total: " << Seconds(elapsedArchive_) << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "Archive count: " << numArchive_ << ", size: " << Bytes(length) + << ", total: " << Bytes(bytesArchive_) << ", time: " << Seconds(elapsed) + << ", total: " << Seconds(elapsedArchive_) << std::endl; } void FDBStats::addLocation(size_t nfields) { @@ -97,14 +91,11 @@ void FDBStats::addRetrieve(size_t length, eckit::Timer& timer) { elapsedRetrieve_ += elapsed; sumRetrieveTimingSquared_ += elapsed * elapsed; - LOG_DEBUG_LIB(LibFdb5) << "Retrieve count: " << numRetrieve_ - << ", size: " << Bytes(length) - << ", total: " << Bytes(bytesRetrieve_) - << ", time: " << Seconds(elapsed) - << ", total: " << Seconds(elapsedRetrieve_) << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "Retrieve count: " << numRetrieve_ << ", size: " << Bytes(length) + << ", total: " << Bytes(bytesRetrieve_) << ", time: " << Seconds(elapsed) + << ", total: " << Seconds(elapsedRetrieve_) << std::endl; } - void FDBStats::addFlush(eckit::Timer& timer) { numFlush_++; @@ -113,13 +104,10 @@ void FDBStats::addFlush(eckit::Timer& timer) { elapsedFlush_ += elapsed; sumFlushTimingSquared_ += elapsed * elapsed; - LOG_DEBUG_LIB(LibFdb5) << "Flush count: " << numFlush_ - << ", time: " << elapsed << "s" - << ", total: " << elapsedFlush_ << "s" << std::endl; - + LOG_DEBUG_LIB(LibFdb5) << "Flush count: " << numFlush_ << ", time: " << elapsed << "s" + << ", total: " << elapsedFlush_ << "s" << std::endl; } - void FDBStats::report(std::ostream& out, const char* prefix) const { // Archive statistics @@ -144,4 +132,4 @@ void FDBStats::report(std::ostream& out, const char* prefix) const { //---------------------------------------------------------------------------------------------------------------------- -} // namespace fbdb5 +} // namespace fdb5 diff --git a/src/fdb5/api/FDBStats.h b/src/fdb5/api/FDBStats.h index cf776379e..174f1a229 100644 --- a/src/fdb5/api/FDBStats.h +++ b/src/fdb5/api/FDBStats.h @@ -24,14 +24,12 @@ #include "eckit/log/Statistics.h" - namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- class FDBStats : public eckit::Statistics { public: - FDBStats(); ~FDBStats(); @@ -39,8 +37,8 @@ class FDBStats : public eckit::Statistics { size_t numLocation() const { return numLocation_; } size_t numFlush() const { return numFlush_; } - void addArchive(size_t length, eckit::Timer& timer, size_t nfields=1); - void addLocation(size_t nfields=1); + void addArchive(size_t length, eckit::Timer& timer, size_t nfields = 1); + void addLocation(size_t nfields = 1); void addRetrieve(size_t length, eckit::Timer& timer); void addFlush(eckit::Timer& timer); @@ -49,7 +47,6 @@ class FDBStats : public eckit::Statistics { FDBStats& operator+=(const FDBStats& rhs); private: // members - size_t numArchive_; size_t numLocation_; size_t numFlush_; diff --git a/src/fdb5/api/LocalFDB.cc b/src/fdb5/api/LocalFDB.cc index 215532105..13b5ff1ea 100644 --- a/src/fdb5/api/LocalFDB.cc +++ b/src/fdb5/api/LocalFDB.cc @@ -17,9 +17,10 @@ #include "eckit/log/Log.h" #include "eckit/message/Message.h" -#include "fdb5/api/helpers/ListIterator.h" -#include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/LibFdb5.h" #include "fdb5/api/LocalFDB.h" +#include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/api/helpers/ListIterator.h" #include "fdb5/database/Archiver.h" #include "fdb5/database/Catalogue.h" #include "fdb5/database/EntryVisitMechanism.h" @@ -27,24 +28,21 @@ #include "fdb5/database/Inspector.h" #include "fdb5/database/Key.h" #include "fdb5/rules/Schema.h" -#include "fdb5/LibFdb5.h" #include "fdb5/api/local/AxesVisitor.h" #include "fdb5/api/local/ControlVisitor.h" #include "fdb5/api/local/DumpVisitor.h" #include "fdb5/api/local/ListVisitor.h" +#include "fdb5/api/local/MoveVisitor.h" #include "fdb5/api/local/PurgeVisitor.h" #include "fdb5/api/local/QueryVisitor.h" #include "fdb5/api/local/StatsVisitor.h" #include "fdb5/api/local/StatusVisitor.h" #include "fdb5/api/local/WipeVisitor.h" -#include "fdb5/api/local/MoveVisitor.h" - using namespace fdb5::api::local; using namespace eckit; - namespace fdb5 { void LocalFDB::archive(const Key& key, const void* data, size_t length) { @@ -57,7 +55,7 @@ void LocalFDB::archive(const Key& key, const void* data, size_t length) { archiver_->archive(key, data, length); } -ListIterator LocalFDB::inspect(const metkit::mars::MarsRequest &request) { +ListIterator LocalFDB::inspect(const metkit::mars::MarsRequest& request) { if (!inspector_) { LOG_DEBUG_LIB(LibFdb5) << *this << ": Constructing new retriever" << std::endl; @@ -67,14 +65,14 @@ ListIterator LocalFDB::inspect(const metkit::mars::MarsRequest &request) { return inspector_->inspect(request); } -template -APIIterator LocalFDB::queryInternal(const FDBToolRequest& request, Ts ... args) { +template +APIIterator LocalFDB::queryInternal(const FDBToolRequest& request, Ts... args) { using ValueType = typename VisitorType::ValueType; using QueryIterator = APIIterator; using AsyncIterator = APIAsyncIterator; - auto async_worker = [this, request, args...] (Queue& queue) { + auto async_worker = [this, request, args...](Queue& queue) { EntryVisitMechanism mechanism(config_); VisitorType visitor(queue, request.request(), args...); mechanism.visit(request, visitor); @@ -88,17 +86,17 @@ ListIterator LocalFDB::list(const FDBToolRequest& request) { return queryInternal(request); } -DumpIterator LocalFDB::dump(const FDBToolRequest &request, bool simple) { +DumpIterator LocalFDB::dump(const FDBToolRequest& request, bool simple) { LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::dump() : " << request << std::endl; return queryInternal(request, simple); } -StatusIterator LocalFDB::status(const FDBToolRequest &request) { +StatusIterator LocalFDB::status(const FDBToolRequest& request) { LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::status() : " << request << std::endl; return queryInternal(request); } -WipeIterator LocalFDB::wipe(const FDBToolRequest &request, bool doit, bool porcelain, bool unsafeWipeAll) { +WipeIterator LocalFDB::wipe(const FDBToolRequest& request, bool doit, bool porcelain, bool unsafeWipeAll) { LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::wipe() : " << request << std::endl; return queryInternal(request, doit, porcelain, unsafeWipeAll); } @@ -118,9 +116,7 @@ StatsIterator LocalFDB::stats(const FDBToolRequest& request) { return queryInternal(request); } -ControlIterator LocalFDB::control(const FDBToolRequest& request, - ControlAction action, - ControlIdentifiers identifiers) { +ControlIterator LocalFDB::control(const FDBToolRequest& request, ControlAction action, ControlIdentifiers identifiers) { LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::control() : " << request << std::endl; return queryInternal(request, action, identifiers); } @@ -137,12 +133,10 @@ void LocalFDB::flush() { } } - -void LocalFDB::print(std::ostream &s) const { +void LocalFDB::print(std::ostream& s) const { s << "LocalFDB(home=" << config_.expandPath("~fdb") << ")"; } - static FDBBuilder localFdbBuilder("local"); static FDBBuilder builder("catalogue"); // Enable type=catalogue to build localFDB (serverside). //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/api/LocalFDB.h b/src/fdb5/api/LocalFDB.h index 05c4d3c8f..c7dafcb26 100644 --- a/src/fdb5/api/LocalFDB.h +++ b/src/fdb5/api/LocalFDB.h @@ -21,7 +21,6 @@ #include "fdb5/api/FDBFactory.h" - namespace fdb5 { class Inspector; @@ -33,7 +32,6 @@ class FDB; class LocalFDB : public FDBBase { public: // methods - using FDBBase::FDBBase; using FDBBase::stats; @@ -53,8 +51,7 @@ class LocalFDB : public FDBBase { StatsIterator stats(const FDBToolRequest& request) override; - ControlIterator control(const FDBToolRequest& request, - ControlAction action, + ControlIterator control(const FDBToolRequest& request, ControlAction action, ControlIdentifiers identifiers) override; MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) override; @@ -64,16 +61,13 @@ class LocalFDB : public FDBBase { void flush() override; protected: // methods - - template - APIIterator queryInternal(const FDBToolRequest& request, Ts ... args); + template + APIIterator queryInternal(const FDBToolRequest& request, Ts... args); private: // methods - void print(std::ostream& s) const override; protected: // members - std::string home_; std::unique_ptr archiver_; diff --git a/src/fdb5/api/RandomFDB.cc b/src/fdb5/api/RandomFDB.cc index 286993b5a..df2ccf2ba 100644 --- a/src/fdb5/api/RandomFDB.cc +++ b/src/fdb5/api/RandomFDB.cc @@ -16,14 +16,13 @@ #include "eckit/exception/Exceptions.h" #include "eckit/log/Log.h" -#include "fdb5/api/FDBFactory.h" #include "fdb5/LibFdb5.h" +#include "fdb5/api/FDBFactory.h" #include using namespace eckit; - namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- @@ -31,28 +30,28 @@ namespace fdb5 { class RandomFDBBuilder : public FDBBuilderBase { public: // methods - RandomFDBBuilder() : FDBBuilderBase("random") {} private: - std::unique_ptr make(const Config& config) const { // Get a list of FDBs to choose between - if (!config.has("fdbs")) throw UserError("No FDBs configured for random FDB frontend", Here()); + if (!config.has("fdbs")) + throw UserError("No FDBs configured for random FDB frontend", Here()); auto fdbConfigs = config.getSubConfigurations("fdbs"); - if (fdbConfigs.empty()) throw UserError("No FDBs configured for random FDB frontend", Here()); + if (fdbConfigs.empty()) + throw UserError("No FDBs configured for random FDB frontend", Here()); // And pick one of them at random to build std::random_device rd; - int choice = std::uniform_int_distribution(0, fdbConfigs.size()-1)(rd); + int choice = std::uniform_int_distribution(0, fdbConfigs.size() - 1)(rd); - LOG_DEBUG_LIB(LibFdb5) << "Constructing random API instance: " << choice+1 - << " / " << fdbConfigs.size() << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "Constructing random API instance: " << choice + 1 << " / " << fdbConfigs.size() + << std::endl; ASSERT(choice >= 0); ASSERT(choice < fdbConfigs.size()); diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc index c2401657f..7da378d37 100644 --- a/src/fdb5/api/RemoteFDB.cc +++ b/src/fdb5/api/RemoteFDB.cc @@ -6,13 +6,13 @@ #include "eckit/serialisation/MemoryStream.h" #include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/LibFdb5.h" #include "fdb5/api/RemoteFDB.h" #include "fdb5/database/Archiver.h" #include "fdb5/database/Inspector.h" -#include "fdb5/LibFdb5.h" -#include "fdb5/remote/client/ClientConnectionRouter.h" #include "fdb5/remote/RemoteFieldLocation.h" +#include "fdb5/remote/client/ClientConnectionRouter.h" using namespace fdb5::remote; using namespace eckit; @@ -24,7 +24,7 @@ struct BaseAPIHelper { typedef T ValueType; - static size_t bufferSize() { return 1024*1024; } + static size_t bufferSize() { return 1024 * 1024; } static size_t queueSize() { return 100; } static fdb5::remote::Message message() { return msgID; } @@ -47,10 +47,14 @@ struct ListHelper : BaseAPIHelper remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(fieldLocationEndpoint), static_cast(elem.location())).make_shared(); + std::shared_ptr remoteLocation = + fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(fieldLocationEndpoint), + static_cast(elem.location())) + .make_shared(); return fdb5::ListElement(elem.key(), remoteLocation, elem.timestamp()); } - std::shared_ptr remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(), elem.location()).make_shared(); + std::shared_ptr remoteLocation = + fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(), elem.location()).make_shared(); return fdb5::ListElement(elem.key(), remoteLocation, elem.timestamp()); } }; @@ -58,9 +62,8 @@ struct ListHelper : BaseAPIHelper { AxesHelper(int level) : level_(level) {} - void encodeExtra(eckit::Stream& s) const { - s << level_; - } + void encodeExtra(eckit::Stream& s) const { s << level_; } + private: int level_; }; @@ -77,15 +80,19 @@ struct InspectHelper : BaseAPIHelper remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(fieldLocationEndpoint), static_cast(elem.location())).make_shared(); + std::shared_ptr remoteLocation = + fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(fieldLocationEndpoint), + static_cast(elem.location())) + .make_shared(); return fdb5::ListElement(elem.key(), remoteLocation, elem.timestamp()); } - std::shared_ptr remoteLocation = fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(), elem.location()).make_shared(); + std::shared_ptr remoteLocation = + fdb5::remote::RemoteFieldLocation(fdb->storeEndpoint(), elem.location()).make_shared(); return fdb5::ListElement(elem.key(), remoteLocation, elem.timestamp()); } }; -} +} // namespace namespace fdb5 { @@ -110,9 +117,9 @@ const eckit::net::Endpoint& RemoteFDB::storeEndpoint(const eckit::net::Endpoint& return it->second; } -RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name): - LocalFDB(config, name), - Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port")), "") { +RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name) + : LocalFDB(config, name) + , Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port")), "") { eckit::Buffer buf = controlWriteReadResponse(remote::Message::Stores, generateRequestID()); eckit::MemoryStream s(buf); @@ -124,8 +131,8 @@ RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name std::vector stores; std::vector fieldLocationEndpoints; - - for (size_t i=0; i> store; size_t numAliases; @@ -134,9 +141,10 @@ RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name if (numAliases == 0) { eckit::net::Endpoint storeEndpoint{store}; storesReadMapping_[storeEndpoint] = storeEndpoint; - LOG_DEBUG_LIB(LibFdb5) << "store endpoint: " << storeEndpoint << " default data location endpoint: " << storeEndpoint << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "store endpoint: " << storeEndpoint + << " default data location endpoint: " << storeEndpoint << std::endl; } else { - for (size_t j=0; j(controlEndpoint())+"/schema", schema); + config_.overrideSchema(static_cast(controlEndpoint()) + "/schema", schema); } // ----------------------------------------------------------------------------------------------------- @@ -179,9 +188,9 @@ RemoteFDB::RemoteFDB(const eckit::Configuration& config, const std::string& name // ii) Encode the request+arguments and send them to the server // iii) Return an AsyncIterator that pulls messages off the queue, and returns them to the caller. - template -auto RemoteFDB::forwardApiCall(const HelperClass& helper, const FDBToolRequest& request) -> APIIterator { +auto RemoteFDB::forwardApiCall(const HelperClass& helper, const FDBToolRequest& request) + -> APIIterator { using ValueType = typename HelperClass::ValueType; using IteratorType = APIIterator; @@ -211,18 +220,17 @@ auto RemoteFDB::forwardApiCall(const HelperClass& helper, const FDBToolRequest& // n.b. Don't worry about catching exceptions in lambda, as // this is handled in the AsyncIterator. new AsyncIterator([messageQueue, remoteFDB](eckit::Queue& queue) { - eckit::Buffer msg{0}; - while (true) { - if (messageQueue->pop(msg) == -1) { - break; - } else { - MemoryStream s(msg); - queue.emplace(HelperClass::valueFromStream(s, remoteFDB)); - } + eckit::Buffer msg{0}; + while (true) { + if (messageQueue->pop(msg) == -1) { + break; + } else { + MemoryStream s(msg); + queue.emplace(HelperClass::valueFromStream(s, remoteFDB)); } - // messageQueue goes out of scope --> destructed - }) - ); + } + // messageQueue goes out of scope --> destructed + })); } ListIterator RemoteFDB::list(const FDBToolRequest& request) { @@ -247,7 +255,7 @@ void RemoteFDB::print(std::ostream& s) const { // Client bool RemoteFDB::handle(remote::Message message, uint32_t requestID) { - + switch (message) { case fdb5::remote::Message::Complete: { diff --git a/src/fdb5/api/RemoteFDB.h b/src/fdb5/api/RemoteFDB.h index 7524896bc..9b42a307a 100644 --- a/src/fdb5/api/RemoteFDB.h +++ b/src/fdb5/api/RemoteFDB.h @@ -34,11 +34,9 @@ class Archiver; class RemoteFDB : public LocalFDB, public remote::Client { public: // types - using MessageQueue = eckit::Queue; public: // method - RemoteFDB(const eckit::Configuration& config, const std::string& name); ~RemoteFDB() override {} @@ -46,7 +44,7 @@ class RemoteFDB : public LocalFDB, public remote::Client { ListIterator list(const FDBToolRequest& request) override; - AxesIterator axesIterator(const FDBToolRequest& request, int level=3) override; + AxesIterator axesIterator(const FDBToolRequest& request, int level = 3) override; DumpIterator dump(const FDBToolRequest& request, bool simple) override { NOTIMP; } @@ -58,9 +56,10 @@ class RemoteFDB : public LocalFDB, public remote::Client { StatsIterator stats(const FDBToolRequest& request) override; - ControlIterator control(const FDBToolRequest& request, - ControlAction action, - ControlIdentifiers identifiers) override { NOTIMP; } + ControlIterator control(const FDBToolRequest& request, ControlAction action, + ControlIdentifiers identifiers) override { + NOTIMP; + } MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) override { NOTIMP; } @@ -68,9 +67,9 @@ class RemoteFDB : public LocalFDB, public remote::Client { const eckit::net::Endpoint& storeEndpoint(const eckit::net::Endpoint& fieldLocationEndpoint) const; private: // methods - template - auto forwardApiCall(const HelperClass& helper, const FDBToolRequest& request) -> APIIterator; + auto forwardApiCall(const HelperClass& helper, const FDBToolRequest& request) + -> APIIterator; void print(std::ostream& s) const override; @@ -81,7 +80,6 @@ class RemoteFDB : public LocalFDB, public remote::Client { bool handle(remote::Message message, uint32_t requestID, eckit::Buffer&& payload) override; private: // members - std::unordered_map storesReadMapping_; std::vector> storesArchiveMapping_; std::vector storesLocalFields_; diff --git a/src/fdb5/api/SelectFDB.cc b/src/fdb5/api/SelectFDB.cc index 5e704e2ab..7405026c5 100644 --- a/src/fdb5/api/SelectFDB.cc +++ b/src/fdb5/api/SelectFDB.cc @@ -15,14 +15,14 @@ #include "eckit/log/Log.h" #include "eckit/message/Message.h" -#include "eckit/utils/Tokenizer.h" #include "eckit/types/Types.h" +#include "eckit/utils/Tokenizer.h" -#include "fdb5/api/helpers/ListIterator.h" +#include "fdb5/LibFdb5.h" #include "fdb5/api/SelectFDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/api/helpers/ListIterator.h" #include "fdb5/io/HandleGatherer.h" -#include "fdb5/LibFdb5.h" using namespace eckit; @@ -32,7 +32,6 @@ static FDBBuilder selectFdbBuilder("select"); //---------------------------------------------------------------------------------------------------------------------- - std::map parseFDBSelect(const eckit::LocalConfiguration& config) { std::map selectDict; @@ -62,9 +61,7 @@ std::map parseFDBSelect(const eckit::LocalConfigurati return selectDict; } - -SelectFDB::SelectFDB(const Config& config, const std::string& name) : - FDBBase(config, name) { +SelectFDB::SelectFDB(const Config& config, const std::string& name) : FDBBase(config, name) { ASSERT(config.getString("type", "") == "select"); @@ -77,7 +74,6 @@ SelectFDB::SelectFDB(const Config& config, const std::string& name) : } } - SelectFDB::~SelectFDB() {} void SelectFDB::archive(const Key& key, const void* data, size_t length) { @@ -117,7 +113,8 @@ ListIterator SelectFDB::inspect(const metkit::mars::MarsRequest& request) { } template -auto SelectFDB::queryInternal(const FDBToolRequest& request, const QueryFN& fn) -> decltype(fn(*(FDB*)(nullptr), request)) { +auto SelectFDB::queryInternal(const FDBToolRequest& request, const QueryFN& fn) + -> decltype(fn(*(FDB*)(nullptr), request)) { using QueryIterator = decltype(fn(*(FDB*)(nullptr), request)); using ValueType = typename QueryIterator::value_type; @@ -139,69 +136,51 @@ auto SelectFDB::queryInternal(const FDBToolRequest& request, const QueryFN& fn) ListIterator SelectFDB::list(const FDBToolRequest& request) { LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::list() >> " << request << std::endl; - return queryInternal(request, - [](FDB& fdb, const FDBToolRequest& request) { - return fdb.list(request); - }); + return queryInternal(request, [](FDB& fdb, const FDBToolRequest& request) { return fdb.list(request); }); } DumpIterator SelectFDB::dump(const FDBToolRequest& request, bool simple) { LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::dump() >> " << request << std::endl; return queryInternal(request, - [simple](FDB& fdb, const FDBToolRequest& request) { - return fdb.dump(request, simple); - }); + [simple](FDB& fdb, const FDBToolRequest& request) { return fdb.dump(request, simple); }); } StatusIterator SelectFDB::status(const FDBToolRequest& request) { LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::status() >> " << request << std::endl; - return queryInternal(request, - [](FDB& fdb, const FDBToolRequest& request) { - return fdb.status(request); - }); + return queryInternal(request, [](FDB& fdb, const FDBToolRequest& request) { return fdb.status(request); }); } WipeIterator SelectFDB::wipe(const FDBToolRequest& request, bool doit, bool porcelain, bool unsafeWipeAll) { LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::wipe() >> " << request << std::endl; - return queryInternal(request, - [doit, porcelain, unsafeWipeAll](FDB& fdb, const FDBToolRequest& request) { - return fdb.wipe(request, doit, porcelain, unsafeWipeAll); + return queryInternal(request, [doit, porcelain, unsafeWipeAll](FDB& fdb, const FDBToolRequest& request) { + return fdb.wipe(request, doit, porcelain, unsafeWipeAll); }); } PurgeIterator SelectFDB::purge(const FDBToolRequest& request, bool doit, bool porcelain) { LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::purge() >> " << request << std::endl; - return queryInternal(request, - [doit, porcelain](FDB& fdb, const FDBToolRequest& request) { - return fdb.purge(request, doit, porcelain); + return queryInternal(request, [doit, porcelain](FDB& fdb, const FDBToolRequest& request) { + return fdb.purge(request, doit, porcelain); }); } -StatsIterator SelectFDB::stats(const FDBToolRequest &request) { +StatsIterator SelectFDB::stats(const FDBToolRequest& request) { LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::stats() >> " << request << std::endl; - return queryInternal(request, - [](FDB& fdb, const FDBToolRequest& request) { - return fdb.stats(request); - }); + return queryInternal(request, [](FDB& fdb, const FDBToolRequest& request) { return fdb.stats(request); }); } -ControlIterator SelectFDB::control(const FDBToolRequest& request, - ControlAction action, +ControlIterator SelectFDB::control(const FDBToolRequest& request, ControlAction action, ControlIdentifiers identifiers) { LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::control >> " << request << std::endl; - return queryInternal(request, - [action, identifiers](FDB& fdb, const FDBToolRequest& request) { - return fdb.control(request, action, identifiers); - + return queryInternal(request, [action, identifiers](FDB& fdb, const FDBToolRequest& request) { + return fdb.control(request, action, identifiers); }); } AxesIterator SelectFDB::axesIterator(const FDBToolRequest& request, int level) { LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::axesIterator() >> " << request << std::endl; return queryInternal(request, - [level](FDB& fdb, const FDBToolRequest& request) { - return fdb.axesIterator(request, level); - }); + [level](FDB& fdb, const FDBToolRequest& request) { return fdb.axesIterator(request, level); }); } void SelectFDB::flush() { @@ -211,12 +190,11 @@ void SelectFDB::flush() { } } - -void SelectFDB::print(std::ostream &s) const { +void SelectFDB::print(std::ostream& s) const { s << "SelectFDB()"; } -bool SelectFDB::matches(const Key& key, const SelectMap &select, bool requireMissing) const { +bool SelectFDB::matches(const Key& key, const SelectMap& select, bool requireMissing) const { for (const auto& kv : select) { @@ -225,7 +203,8 @@ bool SelectFDB::matches(const Key& key, const SelectMap &select, bool requireMis eckit::StringDict::const_iterator i = key.find(k); if (i == key.end()) { - if (requireMissing) return false; + if (requireMissing) + return false; } else if (!re.match(i->second)) { return false; } @@ -234,7 +213,7 @@ bool SelectFDB::matches(const Key& key, const SelectMap &select, bool requireMis return true; } -bool SelectFDB::matches(const metkit::mars::MarsRequest& request, const SelectMap &select, bool requireMissing) const { +bool SelectFDB::matches(const metkit::mars::MarsRequest& request, const SelectMap& select, bool requireMissing) const { for (const auto& kv : select) { @@ -244,7 +223,8 @@ bool SelectFDB::matches(const metkit::mars::MarsRequest& request, const SelectMa const std::vector& request_values = request.values(k, /* emptyOK */ true); if (request_values.size() == 0) { - if (requireMissing) return false; + if (requireMissing) + return false; } else { bool re_match = false; @@ -254,7 +234,8 @@ bool SelectFDB::matches(const metkit::mars::MarsRequest& request, const SelectMa break; } } - if (!re_match) return false; + if (!re_match) + return false; } } diff --git a/src/fdb5/api/SelectFDB.h b/src/fdb5/api/SelectFDB.h index b8678d9ab..c03d7625d 100644 --- a/src/fdb5/api/SelectFDB.h +++ b/src/fdb5/api/SelectFDB.h @@ -19,14 +19,14 @@ #ifndef fdb5_api_SelectFDB_H #define fdb5_api_SelectFDB_H -#include #include #include +#include #include "eckit/utils/Regex.h" -#include "fdb5/api/FDBFactory.h" #include "fdb5/api/FDB.h" +#include "fdb5/api/FDBFactory.h" namespace fdb5 { @@ -35,11 +35,9 @@ namespace fdb5 { class SelectFDB : public FDBBase { private: // types - using SelectMap = std::map; public: // methods - using FDBBase::stats; SelectFDB(const Config& config, const std::string& name); @@ -62,10 +60,9 @@ class SelectFDB : public FDBBase { StatsIterator stats(const FDBToolRequest& request) override; - ControlIterator control(const FDBToolRequest& request, - ControlAction action, + ControlIterator control(const FDBToolRequest& request, ControlAction action, ControlIdentifiers identifiers) override; - + MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) override { NOTIMP; } AxesIterator axesIterator(const FDBToolRequest& request, int level) override; @@ -73,7 +70,6 @@ class SelectFDB : public FDBBase { void flush() override; private: // methods - void print(std::ostream& s) const override; bool matches(const Key& key, const SelectMap& select, bool requireMissing) const; @@ -83,7 +79,6 @@ class SelectFDB : public FDBBase { auto queryInternal(const FDBToolRequest& request, const QueryFN& fn) -> decltype(fn(*(FDB*)(nullptr), request)); private: // members - std::vector> subFdbs_; }; diff --git a/src/fdb5/api/fdb_c.cc b/src/fdb5/api/fdb_c.cc index eb69bc9d5..2ca743295 100644 --- a/src/fdb5/api/fdb_c.cc +++ b/src/fdb5/api/fdb_c.cc @@ -8,20 +8,20 @@ * does it submit to any jurisdiction. */ +#include "eckit/config/YAMLConfiguration.h" #include "eckit/io/MemoryHandle.h" #include "eckit/message/Message.h" #include "eckit/runtime/Main.h" -#include "eckit/config/YAMLConfiguration.h" -#include "metkit/mars/MarsRequest.h" -#include "metkit/mars/MarsExpension.h" #include "eckit/utils/Tokenizer.h" +#include "metkit/mars/MarsExpension.h" +#include "metkit/mars/MarsRequest.h" -#include "fdb5/fdb5_version.h" #include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" #include "fdb5/api/helpers/ListIterator.h" #include "fdb5/database/Key.h" +#include "fdb5/fdb5_version.h" #include "fdb5/api/fdb_c.h" @@ -32,7 +32,6 @@ extern "C" { //---------------------------------------------------------------------------------------------------------------------- - struct fdb_handle_t : public FDB { using FDB::FDB; }; @@ -44,16 +43,14 @@ struct fdb_key_t : public Key { struct fdb_request_t { public: fdb_request_t() : request_(metkit::mars::MarsRequest()) {} - fdb_request_t(std::string str) { - request_ = metkit::mars::MarsRequest(str); - } + fdb_request_t(std::string str) { request_ = metkit::mars::MarsRequest(str); } size_t values(const char* name, char** values[]) { std::string n(name); std::vector vv = request_.values(name); *values = new char*[vv.size()]; for (size_t i = 0; i < vv.size(); i++) { - (*values)[i] = new char[vv[i].size()+1]; + (*values)[i] = new char[vv[i].size() + 1]; strncpy((*values)[i], vv[i].c_str(), vv[i].size()); (*values)[i][vv[i].size()] = '\0'; } @@ -64,7 +61,7 @@ struct fdb_request_t { std::vector vv; Tokenizer parse("/"); - for (int i=0; i result; parse(values[i], result); vv.insert(std::end(vv), std::begin(result), std::end(result)); @@ -83,6 +80,7 @@ struct fdb_request_t { request_ = expand.expand(request_); } const metkit::mars::MarsRequest request() const { return request_; } + private: metkit::mars::MarsRequest request_; }; @@ -111,7 +109,7 @@ struct fdb_split_key_t { } } while (it_ == key_->at(level_).end()) { - if (level_size()-1) { + if (level_ < key_->size() - 1) { level_++; it_ = key_->at(level_).begin(); } else { @@ -215,15 +213,15 @@ static void* g_failure_handler_context = nullptr; const char* fdb_error_string(int err) { switch (err) { - case FDB_SUCCESS: - return "Success"; - case FDB_ERROR_GENERAL_EXCEPTION: - case FDB_ERROR_UNKNOWN_EXCEPTION: - return g_current_error_str.c_str(); - case FDB_ITERATION_COMPLETE: - return "Iteration complete"; - default: - return ""; + case FDB_SUCCESS: + return "Success"; + case FDB_ERROR_GENERAL_EXCEPTION: + case FDB_ERROR_UNKNOWN_EXCEPTION: + return g_current_error_str.c_str(); + case FDB_ITERATION_COMPLETE: + return "Iteration complete"; + default: + return ""; }; } @@ -231,7 +229,6 @@ const char* fdb_error_string(int err) { // Template can't have C linkage - namespace { // Template magic to provide a consistent error-handling approach @@ -275,7 +272,7 @@ int wrapApiFunction(FN f) { ASSERT(false); } -} +} // namespace extern "C" { @@ -284,12 +281,11 @@ extern "C" { // TODO: In a sensible, templated, try catch all exceptions. // --> We ought to have a standardised error return process. - /* * Initialise API * @note This is only required if being used from a context where Main() * is not otherwise initialised -*/ + */ int fdb_initialise() { return wrapApiFunction([] { static bool initialised = false; @@ -306,7 +302,6 @@ int fdb_initialise() { }); } - int fdb_set_failure_handler(fdb_failure_handler_t handler, void* context) { return wrapApiFunction([handler, context] { g_failure_handler = handler; @@ -316,22 +311,15 @@ int fdb_set_failure_handler(fdb_failure_handler_t handler, void* context) { } int fdb_version(const char** version) { - return wrapApiFunction([version]{ - (*version) = fdb5_version_str(); - }); + return wrapApiFunction([version] { (*version) = fdb5_version_str(); }); } int fdb_vcs_version(const char** sha1) { - return wrapApiFunction([sha1]{ - (*sha1) = fdb5_git_sha1(); - }); + return wrapApiFunction([sha1] { (*sha1) = fdb5_git_sha1(); }); } - int fdb_new_handle(fdb_handle_t** fdb) { - return wrapApiFunction([fdb] { - *fdb = new fdb_handle_t(); - }); + return wrapApiFunction([fdb] { *fdb = new fdb_handle_t(); }); } int fdb_new_handle_from_yaml(fdb_handle_t** fdb, const char* system_config, const char* user_config) { @@ -341,7 +329,6 @@ int fdb_new_handle_from_yaml(fdb_handle_t** fdb, const char* system_config, cons cfg.expandConfig(); *fdb = new fdb_handle_t(cfg); }); - } int fdb_archive(fdb_handle_t* fdb, fdb_key_t* key, const char* data, size_t length) { @@ -361,8 +348,7 @@ int fdb_archive_multiple(fdb_handle_t* fdb, fdb_request_t* req, const char* data eckit::MemoryHandle handle(data, length); if (req) { fdb->archive(req->request(), handle); - } - else { + } else { fdb->archive(handle); } }); @@ -374,9 +360,7 @@ int fdb_list(fdb_handle_t* fdb, const fdb_request_t* req, fdb_listiterator_t** i ASSERT(it); std::vector minKeySet; // we consider an empty set - const FDBToolRequest toolRequest( - req ? req->request() : metkit::mars::MarsRequest(), - req == nullptr, minKeySet); + const FDBToolRequest toolRequest(req ? req->request() : metkit::mars::MarsRequest(), req == nullptr, minKeySet); *it = new fdb_listiterator_t(fdb->list(toolRequest, duplicates)); }); @@ -398,7 +382,7 @@ int fdb_flush(fdb_handle_t* fdb) { } int fdb_delete_handle(fdb_handle_t* fdb) { - return wrapApiFunction([fdb]{ + return wrapApiFunction([fdb] { ASSERT(fdb); delete fdb; }); @@ -407,12 +391,10 @@ int fdb_delete_handle(fdb_handle_t* fdb) { /** ancillary functions for creating/destroying FDB objects */ int fdb_new_key(fdb_key_t** key) { - return wrapApiFunction([key] { - *key = new fdb_key_t(); - }); + return wrapApiFunction([key] { *key = new fdb_key_t(); }); } int fdb_key_add(fdb_key_t* key, const char* param, const char* value) { - return wrapApiFunction([key, param, value]{ + return wrapApiFunction([key, param, value] { ASSERT(key); ASSERT(param); ASSERT(value); @@ -421,16 +403,14 @@ int fdb_key_add(fdb_key_t* key, const char* param, const char* value) { } int fdb_delete_key(fdb_key_t* key) { - return wrapApiFunction([key]{ + return wrapApiFunction([key] { ASSERT(key); delete key; }); } int fdb_new_request(fdb_request_t** req) { - return wrapApiFunction([req] { - *req = new fdb_request_t("retrieve"); - }); + return wrapApiFunction([req] { *req = new fdb_request_t("retrieve"); }); } int fdb_request_add(fdb_request_t* req, const char* param, const char* values[], int numValues) { return wrapApiFunction([req, param, values, numValues] { @@ -448,19 +428,17 @@ int fdb_request_get(fdb_request_t* req, const char* param, char** values[], size }); } int fdb_expand_request(fdb_request_t* req) { - return wrapApiFunction([req]{ - req->expand(); - }); + return wrapApiFunction([req] { req->expand(); }); } int fdb_delete_request(fdb_request_t* req) { - return wrapApiFunction([req]{ + return wrapApiFunction([req] { ASSERT(req); delete req; }); } int fdb_listiterator_next(fdb_listiterator_t* it) { - return wrapApiFunction(std::function {[it] { + return wrapApiFunction(std::function{[it] { ASSERT(it); return it->next(); }}); @@ -474,7 +452,7 @@ int fdb_listiterator_attrs(fdb_listiterator_t* it, const char** uri, size_t* off it->attrs(uri, off, len); }); } -int fdb_listiterator_splitkey(fdb_listiterator_t* it, fdb_split_key_t* key){ +int fdb_listiterator_splitkey(fdb_listiterator_t* it, fdb_split_key_t* key) { return wrapApiFunction([it, key] { ASSERT(it); ASSERT(key); @@ -482,20 +460,18 @@ int fdb_listiterator_splitkey(fdb_listiterator_t* it, fdb_split_key_t* key){ }); } int fdb_delete_listiterator(fdb_listiterator_t* it) { - return wrapApiFunction([it]{ + return wrapApiFunction([it] { ASSERT(it); delete it; }); } int fdb_new_splitkey(fdb_split_key_t** key) { - return wrapApiFunction([key] { - *key = new fdb_split_key_t(); - }); + return wrapApiFunction([key] { *key = new fdb_split_key_t(); }); } int fdb_splitkey_next_metadata(fdb_split_key_t* it, const char** key, const char** value, size_t* level) { - return wrapApiFunction(std::function {[it, key, value, level] { + return wrapApiFunction(std::function{[it, key, value, level] { ASSERT(it); ASSERT(key); ASSERT(value); @@ -503,19 +479,17 @@ int fdb_splitkey_next_metadata(fdb_split_key_t* it, const char** key, const char }}); } int fdb_delete_splitkey(fdb_split_key_t* key) { - return wrapApiFunction([key]{ + return wrapApiFunction([key] { ASSERT(key); delete key; }); } int fdb_new_datareader(fdb_datareader_t** dr) { - return wrapApiFunction([dr]{ - *dr = new fdb_datareader_t(); - }); + return wrapApiFunction([dr] { *dr = new fdb_datareader_t(); }); } int fdb_datareader_open(fdb_datareader_t* dr, long* size) { - return wrapApiFunction([dr, size]{ + return wrapApiFunction([dr, size] { ASSERT(dr); long tmp; tmp = dr->open(); @@ -524,32 +498,32 @@ int fdb_datareader_open(fdb_datareader_t* dr, long* size) { }); } int fdb_datareader_close(fdb_datareader_t* dr) { - return wrapApiFunction([dr]{ + return wrapApiFunction([dr] { ASSERT(dr); dr->close(); }); } int fdb_datareader_tell(fdb_datareader_t* dr, long* pos) { - return wrapApiFunction([dr, pos]{ + return wrapApiFunction([dr, pos] { ASSERT(dr); ASSERT(pos); *pos = dr->tell(); }); } int fdb_datareader_seek(fdb_datareader_t* dr, long pos) { - return wrapApiFunction([dr, pos]{ + return wrapApiFunction([dr, pos] { ASSERT(dr); dr->seek(pos); }); } int fdb_datareader_skip(fdb_datareader_t* dr, long count) { - return wrapApiFunction([dr, count]{ + return wrapApiFunction([dr, count] { ASSERT(dr); dr->skip(count); }); } -int fdb_datareader_read(fdb_datareader_t* dr, void *buf, long count, long* read) { - return wrapApiFunction([dr, buf, count, read]{ +int fdb_datareader_read(fdb_datareader_t* dr, void* buf, long count, long* read) { + return wrapApiFunction([dr, buf, count, read] { ASSERT(dr); ASSERT(buf); ASSERT(read); @@ -557,13 +531,13 @@ int fdb_datareader_read(fdb_datareader_t* dr, void *buf, long count, long* read) }); } int fdb_datareader_size(fdb_datareader_t* dr, long* size) { - return wrapApiFunction([=]{ + return wrapApiFunction([=] { ASSERT(dr); *size = dr->size(); }); } int fdb_delete_datareader(fdb_datareader_t* dr) { - return wrapApiFunction([dr]{ + return wrapApiFunction([dr] { ASSERT(dr); dr->set(nullptr); delete dr; diff --git a/src/fdb5/api/fdb_c.h b/src/fdb5/api/fdb_c.h index acaa1186f..0933c71ab 100644 --- a/src/fdb5/api/fdb_c.h +++ b/src/fdb5/api/fdb_c.h @@ -25,7 +25,6 @@ extern "C" { #include #include - /** \defgroup Initialisation */ /** @{ */ @@ -37,7 +36,6 @@ int fdb_initialise(); /** @} */ - /** \defgroup Version Accessors */ /** @{ */ @@ -54,16 +52,15 @@ int fdb_vcs_version(const char** version); ///@} - /** \defgroup Error Handling */ /** @{ */ /** Return codes */ enum FdbErrorValues { - FDB_SUCCESS = 0, - FDB_ERROR_GENERAL_EXCEPTION = 1, - FDB_ERROR_UNKNOWN_EXCEPTION = 2, - FDB_ITERATION_COMPLETE = 3 + FDB_SUCCESS = 0, + FDB_ERROR_GENERAL_EXCEPTION = 1, + FDB_ERROR_UNKNOWN_EXCEPTION = 2, + FDB_ITERATION_COMPLETE = 3 }; /** Returns a human-readable error message for the last error given an error code @@ -86,7 +83,6 @@ int fdb_set_failure_handler(fdb_failure_handler_t handler, void* context); /** @} */ - /** \defgroup Key */ /** @{ */ @@ -116,7 +112,6 @@ int fdb_delete_key(fdb_key_t* key); /** @} */ - /** \defgroup Request */ /** @{ */ @@ -162,7 +157,6 @@ int fdb_delete_request(fdb_request_t* req); /** @} */ - /** \defgroup SplitKey */ /** @{ */ @@ -176,8 +170,8 @@ typedef struct fdb_split_key_t fdb_split_key_t; */ int fdb_new_splitkey(fdb_split_key_t** key); -/** Returns the next set of metadata in a SplitKey object. For a given ListElement, the SplitKey represents the Keys associated with each level of the FDB index. - * Supports multiple fdb_split_key_t iterating over the same key. +/** Returns the next set of metadata in a SplitKey object. For a given ListElement, the SplitKey represents the Keys + * associated with each level of the FDB index. Supports multiple fdb_split_key_t iterating over the same key. * \param it SplitKey instance * \param key Key metadata name * \param value Key metadata value @@ -194,7 +188,6 @@ int fdb_delete_splitkey(fdb_split_key_t* key); /** @} */ - /** \defgroup ListIterator */ /** @{ */ @@ -231,7 +224,6 @@ int fdb_delete_listiterator(fdb_listiterator_t* it); /** @} */ - /** \defgroup DataReader */ /** @{ */ @@ -293,7 +285,7 @@ int fdb_datareader_size(fdb_datareader_t* dr, long* size); * \param read Actual size of the data read from the DataReader into the memory buffer * \returns Return code (#FdbErrorValues) */ -int fdb_datareader_read(fdb_datareader_t* dr, void *buf, long count, long* read); +int fdb_datareader_read(fdb_datareader_t* dr, void* buf, long count, long* read); /** Deallocates DataReader object and associated resources. * \param key DataReader instance @@ -303,7 +295,6 @@ int fdb_delete_datareader(fdb_datareader_t* dr); /** @} */ - /** \defgroup FDB API */ /** @{ */ @@ -337,7 +328,8 @@ int fdb_archive(fdb_handle_t* fdb, fdb_key_t* key, const char* data, size_t leng /** Archives multiple messages to a FDB instance. * \param fdb FDB instance. - * \param req If Request #req is not nullptr, the number of messages and their metadata are checked against the provided request + * \param req If Request #req is not nullptr, the number of messages and their metadata are checked against the provided + * request * \param data Pointer to the binary data to archive. Metadata are extracted from data headers * \param length Size of the data to archive * \returns Return code (#FdbErrorValues) @@ -347,7 +339,8 @@ int fdb_archive_multiple(fdb_handle_t* fdb, fdb_request_t* req, const char* data /** List all available data whose metadata matches a given user request. * \param fdb FDB instance. * \param req User Request - * \param it ListIterator than can be used to retrieve metadata and attributes of all ListElement matching the user Request #req + * \param it ListIterator than can be used to retrieve metadata and attributes of all ListElement matching the user + * Request #req * \param duplicates Boolean flag used to specify if duplicated ListElements are to be reported or not. * \returns Return code (#FdbErrorValues) */ diff --git a/src/fdb5/api/helpers/APIIterator.h b/src/fdb5/api/helpers/APIIterator.h index f75acb85e..1eaf5c376 100644 --- a/src/fdb5/api/helpers/APIIterator.h +++ b/src/fdb5/api/helpers/APIIterator.h @@ -21,10 +21,10 @@ #include "eckit/container/Queue.h" +#include #include #include #include -#include /* * Given a standard, copyable, element, provide a mechanism for iterating over @@ -39,7 +39,6 @@ template class APIIteratorBase { public: // methods - APIIteratorBase() {} virtual ~APIIteratorBase() {} @@ -52,22 +51,19 @@ template class APIIterator { public: // types - using value_type = ValueType; public: // methods - - APIIterator(APIIteratorBase* impl) : - impl_(impl) {} + APIIterator(APIIteratorBase* impl) : impl_(impl) {} /// Get the next element. Return false if at end bool next(ValueType& elem) { - if (!impl_) return false; + if (!impl_) + return false; return impl_->next(elem); } private: // members - std::unique_ptr> impl_; }; @@ -79,9 +75,7 @@ template class APIAggregateIterator : public APIIteratorBase { public: // methods - - APIAggregateIterator(std::queue>&& iterators) : - iterators_(std::move(iterators)) {} + APIAggregateIterator(std::queue>&& iterators) : iterators_(std::move(iterators)) {} ~APIAggregateIterator() override {} @@ -99,11 +93,9 @@ class APIAggregateIterator : public APIIteratorBase { } private: // members - std::queue> iterators_; }; - //---------------------------------------------------------------------------------------------------------------------- // For some uses, we have a generator function (i.e. through a visitor @@ -117,10 +109,8 @@ template class APIAsyncIterator : public APIIteratorBase { public: // methods - - APIAsyncIterator(std::function&)> workerFn, - size_t queueSize=100) : - queue_(queueSize) { + APIAsyncIterator(std::function&)> workerFn, size_t queueSize = 100) + : queue_(queueSize) { // Add a call to set_done() on the eckit::Queue. auto fullWorker = [workerFn, this] { @@ -144,18 +134,14 @@ class APIAsyncIterator : public APIIteratorBase { workerThread_.join(); } - bool next(ValueType& elem) override { - return !(queue_.pop(elem) == -1); - } + bool next(ValueType& elem) override { return !(queue_.pop(elem) == -1); } private: // members - eckit::Queue queue_; std::thread workerThread_; }; - //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 diff --git a/src/fdb5/api/helpers/AxesIterator.cc b/src/fdb5/api/helpers/AxesIterator.cc index 2d4ada8e5..09ef16725 100644 --- a/src/fdb5/api/helpers/AxesIterator.cc +++ b/src/fdb5/api/helpers/AxesIterator.cc @@ -19,8 +19,7 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -AxesElement::AxesElement(Key&& dbKey, IndexAxis&& axes) : - dbKey_(std::move(dbKey)), axes_(std::move(axes)) {} +AxesElement::AxesElement(Key&& dbKey, IndexAxis&& axes) : dbKey_(std::move(dbKey)), axes_(std::move(axes)) {} AxesElement::AxesElement(eckit::Stream& s) { s >> dbKey_; diff --git a/src/fdb5/api/helpers/AxesIterator.h b/src/fdb5/api/helpers/AxesIterator.h index 3da56a2cb..d7653b1d3 100644 --- a/src/fdb5/api/helpers/AxesIterator.h +++ b/src/fdb5/api/helpers/AxesIterator.h @@ -13,14 +13,14 @@ #pragma once -#include "fdb5/database/Key.h" -#include "fdb5/database/IndexAxis.h" #include "fdb5/api/helpers/APIIterator.h" +#include "fdb5/database/IndexAxis.h" +#include "fdb5/database/Key.h" namespace eckit { class Stream; class JSON; -} +} // namespace eckit namespace fdb5 { @@ -28,22 +28,24 @@ namespace fdb5 { class AxesElement { public: // methods - AxesElement() = default; AxesElement(Key&& dbKey, IndexAxis&& axis); explicit AxesElement(eckit::Stream& s); - [[ nodiscard ]] - const Key& key() const { return dbKey_; } + [[nodiscard]] + const Key& key() const { + return dbKey_; + } - [[ nodiscard ]] - const IndexAxis& axes() const { return axes_; } + [[nodiscard]] + const IndexAxis& axes() const { + return axes_; + } void print(std::ostream& out) const; size_t encodeSize() const; private: // methods - void encode(eckit::Stream& s) const; friend std::ostream& operator<<(std::ostream& os, const AxesElement& e) { @@ -57,7 +59,6 @@ class AxesElement { } private: // members - Key dbKey_; IndexAxis axes_; }; diff --git a/src/fdb5/api/helpers/Callback.h b/src/fdb5/api/helpers/Callback.h index a628eb8b7..d55c0a3dc 100644 --- a/src/fdb5/api/helpers/Callback.h +++ b/src/fdb5/api/helpers/Callback.h @@ -14,16 +14,17 @@ */ #pragma once -#include -#include "fdb5/database/Key.h" #include "fdb5/database/FieldLocation.h" +#include "fdb5/database/Key.h" +#include namespace fdb5 { class FDB; class CallbackRegistry; -using ArchiveCallback = std::function>)>; +using ArchiveCallback = std::function>)>; using FlushCallback = std::function; using ConstructorCallback = std::function; @@ -36,15 +37,12 @@ static const ConstructorCallback CALLBACK_CONSTRUCTOR_NOOP = [](auto&&...) {}; // This class provides a common interface for registering callbacks with an FDB object or a Store/Catalogue Handler. class CallbackRegistry { public: - - void registerFlushCallback(FlushCallback callback) {flushCallback_ = callback;} - void registerArchiveCallback(ArchiveCallback callback) {archiveCallback_ = callback;} + void registerFlushCallback(FlushCallback callback) { flushCallback_ = callback; } + void registerArchiveCallback(ArchiveCallback callback) { archiveCallback_ = callback; } protected: - FlushCallback flushCallback_ = CALLBACK_FLUSH_NOOP; ArchiveCallback archiveCallback_ = CALLBACK_ARCHIVE_NOOP; - }; } // namespace fdb5 diff --git a/src/fdb5/api/helpers/ControlIterator.cc b/src/fdb5/api/helpers/ControlIterator.cc index e0ea53602..786d4115f 100644 --- a/src/fdb5/api/helpers/ControlIterator.cc +++ b/src/fdb5/api/helpers/ControlIterator.cc @@ -32,9 +32,9 @@ eckit::Stream& operator>>(eckit::Stream& s, ControlAction& a) { //---------------------------------------------------------------------------------------------------------------------- -ControlIdentifierIterator::ControlIdentifierIterator(const ControlIdentifiers& identifiers) : - value_(0), - remaining_(identifiers.value_) { +ControlIdentifierIterator::ControlIdentifierIterator(const ControlIdentifiers& identifiers) + : value_(0) + , remaining_(identifiers.value_) { if (remaining_ != 0) { nextValue(); @@ -65,7 +65,8 @@ void ControlIdentifierIterator::nextValue() { return; } - if (value_ == 0) value_ = 1; + if (value_ == 0) + value_ = 1; while ((remaining_ & value_) == 0) { value_ <<= 1; @@ -77,14 +78,11 @@ void ControlIdentifierIterator::nextValue() { //---------------------------------------------------------------------------------------------------------------------- -ControlIdentifiers::ControlIdentifiers() : - value_(0) {} +ControlIdentifiers::ControlIdentifiers() : value_(0) {} -ControlIdentifiers::ControlIdentifiers(const ControlIdentifier& val) : - value_(static_cast(val)) { -} +ControlIdentifiers::ControlIdentifiers(const ControlIdentifier& val) : value_(static_cast(val)) {} -ControlIdentifiers::ControlIdentifiers(eckit::Stream &s) { +ControlIdentifiers::ControlIdentifiers(eckit::Stream& s) { s >> value_; } @@ -117,8 +115,8 @@ ControlIdentifiers operator|(const ControlIdentifier& lhs, const ControlIdentifi return (ControlIdentifiers(lhs) |= rhs); } -void ControlIdentifiers::print( std::ostream &out ) const { - std::string separator=""; +void ControlIdentifiers::print(std::ostream& out) const { + std::string separator = ""; out << "ControlIdentifiers["; @@ -131,7 +129,7 @@ void ControlIdentifiers::print( std::ostream &out ) const { out << "]"; } -std::ostream &operator<<(std::ostream &s, const ControlIdentifiers &x) { +std::ostream& operator<<(std::ostream& s, const ControlIdentifiers& x) { x.print(s); return s; } @@ -140,22 +138,18 @@ std::ostream &operator<<(std::ostream &s, const ControlIdentifiers &x) { using value_type = typename std::underlying_type::type; -ControlElement::ControlElement() : - location() {} +ControlElement::ControlElement() : location() {} + +ControlElement::ControlElement(const Catalogue& catalogue) : key(catalogue.key()), location(catalogue.uri()) { -ControlElement::ControlElement(const Catalogue& catalogue) : - key(catalogue.key()), location(catalogue.uri()) { - controlIdentifiers = ControlIdentifier::None; for (auto id : ControlIdentifierList) { - if (!catalogue.enabled(id)) controlIdentifiers |= id; + if (!catalogue.enabled(id)) + controlIdentifiers |= id; } } -ControlElement::ControlElement(eckit::Stream &s) : - key(s), - location(s), - controlIdentifiers(s) {} +ControlElement::ControlElement(eckit::Stream& s) : key(s), location(s), controlIdentifiers(s) {} void ControlElement::encode(eckit::Stream& s) const { s << key; diff --git a/src/fdb5/api/helpers/ControlIterator.h b/src/fdb5/api/helpers/ControlIterator.h index 187ccff9d..0835d4189 100644 --- a/src/fdb5/api/helpers/ControlIterator.h +++ b/src/fdb5/api/helpers/ControlIterator.h @@ -21,9 +21,9 @@ #include "fdb5/database/Key.h" namespace eckit { - class Stream; - class PathName; -} +class Stream; +class PathName; +} // namespace eckit namespace fdb5 { @@ -45,20 +45,16 @@ eckit::Stream& operator>>(eckit::Stream& s, ControlAction& a); enum class ControlIdentifier : uint16_t { None = 0, - List = 1 << 0, - Retrieve = 1 << 1, - Archive = 1 << 2, - Wipe = 1 << 3, + List = 1 << 0, + Retrieve = 1 << 1, + Archive = 1 << 2, + Wipe = 1 << 3, UniqueRoot = 1 << 4 }; -static const std::initializer_list ControlIdentifierList { - ControlIdentifier::List, - ControlIdentifier::Retrieve, - ControlIdentifier::Archive, - ControlIdentifier::Wipe, - ControlIdentifier::UniqueRoot -}; +static const std::initializer_list ControlIdentifierList{ + ControlIdentifier::List, ControlIdentifier::Retrieve, ControlIdentifier::Archive, ControlIdentifier::Wipe, + ControlIdentifier::UniqueRoot}; //---------------------------------------------------------------------------------------------------------------------- // An iterator to facilitate working with the ControlIdentifiers structure @@ -71,7 +67,6 @@ class ControlIdentifierIterator { value_type remaining_; public: // methods - ControlIdentifierIterator(const ControlIdentifiers& identifiers); ControlIdentifier operator*() const; @@ -82,7 +77,6 @@ class ControlIdentifierIterator { ControlIdentifierIterator& operator++(); private: // methods - void nextValue(); }; @@ -108,9 +102,8 @@ class ControlIdentifiers { ControlIdentifierIterator end() const; protected: // methods - - friend std::ostream &operator<<(std::ostream &s, const ControlIdentifiers &x); - void print( std::ostream &out ) const; + friend std::ostream& operator<<(std::ostream& s, const ControlIdentifiers& x); + void print(std::ostream& out) const; private: void encode(eckit::Stream& s) const; @@ -145,7 +138,6 @@ struct ControlElement { ControlIdentifiers controlIdentifiers; protected: // methods - void encode(eckit::Stream& s) const; friend eckit::Stream& operator<<(eckit::Stream& s, const ControlElement& e) { diff --git a/src/fdb5/api/helpers/FDBToolRequest.cc b/src/fdb5/api/helpers/FDBToolRequest.cc index 45762478c..179de946c 100644 --- a/src/fdb5/api/helpers/FDBToolRequest.cc +++ b/src/fdb5/api/helpers/FDBToolRequest.cc @@ -25,18 +25,14 @@ #include "fdb5/LibFdb5.h" - namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- - std::vector FDBToolRequest::requestsFromString(const std::string& request_str, - const std::vector minimumKeys, - bool raw, + const std::vector minimumKeys, bool raw, const std::string& verb) { - std::string full_string = verb + "," + request_str; // Use a dummy verb std::istringstream in(full_string); metkit::mars::MarsParser parser(in); @@ -83,18 +79,15 @@ std::vector FDBToolRequest::requestsFromString(const std::string return requests; } - -FDBToolRequest::FDBToolRequest(const metkit::mars::MarsRequest& r, - bool all, - const std::vector& minimumKeySet) : - request_(r), - all_(all) { +FDBToolRequest::FDBToolRequest(const metkit::mars::MarsRequest& r, bool all, + const std::vector& minimumKeySet) + : request_(r) + , all_(all) { checkMinimumKeys(request_, minimumKeySet); } -FDBToolRequest::FDBToolRequest(eckit::Stream& s) : - request_(s) { +FDBToolRequest::FDBToolRequest(eckit::Stream& s) : request_(s) { s >> all_; } @@ -104,10 +97,9 @@ const metkit::mars::MarsRequest& FDBToolRequest::request() const { bool FDBToolRequest::all() const { return all_; - } -void FDBToolRequest::print(std::ostream &s, const char* cr, const char* tab) const { +void FDBToolRequest::print(std::ostream& s, const char* cr, const char* tab) const { if (all_) { s << " -- ALL --"; @@ -121,7 +113,8 @@ void FDBToolRequest::encode(eckit::Stream& s) const { s << all_; } -void FDBToolRequest::checkMinimumKeys(const metkit::mars::MarsRequest& request, const std::vector& minimumKeys) { +void FDBToolRequest::checkMinimumKeys(const metkit::mars::MarsRequest& request, + const std::vector& minimumKeys) { for (std::vector::const_iterator j = minimumKeys.begin(); j != minimumKeys.end(); ++j) { if (!request.has(*j)) { throw eckit::UserError("Please provide a value for '" + (*j) + "'"); @@ -132,4 +125,3 @@ void FDBToolRequest::checkMinimumKeys(const metkit::mars::MarsRequest& request, //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 - diff --git a/src/fdb5/api/helpers/FDBToolRequest.h b/src/fdb5/api/helpers/FDBToolRequest.h index 52649ddbe..53ed5e152 100644 --- a/src/fdb5/api/helpers/FDBToolRequest.h +++ b/src/fdb5/api/helpers/FDBToolRequest.h @@ -26,11 +26,12 @@ #include "metkit/mars/MarsRequest.h" - namespace eckit { - class Stream; - namespace option { class CmdArgs; } +class Stream; +namespace option { +class CmdArgs; } +} // namespace eckit namespace fdb5 { @@ -38,14 +39,11 @@ namespace fdb5 { class FDBToolRequest { public: // methods - static std::vector requestsFromString(const std::string& request_str, const std::vector minimumKeys = {}, - bool raw=false, - const std::string& verb="retrieve"); + bool raw = false, const std::string& verb = "retrieve"); - FDBToolRequest(const metkit::mars::MarsRequest& r, - bool all=false, + FDBToolRequest(const metkit::mars::MarsRequest& r, bool all = false, const std::vector& minimumKeySet = std::vector()); FDBToolRequest(eckit::Stream&); @@ -54,10 +52,9 @@ class FDBToolRequest { bool all() const; - void print(std::ostream& s, const char* cr="\n", const char* tab="\t") const; + void print(std::ostream& s, const char* cr = "\n", const char* tab = "\t") const; protected: // methods - void encode(eckit::Stream& s) const; friend std::ostream& operator<<(std::ostream& os, const FDBToolRequest& r) { @@ -73,11 +70,9 @@ class FDBToolRequest { } private: // methods - static void checkMinimumKeys(const metkit::mars::MarsRequest& request, const std::vector& minimumKeys); private: // members - metkit::mars::MarsRequest request_; bool all_; diff --git a/src/fdb5/api/helpers/ListIterator.cc b/src/fdb5/api/helpers/ListIterator.cc index 5f17d0f0b..e8952cb5c 100644 --- a/src/fdb5/api/helpers/ListIterator.cc +++ b/src/fdb5/api/helpers/ListIterator.cc @@ -21,10 +21,13 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -ListElement::ListElement(const std::vector& keyParts, std::shared_ptr location, time_t timestamp) : - keyParts_(keyParts), location_(location), timestamp_(timestamp) {} +ListElement::ListElement(const std::vector& keyParts, std::shared_ptr location, + time_t timestamp) + : keyParts_(keyParts) + , location_(location) + , timestamp_(timestamp) {} -ListElement::ListElement(eckit::Stream &s) { +ListElement::ListElement(eckit::Stream& s) { s >> keyParts_; location_.reset(eckit::Reanimator::reanimate(s)); s >> timestamp_; @@ -41,7 +44,8 @@ Key ListElement::combinedKey() const { return combined; } -void ListElement::print(std::ostream& out, bool withLocation, bool withLength, bool withTimestamp, const char* sep) const { +void ListElement::print(std::ostream& out, bool withLocation, bool withLength, bool withTimestamp, + const char* sep) const { for (const auto& bit : keyParts_) { out << bit; } @@ -52,7 +56,8 @@ void ListElement::print(std::ostream& out, bool withLocation, bool withLength, b out << sep << "length=" << location_->length(); } } - if (withTimestamp) out << sep << "timestamp=" << timestamp_; + if (withTimestamp) + out << sep << "timestamp=" << timestamp_; } void ListElement::json(eckit::JSON& json) const { @@ -60,7 +65,7 @@ void ListElement::json(eckit::JSON& json) const { json << "length" << location_->length(); } -void ListElement::encode(eckit::Stream &s) const { +void ListElement::encode(eckit::Stream& s) const { s << keyParts_; s << *location_; s << timestamp_; diff --git a/src/fdb5/api/helpers/ListIterator.h b/src/fdb5/api/helpers/ListIterator.h index 79788d0ef..5e5ef3fbc 100644 --- a/src/fdb5/api/helpers/ListIterator.h +++ b/src/fdb5/api/helpers/ListIterator.h @@ -19,20 +19,20 @@ #ifndef fdb5_ListIterator_H #define fdb5_ListIterator_H -#include -#include -#include -#include #include +#include +#include +#include +#include -#include "fdb5/database/Key.h" -#include "fdb5/database/FieldLocation.h" #include "fdb5/api/helpers/APIIterator.h" +#include "fdb5/database/FieldLocation.h" +#include "fdb5/database/Key.h" namespace eckit { - class Stream; - class JSON; -} +class Stream; +class JSON; +} // namespace eckit namespace fdb5 { @@ -43,7 +43,6 @@ namespace fdb5 { class ListElement { public: // methods - ListElement() = default; ListElement(const std::vector& keyParts, std::shared_ptr location, time_t timestamp); ListElement(eckit::Stream& s); @@ -54,11 +53,11 @@ class ListElement { Key combinedKey() const; - void print(std::ostream& out, bool withLocation=false, bool withLength=false, bool withTimestamp=false, const char* sep = " ") const; + void print(std::ostream& out, bool withLocation = false, bool withLength = false, bool withTimestamp = false, + const char* sep = " ") const; void json(eckit::JSON& json) const; private: // methods - void encode(eckit::Stream& s) const; friend std::ostream& operator<<(std::ostream& os, const ListElement& e) { @@ -77,11 +76,9 @@ class ListElement { } public: // members - std::vector keyParts_; private: // members - std::shared_ptr location_; time_t timestamp_; }; @@ -96,11 +93,15 @@ using ListAsyncIterator = APIAsyncIterator; class ListIterator : public APIIterator { public: - ListIterator(APIIterator&& iter, bool deduplicate=false) : - APIIterator(std::move(iter)), seenKeys_({}), deduplicate_(deduplicate) {} + ListIterator(APIIterator&& iter, bool deduplicate = false) + : APIIterator(std::move(iter)) + , seenKeys_({}) + , deduplicate_(deduplicate) {} - ListIterator(ListIterator&& iter) : - APIIterator(std::move(iter)), seenKeys_(std::move(iter.seenKeys_)), deduplicate_(iter.deduplicate_) {} + ListIterator(ListIterator&& iter) + : APIIterator(std::move(iter)) + , seenKeys_(std::move(iter.seenKeys_)) + , deduplicate_(iter.deduplicate_) {} ListIterator& operator=(ListIterator&& iter) { seenKeys_ = std::move(iter.seenKeys_); @@ -112,7 +113,7 @@ class ListIterator : public APIIterator { bool next(ListElement& elem) { ListElement tmp; while (APIIterator::next(tmp)) { - if(deduplicate_) { + if (deduplicate_) { Key combinedKey = tmp.combinedKey(); if (seenKeys_.find(combinedKey) == seenKeys_.end()) { seenKeys_.emplace(std::move(combinedKey)); diff --git a/src/fdb5/api/helpers/MoveIterator.h b/src/fdb5/api/helpers/MoveIterator.h index 987f0b3bd..7691aa402 100644 --- a/src/fdb5/api/helpers/MoveIterator.h +++ b/src/fdb5/api/helpers/MoveIterator.h @@ -27,15 +27,17 @@ namespace fdb5 { - // class for writing a chunk of the user buffer - used to perform multiple simultaneous writes class FileCopy : public eckit::ThreadPoolTask { public: FileCopy() : src_(""), dest_(""), sync_(false) {} - FileCopy(const eckit::PathName& srcPath, const eckit::PathName& destPath, const std::string& fileName, bool sync=false) : - src_(srcPath / fileName), dest_(destPath / fileName), sync_(sync) {} + FileCopy(const eckit::PathName& srcPath, const eckit::PathName& destPath, const std::string& fileName, + bool sync = false) + : src_(srcPath / fileName) + , dest_(destPath / fileName) + , sync_(sync) {} FileCopy(eckit::Stream& s) { s >> src_; @@ -47,7 +49,7 @@ class FileCopy : public eckit::ThreadPoolTask { s << src_; s << dest_; s << sync_; - } + } bool sync() { return sync_; } @@ -66,10 +68,7 @@ class FileCopy : public eckit::ThreadPoolTask { } private: // methods - - void print(std::ostream& s) const { - s << "FileCopy(src=" << src_ << ",dest=" << dest_ << ",sync=" << sync_ << ")"; - } + void print(std::ostream& s) const { s << "FileCopy(src=" << src_ << ",dest=" << dest_ << ",sync=" << sync_ << ")"; } friend std::ostream& operator<<(std::ostream& s, const FileCopy& f) { f.print(s); diff --git a/src/fdb5/api/helpers/StatsIterator.cc b/src/fdb5/api/helpers/StatsIterator.cc index a3c70bc1c..595ab58ba 100644 --- a/src/fdb5/api/helpers/StatsIterator.cc +++ b/src/fdb5/api/helpers/StatsIterator.cc @@ -19,17 +19,15 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -StatsElement::StatsElement(const IndexStats& iStats, const DbStats& dbStats) : - indexStatistics(iStats), - dbStatistics(dbStats) {} +StatsElement::StatsElement(const IndexStats& iStats, const DbStats& dbStats) + : indexStatistics(iStats) + , dbStatistics(dbStats) {} +StatsElement::StatsElement(eckit::Stream& s) + : indexStatistics(eckit::Reanimator::reanimate(s)) + , dbStatistics(eckit::Reanimator::reanimate(s)) {} -StatsElement::StatsElement(eckit::Stream &s) : - indexStatistics(eckit::Reanimator::reanimate(s)), - dbStatistics(eckit::Reanimator::reanimate(s)) {} - - -void StatsElement::encode(eckit::Stream &s) const { +void StatsElement::encode(eckit::Stream& s) const { s << indexStatistics; s << dbStatistics; } @@ -37,4 +35,3 @@ void StatsElement::encode(eckit::Stream &s) const { //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 - diff --git a/src/fdb5/api/helpers/StatsIterator.h b/src/fdb5/api/helpers/StatsIterator.h index 0e0f7825e..b1fb490cb 100644 --- a/src/fdb5/api/helpers/StatsIterator.h +++ b/src/fdb5/api/helpers/StatsIterator.h @@ -41,7 +41,6 @@ struct StatsElement { DbStats dbStatistics; private: // methods - void encode(eckit::Stream& s) const; friend eckit::Stream& operator<<(eckit::Stream& s, const StatsElement& r) { diff --git a/src/fdb5/api/local/AxesVisitor.cc b/src/fdb5/api/local/AxesVisitor.cc index 4d67c00ec..404bf1a99 100644 --- a/src/fdb5/api/local/AxesVisitor.cc +++ b/src/fdb5/api/local/AxesVisitor.cc @@ -18,13 +18,11 @@ namespace local { //---------------------------------------------------------------------------------------------------------------------- -AxesVisitor::AxesVisitor(eckit::Queue& queue, - const metkit::mars::MarsRequest& request, - const Config& config, - int level) : - QueryVisitor(queue, request), - schema_(config.schema()), - level_(level) {} +AxesVisitor::AxesVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, + const Config& config, int level) + : QueryVisitor(queue, request) + , schema_(config.schema()) + , level_(level) {} #if 0 @@ -62,7 +60,7 @@ bool AxesVisitor::visitIndex(const Index& index) { IndexAxis tmpAxis; tmpAxis.insert(index.key()); tmpAxis.sort(); - axes_.merge(tmpAxis); // avoid sorts on the (growing) main Axes object + axes_.merge(tmpAxis); // avoid sorts on the (growing) main Axes object if (level_ > 2) { axes_.merge(index.axes()); diff --git a/src/fdb5/api/local/AxesVisitor.h b/src/fdb5/api/local/AxesVisitor.h index bbecb67d1..d3aaf850f 100644 --- a/src/fdb5/api/local/AxesVisitor.h +++ b/src/fdb5/api/local/AxesVisitor.h @@ -13,9 +13,8 @@ #pragma once -#include "fdb5/api/local/QueryVisitor.h" #include "fdb5/api/helpers/AxesIterator.h" - +#include "fdb5/api/local/QueryVisitor.h" namespace fdb5 { namespace api { @@ -27,10 +26,7 @@ namespace local { class AxesVisitor : public QueryVisitor { public: - - AxesVisitor(eckit::Queue& queue, - const metkit::mars::MarsRequest& request, - const Config& config, + AxesVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, const Config& config, int level); bool visitIndexes() override { return true; } @@ -41,7 +37,6 @@ class AxesVisitor : public QueryVisitor { void visitDatum(const Field&, const Key&) override { NOTIMP; } private: // members - Key dbKey_; IndexAxis axes_; const Schema& schema_; diff --git a/src/fdb5/api/local/ControlVisitor.cc b/src/fdb5/api/local/ControlVisitor.cc index 074feef3a..73fae6a5b 100644 --- a/src/fdb5/api/local/ControlVisitor.cc +++ b/src/fdb5/api/local/ControlVisitor.cc @@ -18,14 +18,11 @@ namespace local { //---------------------------------------------------------------------------------------------------------------------- -ControlVisitor::ControlVisitor(eckit::Queue& queue, - const metkit::mars::MarsRequest& request, - ControlAction action, - ControlIdentifiers identifiers) : - QueryVisitor(queue, request), - action_(action), - identifiers_(identifiers) {} - +ControlVisitor::ControlVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, + ControlAction action, ControlIdentifiers identifiers) + : QueryVisitor(queue, request) + , action_(action) + , identifiers_(identifiers) {} bool ControlVisitor::visitDatabase(const Catalogue& catalogue) { diff --git a/src/fdb5/api/local/ControlVisitor.h b/src/fdb5/api/local/ControlVisitor.h index 5a98fe001..f2cb2e8ba 100644 --- a/src/fdb5/api/local/ControlVisitor.h +++ b/src/fdb5/api/local/ControlVisitor.h @@ -14,9 +14,8 @@ #ifndef fdb5_api_local_ControlVisitor_H #define fdb5_api_local_ControlVisitor_H -#include "fdb5/api/local/QueryVisitor.h" #include "fdb5/api/helpers/ControlIterator.h" - +#include "fdb5/api/local/QueryVisitor.h" namespace fdb5 { namespace api { @@ -28,10 +27,7 @@ namespace local { class ControlVisitor : public QueryVisitor { public: - - ControlVisitor(eckit::Queue& queue, - const metkit::mars::MarsRequest& request, - ControlAction action, + ControlVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, ControlAction action, ControlIdentifiers identifiers); bool visitIndexes() override { return false; } @@ -42,7 +38,6 @@ class ControlVisitor : public QueryVisitor { void visitDatum(const Field&, const Key&) override { NOTIMP; } private: // members - ControlAction action_; ControlIdentifiers identifiers_; }; diff --git a/src/fdb5/api/local/DumpVisitor.h b/src/fdb5/api/local/DumpVisitor.h index 8fe4785be..741a1ec3d 100644 --- a/src/fdb5/api/local/DumpVisitor.h +++ b/src/fdb5/api/local/DumpVisitor.h @@ -19,9 +19,9 @@ #ifndef fdb5_api_local_DumpVisitor_H #define fdb5_api_local_DumpVisitor_H +#include "fdb5/api/helpers/DumpIterator.h" #include "fdb5/api/local/QueryVisitor.h" #include "fdb5/api/local/QueueStringLogTarget.h" -#include "fdb5/api/helpers/DumpIterator.h" #include "fdb5/database/Catalogue.h" namespace fdb5 { @@ -35,11 +35,10 @@ namespace local { class DumpVisitor : public QueryVisitor { public: - - DumpVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, bool simple) : - QueryVisitor(queue, request), - out_(new QueueStringLogTarget(queue)), - simple_(simple) {} + DumpVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, bool simple) + : QueryVisitor(queue, request) + , out_(new QueueStringLogTarget(queue)) + , simple_(simple) {} bool visitIndexes() override { return false; } bool visitEntries() override { return false; } diff --git a/src/fdb5/api/local/ListVisitor.h b/src/fdb5/api/local/ListVisitor.h index ac9c15ccf..faf4ab4dc 100644 --- a/src/fdb5/api/local/ListVisitor.h +++ b/src/fdb5/api/local/ListVisitor.h @@ -19,12 +19,12 @@ #ifndef fdb5_api_local_ListVisitor_H #define fdb5_api_local_ListVisitor_H +#include "fdb5/api/helpers/ListIterator.h" +#include "fdb5/api/local/QueryVisitor.h" #include "fdb5/database/Catalogue.h" #include "fdb5/database/Index.h" #include "fdb5/database/Key.h" #include "fdb5/rules/Rule.h" -#include "fdb5/api/local/QueryVisitor.h" -#include "fdb5/api/helpers/ListIterator.h" #include "metkit/mars/MarsRequest.h" @@ -70,7 +70,6 @@ struct ListVisitor : public QueryVisitor { bool visitIndex(const Index& index) override { QueryVisitor::visitIndex(index); - if (index.partialMatch(request_)) { // Subselect the parts of the request @@ -94,7 +93,8 @@ struct ListVisitor : public QueryVisitor { ASSERT(currentIndex_); if (datumKey.match(datumRequest_)) { - queue_.emplace(ListElement({currentCatalogue_->key(), currentIndex_->key(), datumKey}, field.stableLocation(), field.timestamp())); + queue_.emplace(ListElement({currentCatalogue_->key(), currentIndex_->key(), datumKey}, + field.stableLocation(), field.timestamp())); } } @@ -103,7 +103,6 @@ struct ListVisitor : public QueryVisitor { } private: // members - metkit::mars::MarsRequest indexRequest_; metkit::mars::MarsRequest datumRequest_; }; diff --git a/src/fdb5/api/local/MoveVisitor.cc b/src/fdb5/api/local/MoveVisitor.cc index bf83fba9c..45fc90282 100644 --- a/src/fdb5/api/local/MoveVisitor.cc +++ b/src/fdb5/api/local/MoveVisitor.cc @@ -16,36 +16,33 @@ #include "fdb5/api/local/MoveVisitor.h" +#include "fdb5/LibFdb5.h" #include "fdb5/api/local/QueueStringLogTarget.h" #include "fdb5/database/Catalogue.h" #include "fdb5/database/Index.h" -#include "fdb5/LibFdb5.h" #include "eckit/os/Stat.h" -#include #include +#include using namespace eckit; - namespace fdb5 { namespace api { namespace local { //---------------------------------------------------------------------------------------------------------------------- -MoveVisitor::MoveVisitor(eckit::Queue& queue, - const metkit::mars::MarsRequest& request, - const eckit::URI& dest) : - QueryVisitor(queue, request), - dest_(dest) {} +MoveVisitor::MoveVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, + const eckit::URI& dest) + : QueryVisitor(queue, request) + , dest_(dest) {} bool MoveVisitor::visitDatabase(const Catalogue& catalogue) { if (catalogue.key().match(request_)) { - catalogue.control( - ControlAction::Disable, - ControlIdentifier::Archive | ControlIdentifier::Wipe | ControlIdentifier::UniqueRoot); + catalogue.control(ControlAction::Disable, + ControlIdentifier::Archive | ControlIdentifier::Wipe | ControlIdentifier::UniqueRoot); // assert the source is locked for archival... ASSERT(!catalogue.enabled(ControlIdentifier::Archive)); diff --git a/src/fdb5/api/local/MoveVisitor.h b/src/fdb5/api/local/MoveVisitor.h index 2e43ac6aa..dc34f5195 100644 --- a/src/fdb5/api/local/MoveVisitor.h +++ b/src/fdb5/api/local/MoveVisitor.h @@ -16,13 +16,12 @@ #include "eckit/distributed/Transport.h" -#include "fdb5/api/local/QueryVisitor.h" #include "fdb5/api/helpers/MoveIterator.h" +#include "fdb5/api/local/QueryVisitor.h" #include "fdb5/database/MoveVisitor.h" #include "eckit/filesystem/PathName.h" - namespace fdb5 { namespace api { namespace local { @@ -34,10 +33,7 @@ namespace local { class MoveVisitor : public QueryVisitor { public: // methods - - MoveVisitor(eckit::Queue& queue, - const metkit::mars::MarsRequest& request, - const eckit::URI& dest); + MoveVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, const eckit::URI& dest); bool visitIndexes() override { return false; } bool visitEntries() override { return false; } @@ -48,12 +44,10 @@ class MoveVisitor : public QueryVisitor { void visitDatum(const Field& field, const std::string& keyFingerprint) override { NOTIMP; } private: // members - const eckit::URI& dest_; std::unique_ptr internalVisitor_; }; - //---------------------------------------------------------------------------------------------------------------------- } // namespace local diff --git a/src/fdb5/api/local/PurgeVisitor.cc b/src/fdb5/api/local/PurgeVisitor.cc index 9cd9ab522..5a14e7b10 100644 --- a/src/fdb5/api/local/PurgeVisitor.cc +++ b/src/fdb5/api/local/PurgeVisitor.cc @@ -18,10 +18,10 @@ #include "eckit/exception/Exceptions.h" +#include "fdb5/LibFdb5.h" #include "fdb5/api/local/QueueStringLogTarget.h" #include "fdb5/database/Catalogue.h" #include "fdb5/database/PurgeVisitor.h" -#include "fdb5/LibFdb5.h" namespace fdb5 { namespace api { @@ -29,15 +29,12 @@ namespace local { //---------------------------------------------------------------------------------------------------------------------- - -PurgeVisitor::PurgeVisitor(eckit::Queue& queue, - const metkit::mars::MarsRequest& request, - bool doit, - bool porcelain) : - QueryVisitor(queue, request), - out_(new QueueStringLogTarget(queue)), - doit_(doit), - porcelain_(porcelain) {} +PurgeVisitor::PurgeVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, bool doit, + bool porcelain) + : QueryVisitor(queue, request) + , out_(new QueueStringLogTarget(queue)) + , doit_(doit) + , porcelain_(porcelain) {} bool PurgeVisitor::visitDatabase(const Catalogue& catalogue) { @@ -45,7 +42,7 @@ bool PurgeVisitor::visitDatabase(const Catalogue& catalogue) { if (!catalogue.enabled(ControlIdentifier::Wipe)) { return false; } - + EntryVisitor::visitDatabase(catalogue); // If the request is overspecified relative to the DB key, then we @@ -54,8 +51,7 @@ bool PurgeVisitor::visitDatabase(const Catalogue& catalogue) { if (!catalogue.key().match(request_)) { std::stringstream ss; ss << "Purging not supported for over-specified requests. " - << "db=" << catalogue.key() - << ", request=" << request_; + << "db=" << catalogue.key() << ", request=" << request_; throw eckit::UserError(ss.str(), Here()); } diff --git a/src/fdb5/api/local/PurgeVisitor.h b/src/fdb5/api/local/PurgeVisitor.h index 985367b25..2e68289dc 100644 --- a/src/fdb5/api/local/PurgeVisitor.h +++ b/src/fdb5/api/local/PurgeVisitor.h @@ -19,14 +19,13 @@ #ifndef fdb5_api_local_PurgeVisitor_H #define fdb5_api_local_PurgeVisitor_H -#include "fdb5/api/local/QueryVisitor.h" #include "fdb5/api/helpers/PurgeIterator.h" +#include "fdb5/api/local/QueryVisitor.h" #include "fdb5/database/PurgeVisitor.h" #include "fdb5/database/Store.h" #include "eckit/filesystem/PathName.h" - namespace fdb5 { namespace api { namespace local { @@ -37,10 +36,7 @@ namespace local { class PurgeVisitor : public QueryVisitor { public: - - PurgeVisitor(eckit::Queue& queue, - const metkit::mars::MarsRequest& request, - bool doit, + PurgeVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, bool doit, bool porcelain); bool visitDatabase(const Catalogue& catalogue) override; @@ -50,7 +46,6 @@ class PurgeVisitor : public QueryVisitor { void visitDatum(const Field&, const Key&) override { NOTIMP; } private: // members - eckit::Channel out_; bool doit_; bool porcelain_; diff --git a/src/fdb5/api/local/QueryVisitor.h b/src/fdb5/api/local/QueryVisitor.h index 7c99054f3..5eb004c7b 100644 --- a/src/fdb5/api/local/QueryVisitor.h +++ b/src/fdb5/api/local/QueryVisitor.h @@ -37,19 +37,17 @@ template class QueryVisitor : public EntryVisitor { public: // methods - using ValueType = T; - QueryVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request) : - queue_(queue), request_(request) {} + QueryVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request) + : queue_(queue) + , request_(request) {} protected: // members - eckit::Queue& queue_; metkit::mars::MarsRequest request_; }; - //---------------------------------------------------------------------------------------------------------------------- } // namespace local diff --git a/src/fdb5/api/local/QueueStringLogTarget.h b/src/fdb5/api/local/QueueStringLogTarget.h index 61317ab45..b8203754b 100644 --- a/src/fdb5/api/local/QueueStringLogTarget.h +++ b/src/fdb5/api/local/QueueStringLogTarget.h @@ -35,12 +35,9 @@ namespace local { class QueueStringLogTarget : public eckit::LineBasedTarget { public: - QueueStringLogTarget(eckit::Queue& queue) : queue_(queue) {} - void line(const char* line) override { - queue_.emplace(std::string(line)); - } + void line(const char* line) override { queue_.emplace(std::string(line)); } private: // members eckit::Queue& queue_; diff --git a/src/fdb5/api/local/StatsVisitor.cc b/src/fdb5/api/local/StatsVisitor.cc index 25f669103..954a8348e 100644 --- a/src/fdb5/api/local/StatsVisitor.cc +++ b/src/fdb5/api/local/StatsVisitor.cc @@ -52,7 +52,7 @@ void StatsVisitor::catalogueComplete(const Catalogue& catalogue) { // Construct the object to push onto the queue - queue_.emplace(StatsElement { internalVisitor_->indexStatistics(), internalVisitor_->dbStatistics() }); + queue_.emplace(StatsElement{internalVisitor_->indexStatistics(), internalVisitor_->dbStatistics()}); // Cleanup diff --git a/src/fdb5/api/local/StatsVisitor.h b/src/fdb5/api/local/StatsVisitor.h index e62dbee20..c7b954988 100644 --- a/src/fdb5/api/local/StatsVisitor.h +++ b/src/fdb5/api/local/StatsVisitor.h @@ -19,12 +19,11 @@ #ifndef fdb5_api_local_StatsVisitor_H #define fdb5_api_local_StatsVisitor_H -#include "fdb5/api/local/QueryVisitor.h" #include "fdb5/api/helpers/StatsIterator.h" +#include "fdb5/api/local/QueryVisitor.h" #include "fdb5/database/StatsReportVisitor.h" #include "fdb5/database/Store.h" - namespace fdb5 { namespace api { namespace local { @@ -35,17 +34,15 @@ namespace local { class StatsVisitor : public QueryVisitor { public: - using QueryVisitor::QueryVisitor; bool visitDatabase(const Catalogue& catalogue) override; bool visitIndex(const Index& index) override; void catalogueComplete(const Catalogue& catalogue) override; void visitDatum(const Field& field, const std::string& keyFingerprint) override; - void visitDatum(const Field&, const Key&) override { NOTIMP; } + void visitDatum(const Field&, const Key&) override { NOTIMP; } private: // members - std::unique_ptr internalVisitor_; }; diff --git a/src/fdb5/api/local/StatusVisitor.h b/src/fdb5/api/local/StatusVisitor.h index 87dd3dbfa..0bf4e3f99 100644 --- a/src/fdb5/api/local/StatusVisitor.h +++ b/src/fdb5/api/local/StatusVisitor.h @@ -19,9 +19,8 @@ #ifndef fdb5_api_local_StatusVisitor_H #define fdb5_api_local_StatusVisitor_H -#include "fdb5/api/local/QueryVisitor.h" #include "fdb5/api/helpers/StatusIterator.h" - +#include "fdb5/api/local/QueryVisitor.h" namespace fdb5 { namespace api { @@ -36,7 +35,10 @@ class StatusVisitor : public QueryVisitor { using QueryVisitor::QueryVisitor; bool visitIndexes() override { return false; } bool visitEntries() override { return false; } - bool visitDatabase(const Catalogue& catalogue) override { queue_.emplace(catalogue); return true; } + bool visitDatabase(const Catalogue& catalogue) override { + queue_.emplace(catalogue); + return true; + } bool visitIndex(const Index&) override { NOTIMP; } void visitDatum(const Field&, const Key&) override { NOTIMP; } diff --git a/src/fdb5/api/local/WipeVisitor.cc b/src/fdb5/api/local/WipeVisitor.cc index 33170ef25..a404d1e38 100644 --- a/src/fdb5/api/local/WipeVisitor.cc +++ b/src/fdb5/api/local/WipeVisitor.cc @@ -16,36 +16,31 @@ #include "fdb5/api/local/WipeVisitor.h" +#include "fdb5/LibFdb5.h" #include "fdb5/api/local/QueueStringLogTarget.h" #include "fdb5/database/Catalogue.h" #include "fdb5/database/Index.h" -#include "fdb5/LibFdb5.h" #include "eckit/os/Stat.h" -#include #include +#include using namespace eckit; - namespace fdb5 { namespace api { namespace local { //---------------------------------------------------------------------------------------------------------------------- -WipeVisitor::WipeVisitor(eckit::Queue& queue, - const metkit::mars::MarsRequest& request, - bool doit, - bool porcelain, - bool unsafeWipeAll) : - QueryVisitor(queue, request), - out_(new QueueStringLogTarget(queue)), - doit_(doit), - porcelain_(porcelain), - unsafeWipeAll_(unsafeWipeAll) {} - +WipeVisitor::WipeVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, bool doit, + bool porcelain, bool unsafeWipeAll) + : QueryVisitor(queue, request) + , out_(new QueueStringLogTarget(queue)) + , doit_(doit) + , porcelain_(porcelain) + , unsafeWipeAll_(unsafeWipeAll) {} bool WipeVisitor::visitDatabase(const Catalogue& catalogue) { diff --git a/src/fdb5/api/local/WipeVisitor.h b/src/fdb5/api/local/WipeVisitor.h index 7ef6f83c4..83c21afde 100644 --- a/src/fdb5/api/local/WipeVisitor.h +++ b/src/fdb5/api/local/WipeVisitor.h @@ -19,13 +19,12 @@ #ifndef fdb5_api_local_WipeVisitor_H #define fdb5_api_local_WipeVisitor_H -#include "fdb5/api/local/QueryVisitor.h" #include "fdb5/api/helpers/WipeIterator.h" +#include "fdb5/api/local/QueryVisitor.h" #include "fdb5/database/WipeVisitor.h" #include "eckit/filesystem/PathName.h" - namespace fdb5 { namespace api { namespace local { @@ -37,11 +36,7 @@ namespace local { class WipeVisitor : public QueryVisitor { public: // methods - - WipeVisitor(eckit::Queue& queue, - const metkit::mars::MarsRequest& request, - bool doit, - bool porcelain, + WipeVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, bool doit, bool porcelain, bool unsafeWipeAll); bool visitEntries() override { return false; } @@ -55,7 +50,6 @@ class WipeVisitor : public QueryVisitor { void onDatabaseNotFound(const fdb5::DatabaseNotFoundException& e) override { throw e; } private: // members - eckit::Channel out_; bool doit_; bool porcelain_; @@ -64,7 +58,6 @@ class WipeVisitor : public QueryVisitor { std::unique_ptr internalVisitor_; }; - //---------------------------------------------------------------------------------------------------------------------- } // namespace local diff --git a/src/fdb5/compat/grib2fdb5.cc b/src/fdb5/compat/grib2fdb5.cc index b1274bd44..690a08c45 100644 --- a/src/fdb5/compat/grib2fdb5.cc +++ b/src/fdb5/compat/grib2fdb5.cc @@ -11,8 +11,8 @@ #include #include "eckit/io/DataHandle.h" -#include "eckit/option/CmdArgs.h" #include "eckit/log/Log.h" +#include "eckit/option/CmdArgs.h" #include "fdb5/message/MessageArchiver.h" #include "fdb5/tools/FDBTool.h" @@ -20,23 +20,21 @@ using namespace eckit; class Grib2Fdb5 : public fdb5::FDBTool { - virtual void execute(const eckit::option::CmdArgs &args); - virtual void usage(const std::string &tool) const; - virtual int minimumPositionalArguments() const { - return 1; - } + virtual void execute(const eckit::option::CmdArgs& args); + virtual void usage(const std::string& tool) const; + virtual int minimumPositionalArguments() const { return 1; } - public: - Grib2Fdb5(int argc, char **argv): fdb5::FDBTool(argc, argv) {} +public: + Grib2Fdb5(int argc, char** argv) : fdb5::FDBTool(argc, argv) {} }; -void Grib2Fdb5::usage(const std::string &tool) const { +void Grib2Fdb5::usage(const std::string& tool) const { eckit::Log::info() << std::endl << "Usage: " << tool << " [-c class] [-e expver] [-T type] [-s stream] [-f file]" << std::endl; fdb5::FDBTool::usage(tool); } -void Grib2Fdb5::execute(const eckit::option::CmdArgs &args) { +void Grib2Fdb5::execute(const eckit::option::CmdArgs& args) { fdb5::Key check; @@ -60,7 +58,7 @@ void Grib2Fdb5::execute(const eckit::option::CmdArgs &args) { } if (k == "-1") { - Log::info() << "Option -1 ignored" << std::endl; + Log::info() << "Option -1 ignored" << std::endl; i++; continue; } @@ -100,15 +98,12 @@ void Grib2Fdb5::execute(const eckit::option::CmdArgs &args) { for (const auto& path : paths) { Log::info() << "Processing " << path << std::endl; Log::info() << "Key " << check << std::endl; - std::unique_ptr dh ( path.fileHandle() ); - archiver.archive( *dh ); + std::unique_ptr dh(path.fileHandle()); + archiver.archive(*dh); } - } - -int main(int argc, char **argv) { +int main(int argc, char** argv) { Grib2Fdb5 app(argc, argv); return app.start(); } - diff --git a/src/fdb5/config/Config.cc b/src/fdb5/config/Config.cc index 9644db66a..4b111e8f7 100644 --- a/src/fdb5/config/Config.cc +++ b/src/fdb5/config/Config.cc @@ -24,7 +24,6 @@ using namespace eckit; - namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- @@ -42,8 +41,9 @@ Config Config::make(const eckit::PathName& path, const eckit::Configuration& use return cfg; } -Config::Config(const Configuration& config, const eckit::Configuration& userConfig) : - LocalConfiguration(config), schemaPathInitialised_(false) { +Config::Config(const Configuration& config, const eckit::Configuration& userConfig) + : LocalConfiguration(config) + , schemaPathInitialised_(false) { userConfig_ = std::make_shared(userConfig); } @@ -109,10 +109,8 @@ Config Config::expandConfig() const { return *this; } - Config::~Config() {} - // TODO: We could add this to expandTilde. PathName Config::expandPath(const std::string& path) const { @@ -153,7 +151,7 @@ void Config::overrideSchema(const eckit::PathName& schemaPath, Schema* schema) { schema->path_ = schemaPath; SchemaRegistry::instance().add(schemaPath, schema); - schemaPath_=schemaPath; + schemaPath_ = schemaPath; schemaPathInitialised_ = true; } @@ -167,8 +165,7 @@ void Config::initializeSchemaPath() const { if (has("schema")) { schemaPath_ = expandPath(getString("schema")); - } - else { + } else { // TODO: deduplicate this with the library-level schemaPath() // N.B. this uses Config expandPath() static std::string fdbSchemaFile = @@ -194,8 +191,7 @@ mode_t Config::umask() const { if (has("permissions")) { return FileMode(getString("permissions")).mask(); } - static eckit::FileMode fdbFileMode( - eckit::Resource("fdbFileMode", std::string("0644"))); + static eckit::FileMode fdbFileMode(eckit::Resource("fdbFileMode", std::string("0644"))); return fdbFileMode.mask(); } @@ -221,7 +217,6 @@ std::vector Config::getSubConfigs() const { return out; } - //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/config/Config.h b/src/fdb5/config/Config.h index f760f1953..cfb1a4ee9 100644 --- a/src/fdb5/config/Config.h +++ b/src/fdb5/config/Config.h @@ -15,14 +15,13 @@ #ifndef fdb5_config_Config_H #define fdb5_config_Config_H -#include // for mode_t +#include // for mode_t #include #include "eckit/config/LocalConfiguration.h" #include "eckit/filesystem/PathName.h" - namespace fdb5 { class Schema; @@ -30,10 +29,11 @@ class Schema; //---------------------------------------------------------------------------------------------------------------------- class Config : public eckit::LocalConfiguration { -public: // static methods - static Config make(const eckit::PathName& path, const eckit::Configuration& userConfig = eckit::LocalConfiguration()); +public: // static methods + static Config make(const eckit::PathName& path, + const eckit::Configuration& userConfig = eckit::LocalConfiguration()); -public: // methods +public: // methods Config(); Config(const eckit::Configuration& config, const eckit::Configuration& userConfig = eckit::LocalConfiguration()); @@ -47,7 +47,6 @@ class Config : public eckit::LocalConfiguration { /// then do the expansion in here. eckit::PathName expandPath(const std::string& path) const; - void overrideSchema(const eckit::PathName& schemaPath, Schema* schema); const eckit::PathName& schemaPath() const; eckit::PathName configPath() const; @@ -61,10 +60,10 @@ class Config : public eckit::LocalConfiguration { std::vector getSubConfigs(const std::string& name) const; std::vector getSubConfigs() const; -private: // methods +private: // methods void initializeSchemaPath() const; -private: // members +private: // members mutable eckit::PathName schemaPath_; mutable bool schemaPathInitialised_; std::shared_ptr userConfig_; @@ -72,6 +71,6 @@ class Config : public eckit::LocalConfiguration { //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 -#endif // fdb5_config_Config_H +#endif // fdb5_config_Config_H diff --git a/src/fdb5/daos/DaosArrayHandle.cc b/src/fdb5/daos/DaosArrayHandle.cc index 5cde01e8a..cf1717ffe 100644 --- a/src/fdb5/daos/DaosArrayHandle.cc +++ b/src/fdb5/daos/DaosArrayHandle.cc @@ -13,11 +13,11 @@ #include "eckit/exception/Exceptions.h" #include "fdb5/daos/DaosArrayHandle.h" -#include "fdb5/daos/DaosPool.h" #include "fdb5/daos/DaosContainer.h" +#include "fdb5/daos/DaosException.h" #include "fdb5/daos/DaosObject.h" +#include "fdb5/daos/DaosPool.h" #include "fdb5/daos/DaosSession.h" -#include "fdb5/daos/DaosException.h" using eckit::Length; using eckit::Offset; @@ -28,8 +28,8 @@ DaosArrayHandle::DaosArrayHandle(const fdb5::DaosArrayName& name) : name_(name), DaosArrayHandle::~DaosArrayHandle() { - if (open_) eckit::Log::error() << "DaosArrayHandle not closed before destruction." << std::endl; - + if (open_) + eckit::Log::error() << "DaosArrayHandle not closed before destruction." << std::endl; } void DaosArrayHandle::print(std::ostream& s) const { @@ -38,7 +38,8 @@ void DaosArrayHandle::print(std::ostream& s) const { void DaosArrayHandle::openForWrite(const Length& len) { - if (open_) throw eckit::SeriousBug{"Handle already opened."}; + if (open_) + throw eckit::SeriousBug{"Handle already opened."}; session(); @@ -50,17 +51,15 @@ void DaosArrayHandle::openForWrite(const Length& len) { /// - attempt array create and check if rc is 0 or DER_EXIST (c.createArray(oid)) /// we do the latter first because it is the most likely to succeed. /// If the operation fails, the DAOS client will generate an ERR log and usually persist - /// it to a log file (potential performance impact). So we want to have as few of these + /// it to a log file (potential performance impact). So we want to have as few of these /// failures as possible. /// @todo: implement DaosContainer::ensureArray, which attempts createArray with a catch+dismiss? - /// @todo: have dummy daos_create_array return DER_EXIST where relevant. - /// This would probably require breaking transactionality of dummy + /// @todo: have dummy daos_create_array return DER_EXIST where relevant. + /// This would probably require breaking transactionality of dummy /// daos_create_array, and/or hit its performance. try { - arr_.emplace( c.createArray(name_.OID()) ); - } catch (fdb5::DaosEntityAlreadyExistsException& e) { - arr_.emplace( session_.value(), name_ ); - } + arr_.emplace(c.createArray(name_.OID())); + } catch (fdb5::DaosEntityAlreadyExistsException& e) { arr_.emplace(session_.value(), name_); } arr_->open(); @@ -69,14 +68,14 @@ void DaosArrayHandle::openForWrite(const Length& len) { open_ = true; offset_ = eckit::Offset(0); - } -/// @note: the array size is retrieved here and ::read. For a more optimised reading +/// @note: the array size is retrieved here and ::read. For a more optimised reading /// if the size is known in advance, see DaosArrayPartHandle. Length DaosArrayHandle::openForRead() { - if (open_) throw eckit::SeriousBug{"Handle already opened."}; + if (open_) + throw eckit::SeriousBug{"Handle already opened."}; session(); @@ -87,7 +86,6 @@ Length DaosArrayHandle::openForRead() { open_ = true; return size(); - } long DaosArrayHandle::write(const void* buf, long len) { @@ -99,7 +97,6 @@ long DaosArrayHandle::write(const void* buf, long len) { offset_ += written; return written; - } long DaosArrayHandle::read(void* buf, long len) { @@ -111,47 +108,43 @@ long DaosArrayHandle::read(void* buf, long len) { /// @note: if the buffer is oversized, daos does not return the actual smaller size read, /// so it is calculated here and returned to the user as expected eckit::Length s = size(); - if (len > s - offset_) read = s - offset_; + if (len > s - offset_) + read = s - offset_; offset_ += read; return read; - } void DaosArrayHandle::close() { - if (!open_) return; + if (!open_) + return; arr_->close(); open_ = false; - } void DaosArrayHandle::flush() { /// empty implmenetation - } Length DaosArrayHandle::size() { return Length(name_.size()); - } Length DaosArrayHandle::estimate() { return size(); - } Offset DaosArrayHandle::position() { /// @todo: should position() crash if unopened? return offset_; - } Offset DaosArrayHandle::seek(const Offset& offset) { @@ -159,26 +152,23 @@ Offset DaosArrayHandle::seek(const Offset& offset) { offset_ = offset; /// @todo: assert offset <= size() ? return offset_; - } bool DaosArrayHandle::canSeek() const { return true; - } std::string DaosArrayHandle::title() const { - - return name_.asString(); + return name_.asString(); } fdb5::DaosSession& DaosArrayHandle::session() { - if (!session_.has_value()) session_.emplace(); + if (!session_.has_value()) + session_.emplace(); return session_.value(); - } -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosArrayHandle.h b/src/fdb5/daos/DaosArrayHandle.h index b7da5d37d..45c83581e 100644 --- a/src/fdb5/daos/DaosArrayHandle.h +++ b/src/fdb5/daos/DaosArrayHandle.h @@ -13,11 +13,11 @@ #pragma once -#include #include "eckit/io/DataHandle.h" +#include -#include "fdb5/daos/DaosSession.h" #include "fdb5/daos/DaosName.h" +#include "fdb5/daos/DaosSession.h" namespace fdb5 { @@ -30,7 +30,6 @@ class DaosArrayName; class DaosArrayHandle : public eckit::DataHandle { public: // methods - DaosArrayHandle(const fdb5::DaosArrayName&); ~DaosArrayHandle(); @@ -65,11 +64,9 @@ class DaosArrayHandle : public eckit::DataHandle { // static const ClassSpec& classSpec() { return classSpec_; } private: // methods - fdb5::DaosSession& session(); private: // members - // mutable because title() calls DaosArrayName::asString which may update (generate) OID mutable fdb5::DaosArrayName name_; std::optional session_; @@ -79,9 +76,8 @@ class DaosArrayHandle : public eckit::DataHandle { // static ClassSpec classSpec_; // static Reanimator reanimator_; - }; //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 \ No newline at end of file +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/daos/DaosArrayPartHandle.cc b/src/fdb5/daos/DaosArrayPartHandle.cc index 5c5ba83a9..620337506 100644 --- a/src/fdb5/daos/DaosArrayPartHandle.cc +++ b/src/fdb5/daos/DaosArrayPartHandle.cc @@ -13,25 +13,28 @@ #include "eckit/exception/Exceptions.h" #include "fdb5/daos/DaosArrayPartHandle.h" -#include "fdb5/daos/DaosPool.h" #include "fdb5/daos/DaosContainer.h" +#include "fdb5/daos/DaosException.h" #include "fdb5/daos/DaosObject.h" +#include "fdb5/daos/DaosPool.h" #include "fdb5/daos/DaosSession.h" -#include "fdb5/daos/DaosException.h" using eckit::Length; using eckit::Offset; namespace fdb5 { -DaosArrayPartHandle::DaosArrayPartHandle(const fdb5::DaosArrayName& name, - const eckit::Offset& off, - const eckit::Length& len) : name_(name), open_(false), offset_(off), len_(len) {} +DaosArrayPartHandle::DaosArrayPartHandle(const fdb5::DaosArrayName& name, const eckit::Offset& off, + const eckit::Length& len) + : name_(name) + , open_(false) + , offset_(off) + , len_(len) {} DaosArrayPartHandle::~DaosArrayPartHandle() { - if (open_) eckit::Log::error() << "DaosArrayPartHandle not closed before destruction." << std::endl; - + if (open_) + eckit::Log::error() << "DaosArrayPartHandle not closed before destruction." << std::endl; } void DaosArrayPartHandle::print(std::ostream& s) const { @@ -40,7 +43,8 @@ void DaosArrayPartHandle::print(std::ostream& s) const { Length DaosArrayPartHandle::openForRead() { - if (open_) throw eckit::SeriousBug{"Handle already opened."}; + if (open_) + throw eckit::SeriousBug{"Handle already opened."}; session(); @@ -51,7 +55,6 @@ Length DaosArrayPartHandle::openForRead() { open_ = true; return size(); - } long DaosArrayPartHandle::read(void* buf, long len) { @@ -61,49 +64,45 @@ long DaosArrayPartHandle::read(void* buf, long len) { /// @note: if the buffer is oversized, daos does not return the actual smaller size read, /// so it is calculated here and returned to the user as expected eckit::Length s = size(); - if (len > s - offset_) len = s - offset_; + if (len > s - offset_) + len = s - offset_; long read = arr_->read(buf, len, offset_); offset_ += read; return read; - } void DaosArrayPartHandle::close() { - if (!open_) return; + if (!open_) + return; arr_->close(); open_ = false; - } void DaosArrayPartHandle::flush() { /// empty implmenetation - } Length DaosArrayPartHandle::size() { return len_; - } Length DaosArrayPartHandle::estimate() { return size(); - } Offset DaosArrayPartHandle::position() { /// @todo: should position() crash if unopened? return offset_; - } Offset DaosArrayPartHandle::seek(const Offset& offset) { @@ -111,13 +110,11 @@ Offset DaosArrayPartHandle::seek(const Offset& offset) { offset_ = offset; /// @todo: assert offset <= size() ? return offset_; - } bool DaosArrayPartHandle::canSeek() const { return true; - } // void DaosArrayHandle::skip(const Length& len) { @@ -127,16 +124,15 @@ bool DaosArrayPartHandle::canSeek() const { // } std::string DaosArrayPartHandle::title() const { - - return name_.asString(); + return name_.asString(); } fdb5::DaosSession& DaosArrayPartHandle::session() { - if (!session_.has_value()) session_.emplace(); + if (!session_.has_value()) + session_.emplace(); return session_.value(); - } -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosArrayPartHandle.h b/src/fdb5/daos/DaosArrayPartHandle.h index e57233d6c..5545f2227 100644 --- a/src/fdb5/daos/DaosArrayPartHandle.h +++ b/src/fdb5/daos/DaosArrayPartHandle.h @@ -17,8 +17,8 @@ #include "eckit/io/DataHandle.h" -#include "fdb5/daos/DaosSession.h" #include "fdb5/daos/DaosName.h" +#include "fdb5/daos/DaosSession.h" namespace fdb5 { @@ -28,15 +28,14 @@ class DaosArray; class DaosArrayName; -/// @note: because daos_array_read does not report actual amount of bytes read, -/// DaosArrayHandle::read must check the actual size and returns it if smaller -/// than the provided buffer, which reduces performance. DaosArrayPartHandle +/// @note: because daos_array_read does not report actual amount of bytes read, +/// DaosArrayHandle::read must check the actual size and returns it if smaller +/// than the provided buffer, which reduces performance. DaosArrayPartHandle /// circumvents that check for cases where the object size is known. -/// @note: see DaosArray::read +/// @note: see DaosArray::read class DaosArrayPartHandle : public eckit::DataHandle { public: // methods - DaosArrayPartHandle(const fdb5::DaosArrayName&, const eckit::Offset&, const eckit::Length&); ~DaosArrayPartHandle(); @@ -71,11 +70,9 @@ class DaosArrayPartHandle : public eckit::DataHandle { // static const ClassSpec& classSpec() { return classSpec_; } private: // methods - fdb5::DaosSession& session(); private: // members - // mutable because title() calls DaosArrayName::asString which may update (generate) OID mutable fdb5::DaosArrayName name_; std::optional session_; @@ -86,9 +83,8 @@ class DaosArrayPartHandle : public eckit::DataHandle { // static ClassSpec classSpec_; // static Reanimator reanimator_; - }; //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 \ No newline at end of file +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/daos/DaosCatalogue.cc b/src/fdb5/daos/DaosCatalogue.cc index 588479349..9b47648ab 100644 --- a/src/fdb5/daos/DaosCatalogue.cc +++ b/src/fdb5/daos/DaosCatalogue.cc @@ -13,14 +13,14 @@ #include "eckit/config/Resource.h" #include "eckit/serialisation/MemoryStream.h" -#include "fdb5/api/helpers/ControlIterator.h" #include "fdb5/LibFdb5.h" +#include "fdb5/api/helpers/ControlIterator.h" #include "fdb5/database/DatabaseNotFoundException.h" #include "fdb5/daos/DaosCatalogue.h" +#include "fdb5/daos/DaosIndex.h" #include "fdb5/daos/DaosName.h" #include "fdb5/daos/DaosSession.h" -#include "fdb5/daos/DaosIndex.h" #include "fdb5/daos/DaosWipeVisitor.h" // using namespace eckit; @@ -29,18 +29,20 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -DaosCatalogue::DaosCatalogue(const Key& key, const fdb5::Config& config) : - CatalogueImpl(key, ControlIdentifiers{}, config), DaosCommon(config, "catalogue", key) { +DaosCatalogue::DaosCatalogue(const Key& key, const fdb5::Config& config) + : CatalogueImpl(key, ControlIdentifiers{}, config) + , DaosCommon(config, "catalogue", key) { // TODO: apply the mechanism in RootManager::directory, using // FileSpaceTables to determine root_pool_name_ according to key // and using DbPathNamerTables to determine db_cont_name_ according // to key - } -DaosCatalogue::DaosCatalogue(const eckit::URI& uri, const ControlIdentifiers& controlIdentifiers, const fdb5::Config& config) : - CatalogueImpl(Key(), controlIdentifiers, config), DaosCommon(config, "catalogue", uri) { +DaosCatalogue::DaosCatalogue(const eckit::URI& uri, const ControlIdentifiers& controlIdentifiers, + const fdb5::Config& config) + : CatalogueImpl(Key(), controlIdentifiers, config) + , DaosCommon(config, "catalogue", uri) { // Read the real DB key into the DB base object try { @@ -55,32 +57,25 @@ DaosCatalogue::DaosCatalogue(const eckit::URI& uri, const ControlIdentifiers& co } catch (fdb5::DaosEntityNotFoundException& e) { - throw fdb5::DatabaseNotFoundException( - std::string("DaosCatalogue database not found ") + - "(pool: '" + pool_ + "', container: '" + db_cont_ + "')" - ); - + throw fdb5::DatabaseNotFoundException(std::string("DaosCatalogue database not found ") + "(pool: '" + pool_ + + "', container: '" + db_cont_ + "')"); } - } bool DaosCatalogue::exists() const { fdb5::DaosKeyValueName catalogue_kv_name{pool_, db_cont_, catalogue_kv_}; return catalogue_kv_name.exists(); - } eckit::URI DaosCatalogue::uri() const { return fdb5::DaosName{db_kv_->poolName(), db_kv_->containerName()}.URI(); - } const Schema& DaosCatalogue::schema() const { return schema_; - } void DaosCatalogue::loadSchema() { @@ -101,10 +96,10 @@ void DaosCatalogue::loadSchema() { std::istringstream stream{std::string(v.begin(), v.end())}; schema_.load(stream); - } -WipeVisitor* DaosCatalogue::wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, bool doit, bool porcelain, bool unsafeWipeAll) const { +WipeVisitor* DaosCatalogue::wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, + bool doit, bool porcelain, bool unsafeWipeAll) const { return new DaosWipeVisitor(*this, store, request, out, doit, porcelain, unsafeWipeAll); } @@ -118,7 +113,7 @@ std::vector DaosCatalogue::indexes(bool) const { /// @note: performed RPCs: /// - db kv open (daos_kv_open) /// - db kv list keys (daos_kv_list) - fdb5::DaosKeyValue catalogue_kv{s, catalogue_kv_name}; /// @note: throws if not exists + fdb5::DaosKeyValue catalogue_kv{s, catalogue_kv_name}; /// @note: throws if not exists std::vector res; @@ -126,7 +121,8 @@ std::vector DaosCatalogue::indexes(bool) const { /// @todo: document these well. Single source these reserved values. /// Ensure where appropriate that user-provided keys do not collide. - if (key == "schema" || key == "key") continue; + if (key == "schema" || key == "key") + continue; /// @note: performed RPCs: /// - db kv get index location size (daos_kv_get without a buffer) @@ -141,10 +137,10 @@ std::vector DaosCatalogue::indexes(bool) const { /// - index kv open (daos_kv_open) /// - index kv get size (daos_kv_get without a buffer) /// - index kv get key (daos_kv_get) - /// @note: the following three lines intend to check whether the index kv exists + /// @note: the following three lines intend to check whether the index kv exists /// or not. The DaosKeyValue constructor calls kv open, which always succeeds, /// so it is not useful on its own to check whether the index KV existed or not. - /// Instead, presence of a "key" key in the KV is used to determine if the index + /// Instead, presence of a "key" key in the KV is used to determine if the index /// KV existed. fdb5::DaosKeyValue index_kv{s, index_kv_name}; std::optional index_key; @@ -158,11 +154,9 @@ std::vector DaosCatalogue::indexes(bool) const { } res.push_back(Index(new fdb5::DaosIndex(index_key.value(), *this, index_kv_name, false))); - } return res; - } std::string DaosCatalogue::type() const { @@ -176,8 +170,8 @@ void DaosCatalogue::remove(const fdb5::DaosNameBase& n, std::ostream& logAlways, logVerbose << "Removing " << (n.hasOID() ? "KV" : "container") << ": "; logAlways << n.URI() << std::endl; - if (doit) n.destroy(); - + if (doit) + n.destroy(); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/daos/DaosCatalogue.h b/src/fdb5/daos/DaosCatalogue.h index cc5faec4a..f7906ef52 100644 --- a/src/fdb5/daos/DaosCatalogue.h +++ b/src/fdb5/daos/DaosCatalogue.h @@ -13,11 +13,11 @@ #pragma once -#include "fdb5/database/Catalogue.h" -#include "fdb5/rules/Schema.h" #include "fdb5/daos/DaosCommon.h" #include "fdb5/daos/DaosEngine.h" #include "fdb5/daos/DaosOID.h" +#include "fdb5/database/Catalogue.h" +#include "fdb5/rules/Schema.h" namespace fdb5 { @@ -28,7 +28,6 @@ namespace fdb5 { class DaosCatalogue : public CatalogueImpl, public DaosCommon { public: // methods - DaosCatalogue(const Key& key, const fdb5::Config& config); DaosCatalogue(const eckit::URI& uri, const ControlIdentifiers& controlIdentifiers, const fdb5::Config& config); @@ -47,28 +46,31 @@ class DaosCatalogue : public CatalogueImpl, public DaosCommon { StatsReportVisitor* statsReportVisitor() const override { NOTIMP; }; PurgeVisitor* purgeVisitor(const Store& store) const override { NOTIMP; }; - WipeVisitor* wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, bool doit, bool porcelain, bool unsafeWipeAll) const override; - MoveVisitor* moveVisitor(const Store& store, const metkit::mars::MarsRequest& request, const eckit::URI& dest, eckit::Queue& queue) const override { NOTIMP; }; + WipeVisitor* wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, bool doit, + bool porcelain, bool unsafeWipeAll) const override; + MoveVisitor* moveVisitor(const Store& store, const metkit::mars::MarsRequest& request, const eckit::URI& dest, + eckit::Queue& queue) const override { + NOTIMP; + }; void maskIndexEntry(const Index& index) const override { NOTIMP; }; void loadSchema() override; - std::vector indexes(bool sorted=false) const override; + std::vector indexes(bool sorted = false) const override; void allMasked(std::set>& metadata, - std::set& data) const override { NOTIMP; }; + std::set& data) const override { + NOTIMP; + }; // Control access properties of the DB void control(const ControlAction& action, const ControlIdentifiers& identifiers) const override { NOTIMP; }; protected: // members - Key currentIndexKey_; private: // members - Schema schema_; - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/daos/DaosCatalogueReader.cc b/src/fdb5/daos/DaosCatalogueReader.cc index 1f81bede7..a50e6b004 100644 --- a/src/fdb5/daos/DaosCatalogueReader.cc +++ b/src/fdb5/daos/DaosCatalogueReader.cc @@ -10,11 +10,11 @@ #include "fdb5/LibFdb5.h" -#include "fdb5/daos/DaosSession.h" #include "fdb5/daos/DaosName.h" +#include "fdb5/daos/DaosSession.h" -#include "fdb5/daos/DaosIndex.h" #include "fdb5/daos/DaosCatalogueReader.h" +#include "fdb5/daos/DaosIndex.h" namespace fdb5 { @@ -23,16 +23,14 @@ namespace fdb5 { /// @note: as opposed to the TOC catalogue, the DAOS catalogue does not pre-load all indexes from storage. /// Instead, it selects and loads only those indexes that are required to fulfil the request. -DaosCatalogueReader::DaosCatalogueReader(const Key& dbKey, const fdb5::Config& config) : - DaosCatalogue(dbKey, config) { +DaosCatalogueReader::DaosCatalogueReader(const Key& dbKey, const fdb5::Config& config) : DaosCatalogue(dbKey, config) { /// @todo: schema is being loaded at DaosCatalogueWriter creation for write, but being loaded /// at DaosCatalogueReader::open for read. Is this OK? - } -DaosCatalogueReader::DaosCatalogueReader(const eckit::URI& uri, const fdb5::Config& config) : - DaosCatalogue(uri, ControlIdentifiers{}, config) {} +DaosCatalogueReader::DaosCatalogueReader(const eckit::URI& uri, const fdb5::Config& config) + : DaosCatalogue(uri, ControlIdentifiers{}, config) {} bool DaosCatalogueReader::selectIndex(const Key& idxKey) { @@ -54,8 +52,8 @@ bool DaosCatalogueReader::selectIndex(const Key& idxKey) { /// - ensure catalogue kv exists (daos_kv_open) fdb5::DaosKeyValue catalogue_kv_obj{s, catalogue_kv}; - int idx_loc_max_len = 512; /// @todo: take from config - std::vector n((long) idx_loc_max_len); + int idx_loc_max_len = 512; /// @todo: take from config + std::vector n((long)idx_loc_max_len); long res; try { @@ -70,7 +68,6 @@ bool DaosCatalogueReader::selectIndex(const Key& idxKey) { /// - close catalogue kv (daos_obj_close) return false; - } fdb5::DaosKeyValueName index_kv{eckit::URI{std::string{n.begin(), std::next(n.begin(), res)}}}; @@ -79,19 +76,16 @@ bool DaosCatalogueReader::selectIndex(const Key& idxKey) { /// @note: performed RPCs: /// - close catalogue kv (daos_obj_close) - } current_ = indexes_[idxKey]; return true; - } void DaosCatalogueReader::deselectIndex() { NOTIMP; //< should not be called - } bool DaosCatalogueReader::open() { @@ -107,10 +101,9 @@ bool DaosCatalogueReader::open() { DaosCatalogue::loadSchema(); return true; - } -bool DaosCatalogueReader::axis(const std::string &keyword, eckit::StringSet &s) const { +bool DaosCatalogueReader::axis(const std::string& keyword, eckit::StringSet& s) const { bool found = false; if (current_.axes().has(keyword)) { @@ -119,7 +112,6 @@ bool DaosCatalogueReader::axis(const std::string &keyword, eckit::StringSet &s) s.insert(a.begin(), a.end()); } return found; - } bool DaosCatalogueReader::retrieve(const Key& key, Field& field) const { @@ -127,10 +119,10 @@ bool DaosCatalogueReader::retrieve(const Key& key, Field& field) const { eckit::Log::debug() << "Trying to retrieve key " << key << std::endl; eckit::Log::debug() << "Scanning index " << current_.location() << std::endl; - if (!current_.mayContain(key)) return false; + if (!current_.mayContain(key)) + return false; return current_.get(key, fdb5::Key(), field); - } static fdb5::CatalogueReaderBuilder builder("daos"); diff --git a/src/fdb5/daos/DaosCatalogueReader.h b/src/fdb5/daos/DaosCatalogueReader.h index 022dd47d6..429d305ee 100644 --- a/src/fdb5/daos/DaosCatalogueReader.h +++ b/src/fdb5/daos/DaosCatalogueReader.h @@ -24,7 +24,6 @@ namespace fdb5 { class DaosCatalogueReader : public DaosCatalogue, public CatalogueReader { public: // methods - DaosCatalogueReader(const Key& key, const fdb5::Config& config); DaosCatalogueReader(const eckit::URI& uri, const fdb5::Config& config); @@ -37,22 +36,19 @@ class DaosCatalogueReader : public DaosCatalogue, public CatalogueReader { void flush(size_t archivedFields) override {} void clean() override {} void close() override {} - - bool axis(const std::string &keyword, eckit::StringSet &s) const override; + + bool axis(const std::string& keyword, eckit::StringSet& s) const override; bool retrieve(const Key& key, Field& field) const override; - void print( std::ostream &out ) const override { NOTIMP; } + void print(std::ostream& out) const override { NOTIMP; } private: // types - - typedef std::map< Key, Index> IndexStore; + typedef std::map IndexStore; private: // members - IndexStore indexes_; Index current_; - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/daos/DaosCatalogueWriter.cc b/src/fdb5/daos/DaosCatalogueWriter.cc index 56f2201a6..5c2cfc4ed 100644 --- a/src/fdb5/daos/DaosCatalogueWriter.cc +++ b/src/fdb5/daos/DaosCatalogueWriter.cc @@ -17,12 +17,12 @@ #include "fdb5/LibFdb5.h" -#include "fdb5/daos/DaosSession.h" -#include "fdb5/daos/DaosName.h" #include "fdb5/daos/DaosKeyValueHandle.h" +#include "fdb5/daos/DaosName.h" +#include "fdb5/daos/DaosSession.h" -#include "fdb5/daos/DaosIndex.h" #include "fdb5/daos/DaosCatalogueWriter.h" +#include "fdb5/daos/DaosIndex.h" // using namespace eckit; @@ -30,9 +30,9 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -DaosCatalogueWriter::DaosCatalogueWriter(const Key& key, const fdb5::Config& config) : - DaosCatalogue(key, config), firstIndexWrite_(false) { - +DaosCatalogueWriter::DaosCatalogueWriter(const Key& key, const fdb5::Config& config) + : DaosCatalogue(key, config) + , firstIndexWrite_(false) { fdb5::DaosSession s{}; @@ -60,19 +60,15 @@ DaosCatalogueWriter::DaosCatalogueWriter(const Key& key, const fdb5::Config& con catalogue_kv_name.create(); /// write schema under "schema" - eckit::Log::debug() << "Copy schema from " - << config_.schemaPath() - << " to " - << catalogue_kv_name.URI().asString() - << " at key 'schema'." - << std::endl; + eckit::Log::debug() << "Copy schema from " << config_.schemaPath() << " to " + << catalogue_kv_name.URI().asString() << " at key 'schema'." << std::endl; eckit::FileHandle in(config_.schemaPath()); std::unique_ptr out(catalogue_kv_name.dataHandle("schema")); in.copyTo(*out); /// write dbKey under "key" - eckit::MemoryHandle h{(size_t) PATH_MAX}; + eckit::MemoryHandle h{(size_t)PATH_MAX}; eckit::HandleStream hs{h}; h.openForWrite(eckit::Length(0)); { @@ -80,22 +76,21 @@ DaosCatalogueWriter::DaosCatalogueWriter(const Key& key, const fdb5::Config& con hs << dbKey_; } - int db_key_max_len = 512; // @todo: take from config + int db_key_max_len = 512; // @todo: take from config if (hs.bytesWritten() > db_key_max_len) throw eckit::Exception("Serialised db key exceeded configured maximum db key length."); - - fdb5::DaosKeyValue{s, catalogue_kv_name}.put("key", h.data(), hs.bytesWritten()); + + fdb5::DaosKeyValue{s, catalogue_kv_name}.put("key", h.data(), hs.bytesWritten()); /// index newly created catalogue kv in main kv - int db_loc_max_len = 512; // @todo: take from config + int db_loc_max_len = 512; // @todo: take from config std::string nstr = catalogue_kv_name.URI().asString(); - if (nstr.length() > db_loc_max_len) + if (nstr.length() > db_loc_max_len) throw eckit::Exception("Serialised db location exceeded configured maximum db location length."); main_kv.put(db_cont_, nstr.data(), nstr.length()); - } - + /// @todo: record or read dbUID /// @note: performed RPCs: @@ -104,21 +99,19 @@ DaosCatalogueWriter::DaosCatalogueWriter(const Key& key, const fdb5::Config& con DaosCatalogue::loadSchema(); /// @todo: TocCatalogue::checkUID(); - } -DaosCatalogueWriter::DaosCatalogueWriter(const eckit::URI &uri, const fdb5::Config& config) : - DaosCatalogue(uri, ControlIdentifiers{}, config), firstIndexWrite_(false) { +DaosCatalogueWriter::DaosCatalogueWriter(const eckit::URI& uri, const fdb5::Config& config) + : DaosCatalogue(uri, ControlIdentifiers{}, config) + , firstIndexWrite_(false) { NOTIMP; - } DaosCatalogueWriter::~DaosCatalogueWriter() { clean(); close(); - } bool DaosCatalogueWriter::selectIndex(const Key& idxKey) { @@ -136,60 +129,47 @@ bool DaosCatalogueWriter::selectIndex(const Key& idxKey) { /// - ensure catalogue kv exists (daos_kv_open) fdb5::DaosKeyValue catalogue_kv_obj{s, catalogue_kv}; - int idx_loc_max_len = 512; /// @todo: take from config + int idx_loc_max_len = 512; /// @todo: take from config try { - std::vector n((long) idx_loc_max_len); + std::vector n((long)idx_loc_max_len); long res; /// @note: performed RPCs: /// - get index location from catalogue kv (daos_kv_get) res = catalogue_kv_obj.get(idxKey.valuesToString(), &n[0], idx_loc_max_len); - indexes_[idxKey] = Index( - new fdb5::DaosIndex( - idxKey, - *this, - fdb5::DaosKeyValueName{eckit::URI{std::string{n.begin(), std::next(n.begin(), res)}}}, - false - ) - ); + indexes_[idxKey] = Index(new fdb5::DaosIndex( + idxKey, *this, fdb5::DaosKeyValueName{eckit::URI{std::string{n.begin(), std::next(n.begin(), res)}}}, + false)); } catch (fdb5::DaosEntityNotFoundException& e) { firstIndexWrite_ = true; - - indexes_[idxKey] = Index( - new fdb5::DaosIndex( - idxKey, - *this, - fdb5::DaosName{pool_, db_cont_} - ) - ); + + indexes_[idxKey] = Index(new fdb5::DaosIndex(idxKey, *this, fdb5::DaosName{pool_, db_cont_})); /// index index kv in catalogue kv std::string nstr{indexes_[idxKey].location().uri().asString()}; if (nstr.length() > idx_loc_max_len) throw eckit::Exception("Serialised index location exceeded configured maximum index location length."); - /// @note: performed RPCs (only if the index wasn't visited yet and index kv doesn't exist yet, i.e. only on first write to an index key): + /// @note: performed RPCs (only if the index wasn't visited yet and index kv doesn't exist yet, i.e. only on + /// first write to an index key): /// - record index kv location into catalogue kv (daos_kv_put) -- always performed catalogue_kv_obj.put(idxKey.valuesToString(), nstr.data(), nstr.length()); /// @note: performed RPCs: /// - close index kv when destroyed (daos_obj_close) - } /// @note: performed RPCs: /// - close catalogue kv (daos_obj_close) - } current_ = indexes_[idxKey]; return true; - } void DaosCatalogueWriter::deselectIndex() { @@ -197,7 +177,6 @@ void DaosCatalogueWriter::deselectIndex() { current_ = Index(); currentIndexKey_ = Key(); firstIndexWrite_ = false; - } void DaosCatalogueWriter::clean() { @@ -205,13 +184,11 @@ void DaosCatalogueWriter::clean() { flush(0); deselectIndex(); - } void DaosCatalogueWriter::close() { closeIndexes(); - } const Index& DaosCatalogueWriter::currentIndex() { @@ -222,13 +199,13 @@ const Index& DaosCatalogueWriter::currentIndex() { } return current_; - } /// @todo: other writers may be simultaneously updating the axes KeyValues in DAOS. Should these /// new updates be retrieved and put into in-memory axes from time to time, e.g. every /// time a value is put in an axis KeyValue? -void DaosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) { +void DaosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, + std::shared_ptr fieldLocation) { if (current_.null()) { ASSERT(!currentIndexKey_.empty()); @@ -251,11 +228,12 @@ void DaosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, std::s for (Key::const_iterator i = datumKey.begin(); i != datumKey.end(); ++i) { - const std::string &keyword = i->first; + const std::string& keyword = i->first; std::string value = datumKey.canonicalValue(keyword); - if (value.length() == 0) continue; + if (value.length() == 0) + continue; axisNames += sep + keyword; sep = ","; @@ -265,14 +243,12 @@ void DaosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, std::s /// empty sets. This is fine. const auto& axis_set = current_.axes().values(keyword); - //if (!axis_set.has_value() || !axis_set->get().contains(value)) { + // if (!axis_set.has_value() || !axis_set->get().contains(value)) { if (!axis_set.contains(value)) { axesToExpand.push_back(keyword); valuesToAdd.push_back(value); - } - } /// index the field and update in-memory axes @@ -302,13 +278,13 @@ void DaosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, std::s kv.put("axes", axisNames.data(), axisNames.length()); firstIndexWrite_ = false; - } /// @todo: axes are supposed to be sorted before persisting. How do we do this with the DAOS approach? /// sort axes every time they are loaded in the read pathway? - if (axesToExpand.empty()) return; + if (axesToExpand.empty()) + return; /// expand axis info in DAOS while (!axesToExpand.empty()) { @@ -331,21 +307,18 @@ void DaosCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, std::s axesToExpand.pop_back(); valuesToAdd.pop_back(); - } - } void DaosCatalogueWriter::flush(size_t archivedFields) { - if (!current_.null()) current_ = Index(); - + if (!current_.null()) + current_ = Index(); } void DaosCatalogueWriter::closeIndexes() { indexes_.clear(); // all indexes instances destroyed - } static fdb5::CatalogueWriterBuilder builder("daos"); diff --git a/src/fdb5/daos/DaosCatalogueWriter.h b/src/fdb5/daos/DaosCatalogueWriter.h index 0ea9f39bc..06ec4c984 100644 --- a/src/fdb5/daos/DaosCatalogueWriter.h +++ b/src/fdb5/daos/DaosCatalogueWriter.h @@ -24,30 +24,30 @@ namespace fdb5 { class DaosCatalogueWriter : public DaosCatalogue, public CatalogueWriter { public: // methods - DaosCatalogueWriter(const Key& key, const fdb5::Config& config); DaosCatalogueWriter(const eckit::URI& uri, const fdb5::Config& config); ~DaosCatalogueWriter() override; - void index(const Key& key, const eckit::URI &uri, eckit::Offset offset, eckit::Length length) override { NOTIMP; }; + void index(const Key& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) override { NOTIMP; }; void reconsolidate() override { NOTIMP; } /// Mount an existing TocCatalogue, which has a different metadata key (within /// constraints) to allow on-line rebadging of data /// variableKeys: The keys that are allowed to differ between the two DBs - void overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) override { NOTIMP; }; + void overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) override { + NOTIMP; + }; -// // Hide the contents of the DB!!! -// void hideContents() override; + // // Hide the contents of the DB!!! + // void hideContents() override; -// bool enabled(const ControlIdentifier& controlIdentifier) const override; + // bool enabled(const ControlIdentifier& controlIdentifier) const override; const Index& currentIndex() override; protected: // methods - bool selectIndex(const Key& idxKey) override; void deselectIndex() override; @@ -58,24 +58,20 @@ class DaosCatalogueWriter : public DaosCatalogue, public CatalogueWriter { void archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) override; - void print( std::ostream &out ) const override { NOTIMP; } + void print(std::ostream& out) const override { NOTIMP; } private: // methods - void closeIndexes(); private: // types - - typedef std::map< Key, Index> IndexStore; + typedef std::map IndexStore; private: // members - - IndexStore indexes_; + IndexStore indexes_; Index current_; bool firstIndexWrite_; - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/daos/DaosCommon.cc b/src/fdb5/daos/DaosCommon.cc index b7fcadb09..2db2b8f74 100644 --- a/src/fdb5/daos/DaosCommon.cc +++ b/src/fdb5/daos/DaosCommon.cc @@ -12,11 +12,11 @@ #include "fdb5/daos/DaosCommon.h" -#include "eckit/exception/Exceptions.h" #include "eckit/config/Resource.h" +#include "eckit/exception/Exceptions.h" -#include "fdb5/daos/DaosSession.h" #include "fdb5/daos/DaosName.h" +#include "fdb5/daos/DaosSession.h" namespace fdb5 { @@ -30,7 +30,6 @@ DaosCommon::DaosCommon(const fdb5::Config& config, const std::string& component, db_cont_ = key.valuesToString(); readConfig(config, component, true); - } DaosCommon::DaosCommon(const fdb5::Config& config, const std::string& component, const eckit::URI& uri) { @@ -43,37 +42,41 @@ DaosCommon::DaosCommon(const fdb5::Config& config, const std::string& component, db_cont_ = db_name.containerName(); readConfig(config, component, false); - } void DaosCommon::readConfig(const fdb5::Config& config, const std::string& component, bool readPool) { - if (readPool) pool_ = "default"; + if (readPool) + pool_ = "default"; root_cont_ = "root"; eckit::LocalConfiguration c{}; - if (config.has("daos")) c = config.getSubConfiguration("daos"); + if (config.has("daos")) + c = config.getSubConfiguration("daos"); if (readPool) - if (c.has(component)) pool_ = c.getSubConfiguration(component).getString("pool", pool_); - if (c.has(component)) root_cont_ = c.getSubConfiguration(component).getString("root_cont", root_cont_); + if (c.has(component)) + pool_ = c.getSubConfiguration(component).getString("pool", pool_); + if (c.has(component)) + root_cont_ = c.getSubConfiguration(component).getString("root_cont", root_cont_); std::string first_cap{component}; first_cap[0] = toupper(component[0]); std::string all_caps{component}; - for (auto & c: all_caps) c = toupper(c); + for (auto& c : all_caps) + c = toupper(c); if (readPool) pool_ = eckit::Resource("fdbDaos" + first_cap + "Pool;$FDB_DAOS_" + all_caps + "_POOL", pool_); - root_cont_ = eckit::Resource("fdbDaos" + first_cap + "RootCont;$FDB_DAOS_" + all_caps + "_ROOT_CONT", root_cont_); + root_cont_ = eckit::Resource("fdbDaos" + first_cap + "RootCont;$FDB_DAOS_" + all_caps + "_ROOT_CONT", + root_cont_); root_kv_.emplace(fdb5::DaosKeyValueName{pool_, root_cont_, main_kv_}); db_kv_.emplace(fdb5::DaosKeyValueName{pool_, db_cont_, catalogue_kv_}); if (c.has("client")) fdb5::DaosManager::instance().configure(c.getSubConfiguration("client")); - } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/daos/DaosCommon.h b/src/fdb5/daos/DaosCommon.h index 5b077d5fc..1e80c7455 100644 --- a/src/fdb5/daos/DaosCommon.h +++ b/src/fdb5/daos/DaosCommon.h @@ -16,18 +16,17 @@ #include "eckit/filesystem/URI.h" -#include "fdb5/database/Key.h" #include "fdb5/config/Config.h" +#include "fdb5/database/Key.h" -#include "fdb5/daos/DaosOID.h" #include "fdb5/daos/DaosName.h" +#include "fdb5/daos/DaosOID.h" namespace fdb5 { class DaosCommon { public: // methods - DaosCommon(const fdb5::Config&, const std::string& component, const fdb5::Key&); DaosCommon(const fdb5::Config&, const std::string& component, const eckit::URI&); @@ -35,11 +34,9 @@ class DaosCommon { const fdb5::DaosKeyValueName& dbKeyValue() const { return db_kv_.value(); } private: // methods - void readConfig(const fdb5::Config&, const std::string& component, bool readPool); protected: // members - std::string component_; std::string pool_; @@ -49,9 +46,8 @@ class DaosCommon { eckit::Optional root_kv_; eckit::Optional db_kv_; - fdb5::DaosOID main_kv_{0, 0, DAOS_OT_KV_HASHED, OC_S1}; /// @todo: take oclass from config - fdb5::DaosOID catalogue_kv_{0, 0, DAOS_OT_KV_HASHED, OC_S1}; /// @todo: take oclass from config - + fdb5::DaosOID main_kv_{0, 0, DAOS_OT_KV_HASHED, OC_S1}; /// @todo: take oclass from config + fdb5::DaosOID catalogue_kv_{0, 0, DAOS_OT_KV_HASHED, OC_S1}; /// @todo: take oclass from config }; -} \ No newline at end of file +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/daos/DaosContainer.cc b/src/fdb5/daos/DaosContainer.cc index 405a02794..966198212 100644 --- a/src/fdb5/daos/DaosContainer.cc +++ b/src/fdb5/daos/DaosContainer.cc @@ -8,32 +8,35 @@ * does it submit to any jurisdiction. */ - #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/TmpDir.h" #include "fdb5/LibFdb5.h" -#include "fdb5/daos/DaosSession.h" -#include "fdb5/daos/DaosPool.h" #include "fdb5/daos/DaosContainer.h" -#include "fdb5/daos/DaosObject.h" #include "fdb5/daos/DaosOID.h" +#include "fdb5/daos/DaosObject.h" +#include "fdb5/daos/DaosPool.h" +#include "fdb5/daos/DaosSession.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -DaosContainer::DaosContainer(DaosContainer&& other) noexcept : - pool_(other.pool_), - label_(std::move(other.label_)), coh_(std::move(other.coh_)), open_(other.open_), - oidAlloc_(std::move(other.oidAlloc_)) { +DaosContainer::DaosContainer(DaosContainer&& other) noexcept + : pool_(other.pool_) + , label_(std::move(other.label_)) + , coh_(std::move(other.coh_)) + , open_(other.open_) + , oidAlloc_(std::move(other.oidAlloc_)) { other.open_ = false; - } /// @todo: should forbid labels with UUID format? -DaosContainer::DaosContainer(fdb5::DaosPool& pool, const std::string& label) : pool_(pool), label_(label), open_(false) {} +DaosContainer::DaosContainer(fdb5::DaosPool& pool, const std::string& label) + : pool_(pool) + , label_(label) + , open_(false) {} DaosContainer::~DaosContainer() { @@ -41,18 +44,19 @@ DaosContainer::~DaosContainer() { /// objects in the container. /// What happens if we do obj.open() and then cont.close()? - if (open_) close(); - + if (open_) + close(); } void DaosContainer::create() { - /// @todo: not sure what to do here. Should probably keep track of whether + /// @todo: not sure what to do here. Should probably keep track of whether // the container has been created or not via this DaosContainer instance, - // and return accordingly. But the container may have been destroyed by another + // and return accordingly. But the container may have been destroyed by another // process or DaosContainer instance. - if (open_) return; + if (open_) + return; ASSERT(label_.size() > 0); @@ -63,12 +67,12 @@ void DaosContainer::create() { DAOS_CALL(daos_cont_create_with_label(poh, label_.c_str(), NULL, NULL, NULL)); } catch (fdb5::DaosEntityAlreadyExistsException& e) {} - } void DaosContainer::open() { - if (open_) return; + if (open_) + return; ASSERT(label_.size() > 0); @@ -77,7 +81,6 @@ void DaosContainer::open() { DAOS_CALL(daos_cont_open(poh, label_.c_str(), DAOS_COO_RW, &coh_, NULL, NULL)); open_ = true; - } void DaosContainer::close() { @@ -86,19 +89,18 @@ void DaosContainer::close() { eckit::Log::warning() << "Closing DaosContainer " << name() << ", container is not open" << std::endl; return; } - + LOG_DEBUG_LIB(LibFdb5) << "DAOS_CALL => daos_cont_close()" << std::endl; int code = daos_cont_close(coh_, NULL); - if (code < 0) eckit::Log::warning() << "DAOS error in call to daos_cont_close(), file " - << __FILE__ << ", line " << __LINE__ << ", function " << __func__ << " [" << code << "] (" - << code << ")" << std::endl; - + if (code < 0) + eckit::Log::warning() << "DAOS error in call to daos_cont_close(), file " << __FILE__ << ", line " << __LINE__ + << ", function " << __func__ << " [" << code << "] (" << code << ")" << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "DAOS_CALL <= daos_cont_close()" << std::endl; open_ = false; - } uint64_t DaosContainer::allocateOIDLo() { @@ -114,14 +116,14 @@ uint64_t DaosContainer::allocateOIDLo() { } return oidAlloc_.next_oid; - } fdb5::DaosArray DaosContainer::createArray(const daos_oclass_id_t& oclass, bool with_attr) { daos_otype_t otype = DAOS_OT_ARRAY; - if (!with_attr) otype = DAOS_OT_ARRAY_BYTE; + if (!with_attr) + otype = DAOS_OT_ARRAY_BYTE; fdb5::DaosOID new_oid{0, allocateOIDLo(), otype, oclass}; new_oid.generateReservedBits(*this); @@ -131,7 +133,6 @@ fdb5::DaosArray DaosContainer::createArray(const daos_oclass_id_t& oclass, bool fdb5::DaosArray obj(*this, new_oid, false); obj.create(); return obj; - } fdb5::DaosArray DaosContainer::createArray(const fdb5::DaosOID& oid) { @@ -143,7 +144,6 @@ fdb5::DaosArray DaosContainer::createArray(const fdb5::DaosOID& oid) { fdb5::DaosArray obj(*this, oid, false); obj.create(); return obj; - } fdb5::DaosKeyValue DaosContainer::createKeyValue(const daos_oclass_id_t& oclass) { @@ -156,7 +156,6 @@ fdb5::DaosKeyValue DaosContainer::createKeyValue(const daos_oclass_id_t& oclass) fdb5::DaosKeyValue obj(*this, new_oid, false); obj.create(); return obj; - } fdb5::DaosKeyValue DaosContainer::createKeyValue(const fdb5::DaosOID& oid) { @@ -168,26 +167,22 @@ fdb5::DaosKeyValue DaosContainer::createKeyValue(const fdb5::DaosOID& oid) { fdb5::DaosKeyValue obj(*this, oid, false); obj.create(); return obj; - } std::vector DaosContainer::listOIDs() { - + /// @todo: proper memory management /// @todo: auto snap destroyer daos_epoch_t e; - DAOS_CALL( - daos_cont_create_snap_opt( - coh_, &e, NULL, (enum daos_snapshot_opts)(DAOS_SNAP_OPT_CR | DAOS_SNAP_OPT_OIT), NULL - ) - ); + DAOS_CALL(daos_cont_create_snap_opt(coh_, &e, NULL, (enum daos_snapshot_opts)(DAOS_SNAP_OPT_CR | DAOS_SNAP_OPT_OIT), + NULL)); daos_handle_t oith; DAOS_CALL(daos_oit_open(coh_, e, &oith, NULL)); std::vector oids; daos_anchor_t anchor = DAOS_ANCHOR_INIT; - int max_oids_per_rpc = 10; /// @todo: take from config + int max_oids_per_rpc = 10; /// @todo: take from config daos_obj_id_t oid_batch[max_oids_per_rpc]; while (!daos_anchor_is_eof(&anchor)) { uint32_t oids_nr = max_oids_per_rpc; @@ -203,14 +198,12 @@ std::vector DaosContainer::listOIDs() { DAOS_CALL(daos_cont_destroy_snap(coh_, epr, NULL)); return oids; - } const daos_handle_t& DaosContainer::getOpenHandle() { - + open(); return coh_; - }; bool DaosContainer::exists() { @@ -219,32 +212,24 @@ bool DaosContainer::exists() { open(); - } catch ( fdb5::DaosEntityNotFoundException& e) { - - return false; + } catch (fdb5::DaosEntityNotFoundException& e) { return false; } - } - return true; - } std::string DaosContainer::name() const { return label_; - } std::string DaosContainer::label() const { return label_; - } fdb5::DaosPool& DaosContainer::getPool() const { return pool_; - } AutoContainerDestroy::~AutoContainerDestroy() noexcept(false) { @@ -262,9 +247,8 @@ AutoContainerDestroy::~AutoContainerDestroy() noexcept(false) { eckit::Log::error() << "** An exception is already in progress" << std::endl; eckit::Log::error() << "** Exception is ignored" << std::endl; } - } //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosContainer.h b/src/fdb5/daos/DaosContainer.h index 2268a80fd..4058a0286 100644 --- a/src/fdb5/daos/DaosContainer.h +++ b/src/fdb5/daos/DaosContainer.h @@ -33,7 +33,6 @@ class DaosPool; class DaosContainer { public: // methods - DaosContainer(DaosContainer&&) noexcept; ~DaosContainer(); @@ -51,7 +50,6 @@ class DaosContainer { fdb5::DaosPool& getPool() const; private: // methods - friend class DaosPool; DaosContainer(fdb5::DaosPool&, const std::string&); @@ -62,30 +60,25 @@ class DaosContainer { bool exists(); private: // members - fdb5::DaosPool& pool_; std::string label_ = std::string(); daos_handle_t coh_; bool open_; OidAlloc oidAlloc_{}; - }; class AutoContainerDestroy { public: // methods - AutoContainerDestroy(fdb5::DaosContainer& cont) : cont_(cont) {} ~AutoContainerDestroy() noexcept(false); private: // members - fdb5::DaosContainer& cont_; - }; //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosEngine.cc b/src/fdb5/daos/DaosEngine.cc index 7a12b98ee..07c1e109d 100644 --- a/src/fdb5/daos/DaosEngine.cc +++ b/src/fdb5/daos/DaosEngine.cc @@ -8,12 +8,12 @@ * does it submit to any jurisdiction. */ -#include "fdb5/LibFdb5.h" #include "fdb5/daos/DaosEngine.h" #include "eckit/serialisation/MemoryStream.h" +#include "fdb5/LibFdb5.h" -#include "fdb5/daos/DaosSession.h" #include "fdb5/daos/DaosName.h" +#include "fdb5/daos/DaosSession.h" using namespace eckit; @@ -23,13 +23,14 @@ namespace fdb5 { void DaosEngine::configureDaos(const Config& config) const { - if (daos_config_.has_value()) return; + if (daos_config_.has_value()) + return; daos_config_.emplace(eckit::LocalConfiguration()); - if (config.has("daos")) daos_config_.emplace(config.getSubConfiguration("daos")); + if (config.has("daos")) + daos_config_.emplace(config.getSubConfiguration("daos")); if (daos_config_->has("client")) fdb5::DaosManager::instance().configure(daos_config_->getSubConfiguration("client")); - } std::string DaosEngine::name() const { @@ -45,7 +46,8 @@ bool DaosEngine::canHandle(const eckit::URI& uri, const Config& config) const { fdb5::DaosName n{uri}; - if (!n.hasOID()) return false; + if (!n.hasOID()) + return false; /// @todo: check containerName is not root_cont_. root_cont_ should be populated in /// configureDaos as done in DaosCommon @@ -61,16 +63,14 @@ bool DaosEngine::canHandle(const eckit::URI& uri, const Config& config) const { bool is_catalogue_kv = (!is_root_name && !is_store_name && (n.OID() == n2.OID())); return is_catalogue_kv && n.exists(); - } -std::vector DaosEngine::visitableLocations(const Key& key, const Config& config) const -{ +std::vector DaosEngine::visitableLocations(const Key& key, const Config& config) const { /// @note: code mostly copied from DaosCommon - /// @note: should rather use DaosCommon, but can't inherit from it here as DaosEngine is - /// always instantiated even if daos is not used, and then DaosCommon would be unnecessarily - /// initialised. If owning a private instance of DaosCommon here, then the private members of + /// @note: should rather use DaosCommon, but can't inherit from it here as DaosEngine is + /// always instantiated even if daos is not used, and then DaosCommon would be unnecessarily + /// initialised. If owning a private instance of DaosCommon here, then the private members of /// DaosCommon are not accessible from here configureDaos(config); @@ -78,13 +78,15 @@ std::vector DaosEngine::visitableLocations(const Key& key, const Con std::string pool = "default"; std::string root_cont = "root"; - if (daos_config_->has("catalogue")) pool = daos_config_->getSubConfiguration("catalogue").getString("pool", pool); - if (daos_config_->has("catalogue")) root_cont = daos_config_->getSubConfiguration("catalogue").getString("root_cont", root_cont); + if (daos_config_->has("catalogue")) + pool = daos_config_->getSubConfiguration("catalogue").getString("pool", pool); + if (daos_config_->has("catalogue")) + root_cont = daos_config_->getSubConfiguration("catalogue").getString("root_cont", root_cont); pool = eckit::Resource("fdbDaosCataloguePool;$FDB_DAOS_CATALOGUE_POOL", pool); root_cont = eckit::Resource("fdbDaosCatalogueRootCont;$FDB_DAOS_CATALOGUE_ROOT_CONT", root_cont); - fdb5::DaosOID main_kv_oid{0, 0, DAOS_OT_KV_HASHED, OC_S1}; /// @todo: take oclass from config + fdb5::DaosOID main_kv_oid{0, 0, DAOS_OT_KV_HASHED, OC_S1}; /// @todo: take oclass from config /// --- @@ -96,15 +98,13 @@ std::vector DaosEngine::visitableLocations(const Key& key, const Con /// @note: performed RPCs: /// - main kv open (daos_kv_open) - + /// @todo: use this Optional technique in all cases where an action needs to be performed /// (i.e. not just throw an exception) if an object does not exist eckit::Optional main_kv; try { main_kv.emplace(s, main_kv_name); - } catch (fdb5::DaosEntityNotFoundException& e) { - return res; - } + } catch (fdb5::DaosEntityNotFoundException& e) { return res; } /// @note: performed RPCs: /// - main kv list keys (daos_kv_list) @@ -122,7 +122,7 @@ std::vector DaosEngine::visitableLocations(const Key& key, const Con eckit::URI uri(std::string(v.begin(), v.end())); ASSERT(uri.scheme() == typeName()); - /// @todo: this exact deserialisation is performed twice. Once here and once + /// @todo: this exact deserialisation is performed twice. Once here and once /// in DaosCatalogue::(uri, ...). Try to avoid one. fdb5::DaosKeyValueName db_kv_name{uri}; @@ -130,7 +130,7 @@ std::vector DaosEngine::visitableLocations(const Key& key, const Con /// - db kv open (daos_kv_open) /// - db key get size (daos_kv_get without a buffer) /// - db key get (daos_kv_get) - fdb5::DaosKeyValue db_kv{s, db_kv_name}; /// @note: includes exist check + fdb5::DaosKeyValue db_kv{s, db_kv_name}; /// @note: includes exist check std::vector data; eckit::MemoryStream ms = db_kv.getMemoryStream(data, "key", "DB kv"); fdb5::Key db_key(ms); @@ -139,31 +139,27 @@ std::vector DaosEngine::visitableLocations(const Key& key, const Con Log::debug() << " found match with " << main_kv_name.URI() << " at key " << k << std::endl; res.push_back(uri); - } - + } catch (eckit::Exception& e) { - eckit::Log::error() << "Error loading FDB database " << k << " from " << main_kv_name.URI() << std::endl; + eckit::Log::error() << "Error loading FDB database " << k << " from " << main_kv_name.URI() << std::endl; eckit::Log::error() << e.what() << std::endl; } - } return res; - } -std::vector DaosEngine::visitableLocations(const metkit::mars::MarsRequest& request, const Config& config) const -{ +std::vector DaosEngine::visitableLocations(const metkit::mars::MarsRequest& request, const Config& config) const { - /// @todo: in the POSIX backend, a set of possible visitable locations is first constructed - /// from the schema, and the set of visitable locations found in the root directories are + /// @todo: in the POSIX backend, a set of possible visitable locations is first constructed + /// from the schema, and the set of visitable locations found in the root directories are /// checked against that set. In the DAOS backend the first step is skipped. /// @note: code mostly copied from DaosCommon - /// @note: should rather use DaosCommon, but can't inherit from it here as DaosEngine is - /// always instantiated even if daos is not used, and then DaosCommon would be unnecessarily - /// initialised. If owning a private instance of DaosCommon here, then the private members of + /// @note: should rather use DaosCommon, but can't inherit from it here as DaosEngine is + /// always instantiated even if daos is not used, and then DaosCommon would be unnecessarily + /// initialised. If owning a private instance of DaosCommon here, then the private members of /// DaosCommon are not accessible from here configureDaos(config); @@ -171,13 +167,15 @@ std::vector DaosEngine::visitableLocations(const metkit::mars::MarsRequest& std::string pool = "default"; std::string root_cont = "root"; - if (daos_config_->has("catalogue")) pool = daos_config_->getSubConfiguration("catalogue").getString("pool", pool); - if (daos_config_->has("catalogue")) root_cont = daos_config_->getSubConfiguration("catalogue").getString("root_cont", root_cont); + if (daos_config_->has("catalogue")) + pool = daos_config_->getSubConfiguration("catalogue").getString("pool", pool); + if (daos_config_->has("catalogue")) + root_cont = daos_config_->getSubConfiguration("catalogue").getString("root_cont", root_cont); pool = eckit::Resource("fdbDaosCataloguePool;$FDB_DAOS_CATALOGUE_POOL", pool); root_cont = eckit::Resource("fdbDaosCatalogueRootCont;$FDB_DAOS_CATALOGUE_ROOT_CONT", root_cont); - fdb5::DaosOID main_kv_oid{0, 0, DAOS_OT_KV_HASHED, OC_S1}; /// @todo: take oclass from config + fdb5::DaosOID main_kv_oid{0, 0, DAOS_OT_KV_HASHED, OC_S1}; /// @todo: take oclass from config /// --- @@ -195,9 +193,7 @@ std::vector DaosEngine::visitableLocations(const metkit::mars::MarsRequest& eckit::Optional main_kv; try { main_kv.emplace(s, main_kv_name); - } catch (fdb5::DaosEntityNotFoundException& e) { - return res; - } + } catch (fdb5::DaosEntityNotFoundException& e) { return res; } /// @note: performed RPCs: /// - main kv list keys (daos_kv_list) @@ -215,7 +211,7 @@ std::vector DaosEngine::visitableLocations(const metkit::mars::MarsRequest& eckit::URI uri(std::string(v.begin(), v.end())); ASSERT(uri.scheme() == typeName()); - /// @todo: this exact deserialisation is performed twice. Once here and once + /// @todo: this exact deserialisation is performed twice. Once here and once /// in DaosCatalogue::(uri, ...). Try to avoid one. fdb5::DaosKeyValueName db_kv_name{uri}; @@ -232,18 +228,15 @@ std::vector DaosEngine::visitableLocations(const metkit::mars::MarsRequest& Log::debug() << " found match with " << main_kv_name.URI() << " at key " << k << std::endl; res.push_back(uri); - } - + } catch (eckit::Exception& e) { - eckit::Log::error() << "Error loading FDB database from " << main_kv_name.URI() << std::endl; + eckit::Log::error() << "Error loading FDB database from " << main_kv_name.URI() << std::endl; eckit::Log::error() << e.what() << std::endl; } - } return res; - } static EngineBuilder daos_builder; diff --git a/src/fdb5/daos/DaosEngine.h b/src/fdb5/daos/DaosEngine.h index 98510aaab..e974fcf02 100644 --- a/src/fdb5/daos/DaosEngine.h +++ b/src/fdb5/daos/DaosEngine.h @@ -26,13 +26,11 @@ namespace fdb5 { class DaosEngine : public fdb5::Engine { public: // methods - DaosEngine() {}; static const char* typeName() { return "daos"; } protected: // methods - std::string name() const override; std::string dbType() const override { NOTIMP; }; @@ -42,22 +40,19 @@ class DaosEngine : public fdb5::Engine { bool canHandle(const eckit::URI&, const Config&) const override; std::vector visitableLocations(const Key& key, const Config& config) const override; - std::vector visitableLocations(const metkit::mars::MarsRequest& rq, const Config& config) const override; + std::vector visitableLocations(const metkit::mars::MarsRequest& rq, + const Config& config) const override; - void print( std::ostream &out ) const override { NOTIMP; }; + void print(std::ostream& out) const override { NOTIMP; }; private: // methods - void configureDaos(const Config&) const; private: // members - mutable eckit::Optional daos_config_; - fdb5::DaosOID catalogue_kv_{0, 0, DAOS_OT_KV_HASHED, OC_S1}; // take oclass from config - + fdb5::DaosOID catalogue_kv_{0, 0, DAOS_OT_KV_HASHED, OC_S1}; // take oclass from config }; //---------------------------------------------------------------------------------------------------------------------- - } // namespace fdb5 diff --git a/src/fdb5/daos/DaosException.cc b/src/fdb5/daos/DaosException.cc index 9780cf0d0..601848a48 100644 --- a/src/fdb5/daos/DaosException.cc +++ b/src/fdb5/daos/DaosException.cc @@ -16,19 +16,18 @@ namespace fdb5 { DaosException::DaosException(const std::string& w) : Exception(w) {} -DaosException::DaosException(const std::string& w, const eckit::CodeLocation& l) : - Exception(w, l) {} +DaosException::DaosException(const std::string& w, const eckit::CodeLocation& l) : Exception(w, l) {} DaosEntityNotFoundException::DaosEntityNotFoundException(const std::string& w) : DaosException(w) {} -DaosEntityNotFoundException::DaosEntityNotFoundException(const std::string& w, const eckit::CodeLocation& l) : - DaosException(w, l) {} +DaosEntityNotFoundException::DaosEntityNotFoundException(const std::string& w, const eckit::CodeLocation& l) + : DaosException(w, l) {} DaosEntityAlreadyExistsException::DaosEntityAlreadyExistsException(const std::string& w) : DaosException(w) {} -DaosEntityAlreadyExistsException::DaosEntityAlreadyExistsException(const std::string& w, const eckit::CodeLocation& l) : - DaosException(w, l) {} +DaosEntityAlreadyExistsException::DaosEntityAlreadyExistsException(const std::string& w, const eckit::CodeLocation& l) + : DaosException(w, l) {} //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosException.h b/src/fdb5/daos/DaosException.h index a2fb153fc..cbd970d86 100644 --- a/src/fdb5/daos/DaosException.h +++ b/src/fdb5/daos/DaosException.h @@ -41,4 +41,4 @@ class DaosEntityAlreadyExistsException : public DaosException { //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosFieldLocation.cc b/src/fdb5/daos/DaosFieldLocation.cc index d77ffabe8..1aa2f0d72 100644 --- a/src/fdb5/daos/DaosFieldLocation.cc +++ b/src/fdb5/daos/DaosFieldLocation.cc @@ -8,30 +8,33 @@ * does it submit to any jurisdiction. */ -#include "eckit/filesystem/URIManager.h" #include "fdb5/daos/DaosFieldLocation.h" +#include "eckit/filesystem/URIManager.h" +#include "fdb5/LibFdb5.h" #include "fdb5/daos/DaosName.h" #include "fdb5/daos/DaosSession.h" -#include "fdb5/LibFdb5.h" namespace fdb5 { -::eckit::ClassSpec DaosFieldLocation::classSpec_ = {&FieldLocation::classSpec(), "DaosFieldLocation",}; +::eckit::ClassSpec DaosFieldLocation::classSpec_ = { + &FieldLocation::classSpec(), + "DaosFieldLocation", +}; ::eckit::Reanimator DaosFieldLocation::reanimator_; //---------------------------------------------------------------------------------------------------------------------- -DaosFieldLocation::DaosFieldLocation(const DaosFieldLocation& rhs) : - FieldLocation(rhs.uri_, rhs.offset_, rhs.length_, rhs.remapKey_) {} +DaosFieldLocation::DaosFieldLocation(const DaosFieldLocation& rhs) + : FieldLocation(rhs.uri_, rhs.offset_, rhs.length_, rhs.remapKey_) {} -DaosFieldLocation::DaosFieldLocation(const eckit::URI &uri) : FieldLocation(uri) {} +DaosFieldLocation::DaosFieldLocation(const eckit::URI& uri) : FieldLocation(uri) {} /// @todo: remove remapKey from signature and always pass empty Key to FieldLocation -DaosFieldLocation::DaosFieldLocation(const eckit::URI &uri, eckit::Offset offset, eckit::Length length, const Key& remapKey) : - FieldLocation(uri, offset, length, remapKey) {} +DaosFieldLocation::DaosFieldLocation(const eckit::URI& uri, eckit::Offset offset, eckit::Length length, + const Key& remapKey) + : FieldLocation(uri, offset, length, remapKey) {} -DaosFieldLocation::DaosFieldLocation(eckit::Stream& s) : - FieldLocation(s) {} +DaosFieldLocation::DaosFieldLocation(eckit::Stream& s) : FieldLocation(s) {} std::shared_ptr DaosFieldLocation::make_shared() const { return std::make_shared(std::move(*this)); @@ -40,10 +43,9 @@ std::shared_ptr DaosFieldLocation::make_shared() const { eckit::DataHandle* DaosFieldLocation::dataHandle() const { return fdb5::DaosArrayName(uri_).dataHandle(offset(), length()); - } -void DaosFieldLocation::print(std::ostream &out) const { +void DaosFieldLocation::print(std::ostream& out) const { out << "DaosFieldLocation[uri=" << uri_ << "]"; } @@ -61,34 +63,31 @@ class DaosURIManager : public eckit::URIManager { eckit::PathName path(const eckit::URI& f) const override { return f.name(); } - bool exists(const eckit::URI& f) override { - - return fdb5::DaosName(f).exists(); - - } + bool exists(const eckit::URI& f) override { return fdb5::DaosName(f).exists(); } eckit::DataHandle* newWriteHandle(const eckit::URI& f) override { - if (fdb5::DaosName(f).OID().otype() != DAOS_OT_ARRAY) NOTIMP; - + if (fdb5::DaosName(f).OID().otype() != DAOS_OT_ARRAY) + NOTIMP; + return fdb5::DaosArrayName(f).dataHandle(); - } eckit::DataHandle* newReadHandle(const eckit::URI& f) override { - if (fdb5::DaosName(f).OID().otype() != DAOS_OT_ARRAY) NOTIMP; - + if (fdb5::DaosName(f).OID().otype() != DAOS_OT_ARRAY) + NOTIMP; + return fdb5::DaosArrayName(f).dataHandle(); - } - eckit::DataHandle* newReadHandle(const eckit::URI& f, const eckit::OffsetList& ol, const eckit::LengthList& ll) override { + eckit::DataHandle* newReadHandle(const eckit::URI& f, const eckit::OffsetList& ol, + const eckit::LengthList& ll) override { + + if (fdb5::DaosName(f).OID().otype() != DAOS_OT_ARRAY) + NOTIMP; - if (fdb5::DaosName(f).OID().otype() != DAOS_OT_ARRAY) NOTIMP; - return fdb5::DaosArrayName(f).dataHandle(); - } std::string asString(const eckit::URI& uri) const override { @@ -101,6 +100,7 @@ class DaosURIManager : public eckit::URIManager { return uri.scheme() + ":" + uri.name() + q + f; } + public: DaosURIManager(const std::string& name) : eckit::URIManager(name) {} }; diff --git a/src/fdb5/daos/DaosFieldLocation.h b/src/fdb5/daos/DaosFieldLocation.h index cf5185884..85939c772 100644 --- a/src/fdb5/daos/DaosFieldLocation.h +++ b/src/fdb5/daos/DaosFieldLocation.h @@ -26,9 +26,8 @@ namespace fdb5 { class DaosFieldLocation : public FieldLocation { public: - DaosFieldLocation(const DaosFieldLocation& rhs); - DaosFieldLocation(const eckit::URI &uri); + DaosFieldLocation(const eckit::URI& uri); DaosFieldLocation(const eckit::URI& uri, eckit::Offset offset, eckit::Length length, const Key& remapKey); DaosFieldLocation(eckit::Stream&); @@ -39,23 +38,18 @@ class DaosFieldLocation : public FieldLocation { void visit(FieldLocationVisitor& visitor) const override; public: // For Streamable - - static const eckit::ClassSpec& classSpec() { return classSpec_;} + static const eckit::ClassSpec& classSpec() { return classSpec_; } protected: // For Streamable - const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } static eckit::ClassSpec classSpec_; static eckit::Reanimator reanimator_; private: // methods - - void print(std::ostream &out) const override; - + void print(std::ostream& out) const override; }; - //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 diff --git a/src/fdb5/daos/DaosIndex.cc b/src/fdb5/daos/DaosIndex.cc index 05a017e67..0dacdb089 100644 --- a/src/fdb5/daos/DaosIndex.cc +++ b/src/fdb5/daos/DaosIndex.cc @@ -8,14 +8,14 @@ * does it submit to any jurisdiction. */ -#include // for PATH_MAX +#include // for PATH_MAX #include "eckit/io/MemoryHandle.h" -#include "eckit/serialisation/MemoryStream.h" #include "eckit/serialisation/HandleStream.h" -#include "fdb5/daos/DaosSession.h" +#include "eckit/serialisation/MemoryStream.h" #include "fdb5/daos/DaosIndex.h" #include "fdb5/daos/DaosLazyFieldLocation.h" +#include "fdb5/daos/DaosSession.h" fdb5::DaosKeyValueName buildIndexKvName(const fdb5::Key& key, const fdb5::DaosName& name) { @@ -27,16 +27,15 @@ fdb5::DaosKeyValueName buildIndexKvName(const fdb5::Key& key, const fdb5::DaosNa fdb5::DaosKeyValueOID index_kv_oid{key.valuesToString(), OC_S1}; return fdb5::DaosKeyValueName{name.poolName(), name.containerName(), index_kv_oid}; - } namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -DaosIndex::DaosIndex(const Key& key, const Catalogue& catalogue, const fdb5::DaosName& name) : - IndexBase(key, "daosKeyValue", catalogue), - location_(buildIndexKvName(key, name), 0) { +DaosIndex::DaosIndex(const Key& key, const Catalogue& catalogue, const fdb5::DaosName& name) + : IndexBase(key, "daosKeyValue", catalogue) + , location_(buildIndexKvName(key, name), 0) { fdb5::DaosSession s{}; @@ -46,7 +45,7 @@ DaosIndex::DaosIndex(const Key& key, const Catalogue& catalogue, const fdb5::Dao fdb5::DaosKeyValue index_kv_obj{s, location_.daosName()}; /// write indexKey under "key" - eckit::MemoryHandle h{(size_t) PATH_MAX}; + eckit::MemoryHandle h{(size_t)PATH_MAX}; eckit::HandleStream hs{h}; h.openForWrite(eckit::Length(0)); { @@ -62,15 +61,14 @@ DaosIndex::DaosIndex(const Key& key, const Catalogue& catalogue, const fdb5::Dao /// @note: performed RPCs: /// - record index key into index kv (daos_kv_put) index_kv_obj.put("key", h.data(), hs.bytesWritten()); - } -DaosIndex::DaosIndex(const Key& key, const Catalogue& catalogue, const fdb5::DaosKeyValueName& name, bool readAxes) : - IndexBase(key, "daosKeyValue", catalogue), - location_(name, 0) { - - if (readAxes) updateAxes(); +DaosIndex::DaosIndex(const Key& key, const Catalogue& catalogue, const fdb5::DaosKeyValueName& name, bool readAxes) + : IndexBase(key, "daosKeyValue", catalogue) + , location_(name, 0) { + if (readAxes) + updateAxes(); } void DaosIndex::updateAxes() { @@ -82,8 +80,8 @@ void DaosIndex::updateAxes() { /// - ensure axis kv exists (daos_obj_open) fdb5::DaosKeyValue index_kv{s, index_kv_name}; - int axis_names_max_len = 512; /// @todo: take from config - std::vector axes_data((long) axis_names_max_len); + int axis_names_max_len = 512; /// @todo: take from config + std::vector axes_data((long)axis_names_max_len); /// @note: performed RPCs: /// - get axes key size and content (daos_kv_get without buffer + daos_kv_get) @@ -109,10 +107,9 @@ void DaosIndex::updateAxes() { } axes_.sort(); - } -bool DaosIndex::get(const Key& key, const Key& remapKey, Field &field) const { +bool DaosIndex::get(const Key& key, const Key& remapKey, Field& field) const { const fdb5::DaosKeyValueName& n = location_.daosName(); @@ -124,8 +121,8 @@ bool DaosIndex::get(const Key& key, const Key& remapKey, Field &field) const { std::string query{key.valuesToString()}; - int field_loc_max_len = 512; /// @todo: read from config - std::vector loc_data((long) field_loc_max_len); + int field_loc_max_len = 512; /// @todo: read from config + std::vector loc_data((long)field_loc_max_len); long res; try { @@ -140,10 +137,9 @@ bool DaosIndex::get(const Key& key, const Key& remapKey, Field &field) const { /// - close index kv (daos_obj_close) return false; - } - eckit::MemoryStream ms{&loc_data[0], (size_t) res}; + eckit::MemoryStream ms{&loc_data[0], (size_t)res}; /// @note: timestamp read for informational purpoes. See note in DaosIndex::add. time_t ts; @@ -156,23 +152,22 @@ bool DaosIndex::get(const Key& key, const Key& remapKey, Field &field) const { /// - close index kv (daos_obj_close) return true; - } -void DaosIndex::add(const Key& key, const Field &field) { +void DaosIndex::add(const Key& key, const Field& field) { - eckit::MemoryHandle h{(size_t) PATH_MAX}; + eckit::MemoryHandle h{(size_t)PATH_MAX}; eckit::HandleStream hs{h}; h.openForWrite(eckit::Length(0)); { eckit::AutoClose closer(h); /// @note: in the POSIX back-end, keeping a timestamp per index is necessary, to allow - /// determining which was the latest indexed field in cases where multiple processes + /// determining which was the latest indexed field in cases where multiple processes /// index a same field or in cases where multiple catalogues are combined with DistFDB. /// In the DAOS back-end, however, determining the latest indexed field is straigthforward - /// as all parallel processes writing fields for a same index key will share a DAOS + /// as all parallel processes writing fields for a same index key will share a DAOS /// key-value, and the last indexing will supersede the previous ones. - /// DistFDB will be obsoleted in favour of a centralised catalogue mechanism which can + /// DistFDB will be obsoleted in favour of a centralised catalogue mechanism which can /// index fields on multiple catalogues. /// Therefore keeping timestamps in DAOS should not be necessary. /// They are kept for now only for informational purposes. @@ -181,7 +176,7 @@ void DaosIndex::add(const Key& key, const Field &field) { hs << field.location(); } - int field_loc_max_len = 512; /// @todo: read from config + int field_loc_max_len = 512; /// @todo: read from config if (hs.bytesWritten() > field_loc_max_len) throw eckit::Exception("Serialised field location exceeded configured maximum location length."); @@ -191,12 +186,11 @@ void DaosIndex::add(const Key& key, const Field &field) { /// - ensure index kv exists (daos_obj_open) /// - record field key and location into index kv (daos_kv_put) /// - close index kv when destroyed (daos_obj_close) - fdb5::DaosKeyValue{s, location_.daosName()}.put(key.valuesToString(), h.data(), hs.bytesWritten()); - + fdb5::DaosKeyValue{s, location_.daosName()}.put(key.valuesToString(), h.data(), hs.bytesWritten()); } -void DaosIndex::entries(EntryVisitor &visitor) const { - +void DaosIndex::entries(EntryVisitor& visitor) const { + Index instantIndex(const_cast(this)); // Allow the visitor to selectively decline to visit the entries in this index @@ -211,22 +205,21 @@ void DaosIndex::entries(EntryVisitor &visitor) const { for (const auto& key : index_kv.keys()) { - if (key == "axes" || key == "key") continue; + if (key == "axes" || key == "key") + continue; - /// @note: the DaosCatalogue is currently indexing a serialised DaosFieldLocation for each - /// archived field key. In the list pathway, DaosLazyFieldLocations are built for all field - /// keys present in an index -- without retrieving the actual location --, and + /// @note: the DaosCatalogue is currently indexing a serialised DaosFieldLocation for each + /// archived field key. In the list pathway, DaosLazyFieldLocations are built for all field + /// keys present in an index -- without retrieving the actual location --, and /// ListVisitor::visitDatum is called for each (see note at the top of DaosLazyFieldLocation.h). - /// When a field key is matched in visitDatum, DaosLazyFieldLocation::stableLocation is called, - /// which in turn calls this method here and triggers retrieval and deserialisation of the - /// indexed DaosFieldLocation, and returns it. Since the deserialised instance is of a + /// When a field key is matched in visitDatum, DaosLazyFieldLocation::stableLocation is called, + /// which in turn calls this method here and triggers retrieval and deserialisation of the + /// indexed DaosFieldLocation, and returns it. Since the deserialised instance is of a /// polymorphic class, it needs to be reanimated. fdb5::FieldLocation* loc = new fdb5::DaosLazyFieldLocation(location_.daosName(), key); fdb5::Field field(std::move(*loc), time_t(), fdb5::FieldDetails()); visitor.visitDatum(field, key); - } - } } @@ -235,11 +228,13 @@ const std::vector DaosIndex::dataURIs() const { /// @note: if daos index + daos store, this will return a uri to a DAOS array for each indexed field /// @note: if daos index + posix store, this will return a vector of unique uris to all referenced posix files /// in this index (one for each writer process that has written to the index) - /// @note: in the case where we have a daos store, the current implementation of dataURIs is unnecessarily inefficient. - /// This method is only called in DaosWipeVisitor, where the uris obtained from this method are processed to obtain - /// unique store container paths - will always result in just one container uri! Having a URI store for each index in - /// DAOS could make this process more efficient, but it would imply more KV operations and slow down field writes. - /// @note: in the case where we have a posix store there will be more than one unique store file paths. The current + /// @note: in the case where we have a daos store, the current implementation of dataURIs is unnecessarily + /// inefficient. + /// This method is only called in DaosWipeVisitor, where the uris obtained from this method are processed to + /// obtain unique store container paths - will always result in just one container uri! Having a URI store for + /// each index in DAOS could make this process more efficient, but it would imply more KV operations and slow down + /// field writes. + /// @note: in the case where we have a posix store there will be more than one unique store file paths. The current /// implementation is still inefficient but preferred to maintaining a URI store in the DAOS catalogue fdb5::DaosSession s{}; @@ -249,21 +244,20 @@ const std::vector DaosIndex::dataURIs() const { for (const auto& key : index_kv.keys()) { - if (key == "axes" || key == "key") continue; + if (key == "axes" || key == "key") + continue; std::vector data; eckit::MemoryStream ms = index_kv.getMemoryStream(data, key, "index kv"); - + time_t ts; ms >> ts; std::unique_ptr fl(eckit::Reanimator::reanimate(ms)); res.insert(fl->uri()); - } return std::vector(res.begin(), res.end()); - } //----------------------------------------------------------------------------- diff --git a/src/fdb5/daos/DaosIndex.h b/src/fdb5/daos/DaosIndex.h index df83ed079..6837fb60d 100644 --- a/src/fdb5/daos/DaosIndex.h +++ b/src/fdb5/daos/DaosIndex.h @@ -13,19 +13,17 @@ #pragma once -#include "fdb5/database/Index.h" -#include "fdb5/daos/DaosName.h" #include "fdb5/daos/DaosIndexLocation.h" +#include "fdb5/daos/DaosName.h" +#include "fdb5/database/Index.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- - class DaosIndex : public IndexBase { public: // methods - /// @note: creates a new index in DAOS, in the container pointed to by 'name' DaosIndex(const Key& key, const Catalogue& catalogue, const fdb5::DaosName& name); /// @note: used to represent and operate with an index which already exists in DAOS @@ -35,7 +33,6 @@ class DaosIndex : public IndexBase { void funlock() const override { NOTIMP; } private: // methods - const IndexLocation& location() const override { return location_; } const std::vector dataURIs() const override; @@ -47,14 +44,16 @@ class DaosIndex : public IndexBase { void visit(IndexLocationVisitor& visitor) const override { NOTIMP; } - bool get(const Key& key, const Key& remapKey, Field &field ) const override; - void add(const Key& key, const Field &field ) override; + bool get(const Key& key, const Key& remapKey, Field& field) const override; + void add(const Key& key, const Field& field) override; void flush() override { NOTIMP; } void encode(eckit::Stream& s, const int version) const override { NOTIMP; } void entries(EntryVisitor& visitor) const override; - void print( std::ostream &out ) const override { NOTIMP; } - void dump(std::ostream& out, const char* indent, bool simple = false, bool dumpFields = false) const override { NOTIMP; } + void print(std::ostream& out) const override { NOTIMP; } + void dump(std::ostream& out, const char* indent, bool simple = false, bool dumpFields = false) const override { + NOTIMP; + } IndexStats statistics() const override { NOTIMP; } @@ -62,9 +61,7 @@ class DaosIndex : public IndexBase { void updateAxes(); private: // members - fdb5::DaosIndexLocation location_; - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/daos/DaosIndexLocation.cc b/src/fdb5/daos/DaosIndexLocation.cc index 8d2508332..5bce10a2b 100644 --- a/src/fdb5/daos/DaosIndexLocation.cc +++ b/src/fdb5/daos/DaosIndexLocation.cc @@ -16,10 +16,9 @@ namespace fdb5 { DaosIndexLocation::DaosIndexLocation(const fdb5::DaosKeyValueName& name, off_t offset) : name_(name), offset_(offset) {} -void DaosIndexLocation::print(std::ostream &out) const { +void DaosIndexLocation::print(std::ostream& out) const { out << "(" << name_.URI().asString() << ":" << offset_ << ")"; - } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/daos/DaosIndexLocation.h b/src/fdb5/daos/DaosIndexLocation.h index 217dfb6d8..691ae0cfc 100644 --- a/src/fdb5/daos/DaosIndexLocation.h +++ b/src/fdb5/daos/DaosIndexLocation.h @@ -26,7 +26,6 @@ namespace fdb5 { class DaosIndexLocation : public IndexLocation { public: // methods - DaosIndexLocation(const fdb5::DaosKeyValueName& name, off_t offset); eckit::URI uri() const override { return name_.URI(); } @@ -36,19 +35,15 @@ class DaosIndexLocation : public IndexLocation { const fdb5::DaosKeyValueName& daosName() const { return name_; }; protected: // For Streamable - void encode(eckit::Stream&) const override { NOTIMP; } private: // methods - - void print(std::ostream &out) const override; + void print(std::ostream& out) const override; private: // members - fdb5::DaosKeyValueName name_; off_t offset_; - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/daos/DaosKeyValueHandle.cc b/src/fdb5/daos/DaosKeyValueHandle.cc index bd3fa17cb..660633741 100644 --- a/src/fdb5/daos/DaosKeyValueHandle.cc +++ b/src/fdb5/daos/DaosKeyValueHandle.cc @@ -12,24 +12,28 @@ #include "eckit/exception/Exceptions.h" -#include "fdb5/daos/DaosKeyValueHandle.h" -#include "fdb5/daos/DaosPool.h" #include "fdb5/daos/DaosContainer.h" +#include "fdb5/daos/DaosException.h" +#include "fdb5/daos/DaosKeyValueHandle.h" #include "fdb5/daos/DaosObject.h" +#include "fdb5/daos/DaosPool.h" #include "fdb5/daos/DaosSession.h" -#include "fdb5/daos/DaosException.h" using eckit::Length; using eckit::Offset; namespace fdb5 { -DaosKeyValueHandle::DaosKeyValueHandle(const fdb5::DaosKeyValueName& name, const std::string& key) : name_(name), key_(key), open_(false), offset_(0) {} +DaosKeyValueHandle::DaosKeyValueHandle(const fdb5::DaosKeyValueName& name, const std::string& key) + : name_(name) + , key_(key) + , open_(false) + , offset_(0) {} DaosKeyValueHandle::~DaosKeyValueHandle() { - if (open_) eckit::Log::error() << "DaosKeyValueHandle not closed before destruction." << std::endl; - + if (open_) + eckit::Log::error() << "DaosKeyValueHandle not closed before destruction." << std::endl; } void DaosKeyValueHandle::print(std::ostream& s) const { @@ -38,31 +42,32 @@ void DaosKeyValueHandle::print(std::ostream& s) const { void DaosKeyValueHandle::openForWrite(const Length& len) { - if (open_) throw eckit::SeriousBug{"Handle already opened."}; + if (open_) + throw eckit::SeriousBug{"Handle already opened."}; session(); /// @todo: alternatively call name_.create() and the like fdb5::DaosPool& p = session_->getPool(name_.poolName()); fdb5::DaosContainer& c = p.ensureContainer(name_.containerName()); - + /// @note: only way to check kv existence without generating a snapshot is /// to attempt open, which results in creation without an error rc if n.e. /// A kv open is performed in both c.createKeyValue and DaosKeyValue(session, name). /// The former is used here. - kv_.emplace( c.createKeyValue(name_.OID()) ); + kv_.emplace(c.createKeyValue(name_.OID())); kv_->open(); open_ = true; offset_ = eckit::Offset(0); - } Length DaosKeyValueHandle::openForRead() { - if (open_) throw eckit::SeriousBug{"Handle already opened."}; + if (open_) + throw eckit::SeriousBug{"Handle already opened."}; session(); @@ -75,7 +80,6 @@ Length DaosKeyValueHandle::openForRead() { offset_ = eckit::Offset(0); return Length(kv_->size(key_)); - } long DaosKeyValueHandle::write(const void* buf, long len) { @@ -87,7 +91,6 @@ long DaosKeyValueHandle::write(const void* buf, long len) { offset_ += written; return written; - } long DaosKeyValueHandle::read(void* buf, long len) { @@ -99,23 +102,21 @@ long DaosKeyValueHandle::read(void* buf, long len) { offset_ += read; return read; - } void DaosKeyValueHandle::close() { - if (!open_) return; + if (!open_) + return; kv_->close(); open_ = false; - } void DaosKeyValueHandle::flush() { /// empty implmenetation - } Length DaosKeyValueHandle::size() { @@ -123,39 +124,34 @@ Length DaosKeyValueHandle::size() { fdb5::DaosKeyValue kv{session(), name_}; return Length(kv.size(key_)); - } Length DaosKeyValueHandle::estimate() { return size(); - } Offset DaosKeyValueHandle::position() { /// @todo: should position() crash if unopened? return offset_; - } bool DaosKeyValueHandle::canSeek() const { return false; - } std::string DaosKeyValueHandle::title() const { - - return name_.asString(); + return name_.asString(); } fdb5::DaosSession& DaosKeyValueHandle::session() { - if (!session_.has_value()) session_.emplace(); + if (!session_.has_value()) + session_.emplace(); return session_.value(); - } -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosKeyValueHandle.h b/src/fdb5/daos/DaosKeyValueHandle.h index d7afa44b8..4fe23425b 100644 --- a/src/fdb5/daos/DaosKeyValueHandle.h +++ b/src/fdb5/daos/DaosKeyValueHandle.h @@ -31,7 +31,6 @@ class DaosKeyValueName; class DaosKeyValueHandle : public eckit::DataHandle { public: // methods - DaosKeyValueHandle(const fdb5::DaosKeyValueName&, const std::string& key); ~DaosKeyValueHandle(); @@ -54,20 +53,17 @@ class DaosKeyValueHandle : public eckit::DataHandle { std::string title() const override; private: // methods - fdb5::DaosSession& session(); private: // members - mutable fdb5::DaosKeyValueName name_; std::string key_; std::optional session_; std::optional kv_; bool open_; eckit::Offset offset_; - }; //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 \ No newline at end of file +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/daos/DaosLazyFieldLocation.cc b/src/fdb5/daos/DaosLazyFieldLocation.cc index d091cb7f5..bbd405235 100644 --- a/src/fdb5/daos/DaosLazyFieldLocation.cc +++ b/src/fdb5/daos/DaosLazyFieldLocation.cc @@ -18,11 +18,15 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -DaosLazyFieldLocation::DaosLazyFieldLocation(const fdb5::DaosLazyFieldLocation& rhs) : - FieldLocation(), index_(rhs.index_), key_(rhs.key_) {} +DaosLazyFieldLocation::DaosLazyFieldLocation(const fdb5::DaosLazyFieldLocation& rhs) + : FieldLocation() + , index_(rhs.index_) + , key_(rhs.key_) {} -DaosLazyFieldLocation::DaosLazyFieldLocation(const fdb5::DaosKeyValueName& index, const std::string& key) : - FieldLocation(), index_(index), key_(key) {} +DaosLazyFieldLocation::DaosLazyFieldLocation(const fdb5::DaosKeyValueName& index, const std::string& key) + : FieldLocation() + , index_(index) + , key_(key) {} std::shared_ptr DaosLazyFieldLocation::make_shared() const { return std::make_shared(std::move(*this)); @@ -31,10 +35,9 @@ std::shared_ptr DaosLazyFieldLocation::make_shared() const eckit::DataHandle* DaosLazyFieldLocation::dataHandle() const { return realise()->dataHandle(); - } -void DaosLazyFieldLocation::print(std::ostream &out) const { +void DaosLazyFieldLocation::print(std::ostream& out) const { out << *realise(); } @@ -48,7 +51,8 @@ std::shared_ptr DaosLazyFieldLocation::stableLocation() con std::unique_ptr& DaosLazyFieldLocation::realise() const { - if (fl_) return fl_; + if (fl_) + return fl_; /// @note: performed RPCs: /// - index kv get (daos_kv_get) @@ -64,7 +68,6 @@ std::unique_ptr& DaosLazyFieldLocation::realise() const { fl_.reset(eckit::Reanimator::reanimate(ms)); return fl_; - } } // namespace fdb5 diff --git a/src/fdb5/daos/DaosLazyFieldLocation.h b/src/fdb5/daos/DaosLazyFieldLocation.h index bd324f837..d2ee0e268 100644 --- a/src/fdb5/daos/DaosLazyFieldLocation.h +++ b/src/fdb5/daos/DaosLazyFieldLocation.h @@ -21,16 +21,15 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -/// @note: used in fdb-list index visiting, in DaosIndex::entries. During -/// visitation, DaosFieldLocations are built, which normally require +/// @note: used in fdb-list index visiting, in DaosIndex::entries. During +/// visitation, DaosFieldLocations are built, which normally require /// retrieving the location information from DAOS, inflicting RPCs. -/// This DaosLazyFieldLocation, instead, remains empty and the actual +/// This DaosLazyFieldLocation, instead, remains empty and the actual /// information is only be retrieved from DAOS when stableLocation() /// is called. This allows the visiting mechanism to discard unmatching -/// FieldLocations before any RPC is performed for them. +/// FieldLocations before any RPC is performed for them. class DaosLazyFieldLocation : public FieldLocation { public: - DaosLazyFieldLocation(const fdb5::DaosLazyFieldLocation& rhs); DaosLazyFieldLocation(const fdb5::DaosKeyValueName& index, const std::string& key); @@ -43,20 +42,16 @@ class DaosLazyFieldLocation : public FieldLocation { std::shared_ptr stableLocation() const override; private: // methods - std::unique_ptr& realise() const; - void print(std::ostream &out) const override; + void print(std::ostream& out) const override; private: // members - fdb5::DaosKeyValueName index_; std::string key_; mutable std::unique_ptr fl_; - }; - //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 diff --git a/src/fdb5/daos/DaosName.cc b/src/fdb5/daos/DaosName.cc index 922965ed8..def3f81e0 100644 --- a/src/fdb5/daos/DaosName.cc +++ b/src/fdb5/daos/DaosName.cc @@ -9,17 +9,17 @@ */ #include "eckit/exception/Exceptions.h" -#include "eckit/utils/Tokenizer.h" #include "eckit/filesystem/PathName.h" +#include "eckit/utils/Tokenizer.h" -#include "fdb5/daos/DaosName.h" -#include "fdb5/daos/DaosSession.h" -#include "fdb5/daos/DaosPool.h" -#include "fdb5/daos/DaosContainer.h" #include "fdb5/daos/DaosArrayHandle.h" #include "fdb5/daos/DaosArrayPartHandle.h" -#include "fdb5/daos/DaosKeyValueHandle.h" +#include "fdb5/daos/DaosContainer.h" #include "fdb5/daos/DaosException.h" +#include "fdb5/daos/DaosKeyValueHandle.h" +#include "fdb5/daos/DaosName.h" +#include "fdb5/daos/DaosPool.h" +#include "fdb5/daos/DaosSession.h" namespace fdb5 { @@ -29,9 +29,12 @@ DaosNameBase::DaosNameBase(const std::string& pool) : pool_(pool) {} DaosNameBase::DaosNameBase(const std::string& pool, const std::string& cont) : pool_(pool), cont_(cont) {} -DaosNameBase::DaosNameBase(const std::string& pool, const std::string& cont, const fdb5::DaosOID& oid) : pool_(pool), cont_(cont), oid_(oid) {} +DaosNameBase::DaosNameBase(const std::string& pool, const std::string& cont, const fdb5::DaosOID& oid) + : pool_(pool) + , cont_(cont) + , oid_(oid) {} -DaosNameBase::DaosNameBase(const eckit::URI& uri) { +DaosNameBase::DaosNameBase(const eckit::URI& uri) { /// @note: uri expects a string with the following format: /// daos:pool[/cont[/oidhi.oidlo]] @@ -48,15 +51,16 @@ DaosNameBase::DaosNameBase(const eckit::URI& uri) { ASSERT(bits.size() < 4); pool_ = bits[0]; - if (bits.size() > 1) cont_.emplace(bits[1]); - if (bits.size() > 2) oid_.emplace(bits[2]); - + if (bits.size() > 1) + cont_.emplace(bits[1]); + if (bits.size() > 2) + oid_.emplace(bits[2]); } -DaosNameBase::DaosNameBase(const fdb5::DaosObject& obj) : - pool_(obj.getContainer().getPool().name()), - cont_(obj.getContainer().name()), - oid_(obj.OID()) {} +DaosNameBase::DaosNameBase(const fdb5::DaosObject& obj) + : pool_(obj.getContainer().getPool().name()) + , cont_(obj.getContainer().name()) + , oid_(obj.OID()) {} fdb5::DaosOID DaosNameBase::allocOID(const daos_otype_t& otype, const daos_oclass_id_t& oclass) const { @@ -68,25 +72,24 @@ fdb5::DaosOID DaosNameBase::allocOID(const daos_otype_t& otype, const daos_oclas fdb5::DaosContainer& c = p.getContainer(cont_.value()); return fdb5::DaosOID{0, c.allocateOIDLo(), otype, oclass}; - } -/// @note: to be called in all DaosName* methods and DaosName* user code that intend -/// having this Name instance's OID generated (having its reserved bits populated) +/// @note: to be called in all DaosName* methods and DaosName* user code that intend +/// having this Name instance's OID generated (having its reserved bits populated) /// as a post-condition. void DaosNameBase::ensureGeneratedOID() const { ASSERT(cont_.has_value()); ASSERT(oid_.has_value()); - if (oid_.value().wasGenerated()) return; - + if (oid_.value().wasGenerated()) + return; + fdb5::DaosSession s{}; fdb5::DaosPool& p = s.getPool(pool_); fdb5::DaosContainer& c = p.getContainer(cont_.value()); oid_.value().generateReservedBits(c); - } /// @todo: the otype parameter is unnecessary @@ -95,9 +98,9 @@ std::unique_ptr DaosNameBase::buildObject(fdb5::DaosSession& s /// @note: will throw DaosEntityNotFoundException if not exists if (oid_->otype() == DAOS_OT_ARRAY || oid_->otype() == DAOS_OT_ARRAY_BYTE) return std::make_unique(s, *this); - if (oid_->otype() == DAOS_OT_KV_HASHED) return std::make_unique(s, *this); + if (oid_->otype() == DAOS_OT_KV_HASHED) + return std::make_unique(s, *this); throw eckit::Exception("Provided otype not recognised."); - } void DaosNameBase::create() const { @@ -108,13 +111,15 @@ void DaosNameBase::create() const { fdb5::DaosPool& p = s.getPool(pool_); fdb5::DaosContainer& c = p.ensureContainer(cont_.value()); - - if (!oid_.has_value()) return; + + if (!oid_.has_value()) + return; ensureGeneratedOID(); switch (oid_.value().otype()) { - case DAOS_OT_ARRAY: case DAOS_OT_ARRAY_BYTE: + case DAOS_OT_ARRAY: + case DAOS_OT_ARRAY_BYTE: c.createArray(oid_.value()); break; case DAOS_OT_KV_HASHED: @@ -123,7 +128,6 @@ void DaosNameBase::create() const { default: throw eckit::Exception("Provided otype not recognised."); } - } void DaosNameBase::destroy() const { @@ -145,7 +149,7 @@ void DaosNameBase::destroy() const { fdb5::DaosContainer& c = p.getContainer(cont_.value()); ensureGeneratedOID(); - + /// @todo: improve this code if (oid_->otype() == DAOS_OT_KV_HASHED) { @@ -165,20 +169,18 @@ void DaosNameBase::destroy() const { } else { NOTIMP; - } - } eckit::Length DaosNameBase::size() const { - if (!oid_.has_value()) NOTIMP; + if (!oid_.has_value()) + NOTIMP; ensureGeneratedOID(); fdb5::DaosSession s{}; return eckit::Length(buildObject(s)->size()); - } bool DaosNameBase::exists() const { @@ -197,25 +199,26 @@ bool DaosNameBase::exists() const { buildObject(s); } - } catch (const fdb5::DaosEntityNotFoundException& e) { - return false; - } + } catch (const fdb5::DaosEntityNotFoundException& e) { return false; } return true; - } std::string DaosNameBase::asString() const { - if (as_string_.has_value()) return as_string_.value(); + if (as_string_.has_value()) + return as_string_.value(); - if (oid_.has_value()) ensureGeneratedOID(); + if (oid_.has_value()) + ensureGeneratedOID(); eckit::StringList v{pool_}; - if (cont_.has_value()) v.push_back(cont_.value()); - if (oid_.has_value()) v.push_back(oid_.value().asString()); + if (cont_.has_value()) + v.push_back(cont_.value()); + if (oid_.has_value()) + v.push_back(oid_.value().asString()); std::ostringstream oss; - const char *sep = ""; + const char* sep = ""; for (eckit::StringList::const_iterator j = v.begin(); j != v.end(); ++j) { oss << sep; oss << *j; @@ -225,26 +228,22 @@ std::string DaosNameBase::asString() const { as_string_.emplace(oss.str()); return as_string_.value(); - } eckit::URI DaosNameBase::URI() const { return eckit::URI("daos", eckit::PathName(asString())); - } const std::string& DaosNameBase::poolName() const { return pool_; - } const std::string& DaosNameBase::containerName() const { ASSERT(cont_.has_value()); return cont_.value(); - } const fdb5::DaosOID& DaosNameBase::OID() const { @@ -252,7 +251,6 @@ const fdb5::DaosOID& DaosNameBase::OID() const { ASSERT(oid_.has_value()); ensureGeneratedOID(); return oid_.value(); - } bool DaosNameBase::operator<(const DaosNameBase& other) const { @@ -279,17 +277,16 @@ bool DaosNameBase::operator>=(const DaosNameBase& other) const { return this->asString() >= other.asString(); } -DaosArrayName::DaosArrayName(const std::string& pool, const std::string& cont, const fdb5::DaosOID& oid) : DaosNameBase(pool, cont, oid) { - +DaosArrayName::DaosArrayName(const std::string& pool, const std::string& cont, const fdb5::DaosOID& oid) + : DaosNameBase(pool, cont, oid) { + ASSERT(oid_.value().otype() == DAOS_OT_ARRAY || oid_.value().otype() == DAOS_OT_ARRAY_BYTE); - } DaosArrayName::DaosArrayName(const eckit::URI& uri) : DaosNameBase(uri) { ASSERT(oid_.has_value()); ASSERT(oid_.value().otype() == DAOS_OT_ARRAY || oid_.value().otype() == DAOS_OT_ARRAY_BYTE); - } DaosArrayName::DaosArrayName(const fdb5::DaosArray& arr) : DaosNameBase(arr) {} @@ -297,26 +294,23 @@ DaosArrayName::DaosArrayName(const fdb5::DaosArray& arr) : DaosNameBase(arr) {} eckit::DataHandle* DaosArrayName::dataHandle(bool overwrite) const { return new fdb5::DaosArrayHandle(*this); - } eckit::DataHandle* DaosArrayName::dataHandle(const eckit::Offset& off, const eckit::Length& len) const { return new fdb5::DaosArrayPartHandle(*this, off, len); - } -DaosKeyValueName::DaosKeyValueName(const std::string& pool, const std::string& cont, const fdb5::DaosOID& oid) : DaosNameBase(pool, cont, oid) { - +DaosKeyValueName::DaosKeyValueName(const std::string& pool, const std::string& cont, const fdb5::DaosOID& oid) + : DaosNameBase(pool, cont, oid) { + ASSERT(oid_.value().otype() == DAOS_OT_KV_HASHED); - } DaosKeyValueName::DaosKeyValueName(const eckit::URI& uri) : DaosNameBase(uri) { ASSERT(oid_.has_value()); ASSERT(oid_.value().otype() == DAOS_OT_KV_HASHED); - } DaosKeyValueName::DaosKeyValueName(const fdb5::DaosKeyValue& kv) : DaosNameBase(kv) {} @@ -329,21 +323,20 @@ bool DaosKeyValueName::has(const std::string& key) const { fdb5::DaosKeyValue kv{s, *this}; return kv.has(key); - } eckit::DataHandle* DaosKeyValueName::dataHandle(const std::string& key, bool overwrite) const { /// @todo: OK to serialise pointers in DaosName? return new fdb5::DaosKeyValueHandle(*this, key); - } DaosName::DaosName(const std::string& pool) : DaosNameBase(pool) {} DaosName::DaosName(const std::string& pool, const std::string& cont) : DaosNameBase(pool, cont) {} -DaosName::DaosName(const std::string& pool, const std::string& cont, const fdb5::DaosOID& oid) : DaosNameBase(pool, cont, oid) {} +DaosName::DaosName(const std::string& pool, const std::string& cont, const fdb5::DaosOID& oid) + : DaosNameBase(pool, cont, oid) {} DaosName::DaosName(const eckit::URI& uri) : DaosNameBase(uri) {} @@ -358,7 +351,6 @@ DaosArrayName DaosName::createArrayName(const daos_oclass_id_t& oclass, bool wit } else { return DaosArrayName{pool_, cont_.value(), allocOID(DAOS_OT_ARRAY_BYTE, oclass)}; } - } DaosKeyValueName DaosName::createKeyValueName(const daos_oclass_id_t& oclass) const { @@ -366,7 +358,6 @@ DaosKeyValueName DaosName::createKeyValueName(const daos_oclass_id_t& oclass) co ASSERT(hasContainerName() && !hasOID()); create(); return DaosKeyValueName{pool_, cont_.value(), allocOID(DAOS_OT_KV_HASHED, oclass)}; - } std::vector DaosName::listOIDs() const { @@ -378,9 +369,8 @@ std::vector DaosName::listOIDs() const { fdb5::DaosContainer& c = p.getContainer(cont_.value()); return c.listOIDs(); - } //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosName.h b/src/fdb5/daos/DaosName.h index c1f1bcab8..4604e4278 100644 --- a/src/fdb5/daos/DaosName.h +++ b/src/fdb5/daos/DaosName.h @@ -15,15 +15,15 @@ #include -#include #include +#include -#include "eckit/utils/Optional.h" #include "eckit/filesystem/URI.h" #include "eckit/io/DataHandle.h" +#include "eckit/utils/Optional.h" -#include "fdb5/daos/DaosObject.h" #include "fdb5/daos/DaosOID.h" +#include "fdb5/daos/DaosObject.h" namespace fdb5 { @@ -36,7 +36,6 @@ class DaosContainer; class DaosNameBase { public: // methods - /// @todo: implement DaosName::containerName to return a container DaosName, /// and rename poolName and containerName to something else // DaosName poolName() const; @@ -54,7 +53,7 @@ class DaosNameBase { /// @todo: asString should only be used for debugging. private print? /// @todo: asString currently ensures the OID, if present, has been generated, but /// OID generation fails if the pool or container do not exist, and asString does - /// not succeed. A method should be implemented which supports stringifying + /// not succeed. A method should be implemented which supports stringifying /// non-existing objects. std::string asString() const; eckit::URI URI() const; @@ -72,7 +71,6 @@ class DaosNameBase { bool operator>=(const DaosNameBase& other) const; protected: // methods - DaosNameBase(const std::string& pool); DaosNameBase(const std::string& pool, const std::string& cont); DaosNameBase(const std::string& pool, const std::string& cont, const fdb5::DaosOID& oid); @@ -82,22 +80,18 @@ class DaosNameBase { fdb5::DaosOID allocOID(const daos_otype_t&, const daos_oclass_id_t&) const; private: // methods - std::unique_ptr buildObject(fdb5::DaosSession&) const; protected: // members - std::string pool_; eckit::Optional cont_; mutable eckit::Optional oid_; mutable eckit::Optional as_string_; - }; class DaosArrayName : public DaosNameBase { public: // methods - DaosArrayName(const std::string& pool, const std::string& cont, const fdb5::DaosOID& oid); DaosArrayName(const eckit::URI&); DaosArrayName(const fdb5::DaosArray&); @@ -105,13 +99,11 @@ class DaosArrayName : public DaosNameBase { /// @todo: think if this overwrite parameter makes sense in DAOS eckit::DataHandle* dataHandle(bool overwrite = false) const; eckit::DataHandle* dataHandle(const eckit::Offset&, const eckit::Length&) const; - }; class DaosKeyValueName : public DaosNameBase { public: // methods - DaosKeyValueName(const std::string& pool, const std::string& cont, const fdb5::DaosOID& oid); DaosKeyValueName(const eckit::URI&); DaosKeyValueName(const fdb5::DaosKeyValue&); @@ -120,14 +112,12 @@ class DaosKeyValueName : public DaosNameBase { /// @todo: think if this overwrite parameter makes sense in DAOS eckit::DataHandle* dataHandle(const std::string& key, bool overwrite = false) const; - }; /// @note: DaosName can represent a pool, container or object. Provides convenient functionality for containers class DaosName : public DaosNameBase { public: // methods - DaosName(const std::string& pool); DaosName(const std::string& pool, const std::string& cont); DaosName(const std::string& pool, const std::string& cont, const fdb5::DaosOID& oid); @@ -137,9 +127,8 @@ class DaosName : public DaosNameBase { fdb5::DaosArrayName createArrayName(const daos_oclass_id_t& oclass = OC_S1, bool with_attr = true) const; fdb5::DaosKeyValueName createKeyValueName(const daos_oclass_id_t& oclass = OC_S1) const; std::vector listOIDs() const; - }; //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosOID.cc b/src/fdb5/daos/DaosOID.cc index 30eeeebf5..6b0893043 100644 --- a/src/fdb5/daos/DaosOID.cc +++ b/src/fdb5/daos/DaosOID.cc @@ -8,16 +8,16 @@ * does it submit to any jurisdiction. */ -#include -#include #include +#include +#include #include "eckit/exception/Exceptions.h" -#include "eckit/utils/Translator.h" #include "eckit/utils/MD5.h" +#include "eckit/utils/Translator.h" -#include "fdb5/daos/DaosOID.h" #include "fdb5/daos/DaosContainer.h" +#include "fdb5/daos/DaosOID.h" #include "fdb5/daos/DaosSession.h" namespace fdb5 { @@ -28,13 +28,11 @@ void DaosOID::parseReservedBits() { otype_ = static_cast((oid_.hi & OID_FMT_TYPE_MASK) >> OID_FMT_TYPE_SHIFT); oclass_ = OBJ_CLASS_DEF(OR_RP_1, (oid_.hi & OID_FMT_CLASS_MASK) >> OID_FMT_CLASS_SHIFT); - } DaosOID::DaosOID(const uint64_t& hi, const uint64_t& lo) : oid_({lo, hi}), wasGenerated_(true) { parseReservedBits(); - } DaosOID::DaosOID(const std::string& s) : wasGenerated_(true) { @@ -46,45 +44,48 @@ DaosOID::DaosOID(const std::string& s) : wasGenerated_(true) { oid_.lo = std::stoull(s.substr(16, 16), nullptr, 16); parseReservedBits(); - } -DaosOID::DaosOID(const uint32_t& hi, const uint64_t& lo, const enum daos_otype_t& otype, const daos_oclass_id_t& oclass) : - otype_(otype), oid_({lo, hi}), oclass_(oclass), wasGenerated_(false) {} +DaosOID::DaosOID(const uint32_t& hi, const uint64_t& lo, const enum daos_otype_t& otype, const daos_oclass_id_t& oclass) + : otype_(otype) + , oid_({lo, hi}) + , oclass_(oclass) + , wasGenerated_(false) {} -DaosOID::DaosOID(const std::string& name, const enum daos_otype_t& otype, const daos_oclass_id_t& oclass) : - otype_(otype), oclass_(oclass), wasGenerated_(false) { +DaosOID::DaosOID(const std::string& name, const enum daos_otype_t& otype, const daos_oclass_id_t& oclass) + : otype_(otype) + , oclass_(oclass) + , wasGenerated_(false) { eckit::MD5 md5(name); /// @todo: calculate digests of 12 bytes (24 hex characters) rather than 16 bytes (32 hex characters) /// I have tried redefining MD5_DIGEST_LENGTH but it had no effect oid_.hi = std::stoull(md5.digest().substr(0, 8), nullptr, 16); oid_.lo = std::stoull(md5.digest().substr(8, 16), nullptr, 16); - } bool DaosOID::operator==(const DaosOID& rhs) const { - return oid_.hi == rhs.oid_.hi && oid_.lo == rhs.oid_.lo && - oclass_ == rhs.oclass_ && wasGenerated_ == rhs.wasGenerated_; - + return oid_.hi == rhs.oid_.hi && oid_.lo == rhs.oid_.lo && oclass_ == rhs.oclass_ && + wasGenerated_ == rhs.wasGenerated_; } void DaosOID::generateReservedBits(fdb5::DaosContainer& cont) { - if (wasGenerated_) return; - + if (wasGenerated_) + return; + DAOS_CALL(daos_obj_generate_oid(cont.getOpenHandle(), &oid_, otype(), oclass(), 0, 0)); wasGenerated_ = true; return; - } std::string DaosOID::asString() const { - if (as_string_.has_value()) return as_string_.value(); + if (as_string_.has_value()) + return as_string_.value(); ASSERT(wasGenerated_); @@ -93,57 +94,65 @@ std::string DaosOID::asString() const { os << std::setw(16) << std::setfill('0') << std::hex << oid_.lo; as_string_.emplace(os.str()); return as_string_.value(); - } daos_obj_id_t& DaosOID::asDaosObjIdT() { return oid_; - } enum daos_otype_t DaosOID::otype() const { return otype_; - } daos_oclass_id_t DaosOID::oclass() const { return oclass_; - } -DaosArrayOID::DaosArrayOID(const uint64_t& hi, const uint64_t& lo) : DaosOID(hi, lo) { ASSERT(otype_ == DAOS_OT_ARRAY); } +DaosArrayOID::DaosArrayOID(const uint64_t& hi, const uint64_t& lo) : DaosOID(hi, lo) { + ASSERT(otype_ == DAOS_OT_ARRAY); +} -DaosArrayOID::DaosArrayOID(const std::string& oid) : DaosOID(oid) { ASSERT(otype_ == DAOS_OT_ARRAY); } +DaosArrayOID::DaosArrayOID(const std::string& oid) : DaosOID(oid) { + ASSERT(otype_ == DAOS_OT_ARRAY); +} -DaosArrayOID::DaosArrayOID(const uint32_t& hi, const uint64_t& lo, const daos_oclass_id_t& oclass) : - DaosOID(hi, lo, DAOS_OT_ARRAY, oclass) {} +DaosArrayOID::DaosArrayOID(const uint32_t& hi, const uint64_t& lo, const daos_oclass_id_t& oclass) + : DaosOID(hi, lo, DAOS_OT_ARRAY, oclass) {} -DaosArrayOID::DaosArrayOID(const std::string& name, const daos_oclass_id_t& oclass) : - DaosOID(name, DAOS_OT_ARRAY, oclass) {} +DaosArrayOID::DaosArrayOID(const std::string& name, const daos_oclass_id_t& oclass) + : DaosOID(name, DAOS_OT_ARRAY, oclass) {} -DaosByteArrayOID::DaosByteArrayOID(const uint64_t& hi, const uint64_t& lo) : DaosOID(hi, lo) { ASSERT(otype_ == DAOS_OT_ARRAY_BYTE); } +DaosByteArrayOID::DaosByteArrayOID(const uint64_t& hi, const uint64_t& lo) : DaosOID(hi, lo) { + ASSERT(otype_ == DAOS_OT_ARRAY_BYTE); +} -DaosByteArrayOID::DaosByteArrayOID(const std::string& oid) : DaosOID(oid) { ASSERT(otype_ == DAOS_OT_ARRAY_BYTE); } +DaosByteArrayOID::DaosByteArrayOID(const std::string& oid) : DaosOID(oid) { + ASSERT(otype_ == DAOS_OT_ARRAY_BYTE); +} -DaosByteArrayOID::DaosByteArrayOID(const uint32_t& hi, const uint64_t& lo, const daos_oclass_id_t& oclass) : - DaosOID(hi, lo, DAOS_OT_ARRAY_BYTE, oclass) {} +DaosByteArrayOID::DaosByteArrayOID(const uint32_t& hi, const uint64_t& lo, const daos_oclass_id_t& oclass) + : DaosOID(hi, lo, DAOS_OT_ARRAY_BYTE, oclass) {} -DaosByteArrayOID::DaosByteArrayOID(const std::string& name, const daos_oclass_id_t& oclass) : - DaosOID(name, DAOS_OT_ARRAY_BYTE, oclass) {} +DaosByteArrayOID::DaosByteArrayOID(const std::string& name, const daos_oclass_id_t& oclass) + : DaosOID(name, DAOS_OT_ARRAY_BYTE, oclass) {} -DaosKeyValueOID::DaosKeyValueOID(const uint64_t& hi, const uint64_t& lo) : DaosOID(hi, lo) { ASSERT(otype_ == DAOS_OT_KV_HASHED); } +DaosKeyValueOID::DaosKeyValueOID(const uint64_t& hi, const uint64_t& lo) : DaosOID(hi, lo) { + ASSERT(otype_ == DAOS_OT_KV_HASHED); +} -DaosKeyValueOID::DaosKeyValueOID(const std::string& oid) : DaosOID(oid) { ASSERT(otype_ == DAOS_OT_KV_HASHED); } +DaosKeyValueOID::DaosKeyValueOID(const std::string& oid) : DaosOID(oid) { + ASSERT(otype_ == DAOS_OT_KV_HASHED); +} -DaosKeyValueOID::DaosKeyValueOID(const uint32_t& hi, const uint64_t& lo, const daos_oclass_id_t& oclass) : - DaosOID(hi, lo, DAOS_OT_KV_HASHED, oclass) {} +DaosKeyValueOID::DaosKeyValueOID(const uint32_t& hi, const uint64_t& lo, const daos_oclass_id_t& oclass) + : DaosOID(hi, lo, DAOS_OT_KV_HASHED, oclass) {} -DaosKeyValueOID::DaosKeyValueOID(const std::string& name, const daos_oclass_id_t& oclass) : - DaosOID(name, DAOS_OT_KV_HASHED, oclass) {} +DaosKeyValueOID::DaosKeyValueOID(const std::string& name, const daos_oclass_id_t& oclass) + : DaosOID(name, DAOS_OT_KV_HASHED, oclass) {} //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosOID.h b/src/fdb5/daos/DaosOID.h index e63559022..11c5dfc04 100644 --- a/src/fdb5/daos/DaosOID.h +++ b/src/fdb5/daos/DaosOID.h @@ -26,7 +26,6 @@ class DaosContainer; class DaosOID { public: // methods - DaosOID(const uint64_t& hi, const uint64_t& lo); DaosOID(const std::string&); DaosOID(const uint32_t& hi, const uint64_t& lo, const enum daos_otype_t& otype, const daos_oclass_id_t& oclass); @@ -43,55 +42,45 @@ class DaosOID { bool wasGenerated() const { return wasGenerated_; }; private: // methods - void parseReservedBits(); protected: // members - enum daos_otype_t otype_; private: // members - daos_obj_id_t oid_; daos_oclass_id_t oclass_; bool wasGenerated_; mutable eckit::Optional as_string_; - }; class DaosArrayOID : public DaosOID { -public: //methods - +public: // methods DaosArrayOID(const uint64_t& hi, const uint64_t& lo); DaosArrayOID(const std::string&); DaosArrayOID(const uint32_t& hi, const uint64_t& lo, const daos_oclass_id_t& oclass); DaosArrayOID(const std::string&, const daos_oclass_id_t& oclass); - }; class DaosByteArrayOID : public DaosOID { -public: //methods - +public: // methods DaosByteArrayOID(const uint64_t& hi, const uint64_t& lo); DaosByteArrayOID(const std::string&); DaosByteArrayOID(const uint32_t& hi, const uint64_t& lo, const daos_oclass_id_t& oclass); DaosByteArrayOID(const std::string&, const daos_oclass_id_t& oclass); - }; class DaosKeyValueOID : public DaosOID { -public: //methods - +public: // methods DaosKeyValueOID(const uint64_t& hi, const uint64_t& lo); DaosKeyValueOID(const std::string&); DaosKeyValueOID(const uint32_t& hi, const uint64_t& lo, const daos_oclass_id_t& oclass); DaosKeyValueOID(const std::string&, const daos_oclass_id_t& oclass); - }; //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 \ No newline at end of file +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/daos/DaosObject.cc b/src/fdb5/daos/DaosObject.cc index 2e94085f5..14440b2f7 100644 --- a/src/fdb5/daos/DaosObject.cc +++ b/src/fdb5/daos/DaosObject.cc @@ -14,11 +14,11 @@ #include "eckit/io/Buffer.h" #include "fdb5/LibFdb5.h" -#include "fdb5/daos/DaosSession.h" #include "fdb5/daos/DaosContainer.h" -#include "fdb5/daos/DaosObject.h" -#include "fdb5/daos/DaosName.h" #include "fdb5/daos/DaosException.h" +#include "fdb5/daos/DaosName.h" +#include "fdb5/daos/DaosObject.h" +#include "fdb5/daos/DaosSession.h" namespace fdb5 { @@ -27,38 +27,35 @@ namespace fdb5 { DaosObject::DaosObject(fdb5::DaosContainer& cont, const fdb5::DaosOID& oid) : cont_(cont), oid_(oid), open_(false) { oid_.generateReservedBits(cont); - } -DaosObject::DaosObject(DaosObject&& other) noexcept : cont_(other.cont_), oid_(std::move(other.oid_)), - oh_(std::move(other.oh_)), open_(other.open_) { +DaosObject::DaosObject(DaosObject&& other) noexcept + : cont_(other.cont_) + , oid_(std::move(other.oid_)) + , oh_(std::move(other.oh_)) + , open_(other.open_) { other.open_ = false; - } std::string DaosObject::name() const { return oid_.asString(); - } const fdb5::DaosOID& DaosObject::OID() const { return oid_; - } eckit::URI DaosObject::URI() const { return fdb5::DaosName(*this).URI(); - } fdb5::DaosContainer& DaosObject::getContainer() const { return cont_; - } fdb5::DaosContainer& name_to_cont_ref(fdb5::DaosSession& session, const fdb5::DaosNameBase& name) { @@ -73,7 +70,6 @@ fdb5::DaosContainer& name_to_cont_ref(fdb5::DaosSession& session, const fdb5::Da } return pool->getContainer(name.containerName()); - } DaosArray::DaosArray(DaosArray&& other) noexcept : DaosObject::DaosObject(std::move(other)) {} @@ -83,27 +79,24 @@ DaosArray::DaosArray(fdb5::DaosContainer& cont, const fdb5::DaosOID& oid, bool v ASSERT(oid_.otype() == DAOS_OT_ARRAY || oid_.otype() == DAOS_OT_ARRAY_BYTE); if (verify && !exists()) { - throw fdb5::DaosEntityNotFoundException( - "Array with oid " + oid.asString() + " not found", - Here()); + throw fdb5::DaosEntityNotFoundException("Array with oid " + oid.asString() + " not found", Here()); } - } DaosArray::DaosArray(fdb5::DaosContainer& cont, const fdb5::DaosOID& oid) : DaosArray(cont, oid, true) {} -DaosArray::DaosArray(fdb5::DaosSession& session, const fdb5::DaosNameBase& name) : - DaosArray(name_to_cont_ref(session, name), name.OID()) {} +DaosArray::DaosArray(fdb5::DaosSession& session, const fdb5::DaosNameBase& name) + : DaosArray(name_to_cont_ref(session, name), name.OID()) {} DaosArray::DaosArray(fdb5::DaosSession& session, const eckit::URI& uri) : DaosArray(session, DaosName(uri)) {} DaosArray::~DaosArray() { - if (open_) close(); - + if (open_) + close(); } -/// @note: non-existing arrays (DAOS_OT_ARRAY) and byte-arrays +/// @note: non-existing arrays (DAOS_OT_ARRAY) and byte-arrays /// (DAOS_OT_ARRAY_BYTE) are reported as existing with the current approach. /// @todo: implement proper exist check for DAOS_OT_ARRAY_BYTE. bool DaosArray::exists() { @@ -112,82 +105,65 @@ bool DaosArray::exists() { open(); - } catch ( fdb5::DaosEntityNotFoundException& e) { - - return false; + } catch (fdb5::DaosEntityNotFoundException& e) { return false; } - } - return true; - } void DaosArray::create() { - if (open_) throw eckit::SeriousBug("Attempted create() on an open DaosArray"); + if (open_) + throw eckit::SeriousBug("Attempted create() on an open DaosArray"); - if (oid_.otype() == DAOS_OT_ARRAY_BYTE) return open(); + if (oid_.otype() == DAOS_OT_ARRAY_BYTE) + return open(); const daos_handle_t& coh = cont_.getOpenHandle(); - DAOS_CALL( - daos_array_create( - coh, oid_.asDaosObjIdT(), DAOS_TX_NONE, - fdb5::DaosSession().objectCreateCellSize(), - fdb5::DaosSession().objectCreateChunkSize(), - &oh_, NULL - ) - ); + DAOS_CALL(daos_array_create(coh, oid_.asDaosObjIdT(), DAOS_TX_NONE, fdb5::DaosSession().objectCreateCellSize(), + fdb5::DaosSession().objectCreateChunkSize(), &oh_, NULL)); open_ = true; - } void DaosArray::destroy() { - open(); + open(); DAOS_CALL(daos_array_destroy(oh_, DAOS_TX_NONE, NULL)); close(); - } void DaosArray::open() { - if (open_) return; + if (open_) + return; const daos_handle_t& coh = cont_.getOpenHandle(); if (oid_.otype() == DAOS_OT_ARRAY_BYTE) { /// @note: when using daos_array_open_with_attr to "create" an array, - /// cell and chunk size must be specified. When opening it for + /// cell and chunk size must be specified. When opening it for /// subsequent reads, the same values have to be provided for consistent /// access to the written data. - /// In principle these values should be constant, so + /// In principle these values should be constant, so /// inconsistencies should never happen. However if these values are /// going to be reconfigured, they should be recorded in the database /// on creation, and read back on database access. - DAOS_CALL( - daos_array_open_with_attr( - coh, oid_.asDaosObjIdT(), DAOS_TX_NONE, DAOS_OO_RW, - fdb5::DaosSession().objectCreateCellSize(), - fdb5::DaosSession().objectCreateChunkSize(), - &oh_, NULL - ) - ); + DAOS_CALL(daos_array_open_with_attr(coh, oid_.asDaosObjIdT(), DAOS_TX_NONE, DAOS_OO_RW, + fdb5::DaosSession().objectCreateCellSize(), + fdb5::DaosSession().objectCreateChunkSize(), &oh_, NULL)); } else { daos_size_t cell_size, csize; DAOS_CALL(daos_array_open(coh, oid_.asDaosObjIdT(), DAOS_TX_NONE, DAOS_OO_RW, &cell_size, &csize, &oh_, NULL)); - } open_ = true; - } void DaosArray::close() { @@ -196,22 +172,21 @@ void DaosArray::close() { eckit::Log::warning() << "Closing DaosArray " << name() << ", object is not open" << std::endl; return; } - + LOG_DEBUG_LIB(LibFdb5) << "DAOS_CALL => daos_array_close()" << std::endl; int code = daos_array_close(oh_, NULL); - if (code < 0) eckit::Log::warning() << "DAOS error in call to daos_array_close(), file " - << __FILE__ << ", line " << __LINE__ << ", function " << __func__ << " [" << code << "] (" - << code << ")" << std::endl; - + if (code < 0) + eckit::Log::warning() << "DAOS error in call to daos_array_close(), file " << __FILE__ << ", line " << __LINE__ + << ", function " << __func__ << " [" << code << "] (" << code << ")" << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "DAOS_CALL <= daos_array_close()" << std::endl; open_ = false; - } -/// @note: a buffer (buf) and its length (len) must be provided. A full write of +/// @note: a buffer (buf) and its length (len) must be provided. A full write of /// the buffer into the DAOS array will be attempted. /// @note: daos_array_write fails if len to write is too large. /// DaosArray::write therefore always returns a value equal to the provided @@ -227,32 +202,31 @@ uint64_t DaosArray::write(const void* buf, const uint64_t& len, const eckit::Off d_iov_t iov; iod.arr_nr = 1; - rg.rg_len = (daos_size_t) len; - rg.rg_idx = (daos_off_t) off; + rg.rg_len = (daos_size_t)len; + rg.rg_idx = (daos_off_t)off; iod.arr_rgs = &rg; sgl.sg_nr = 1; - d_iov_set(&iov, (void*) buf, (size_t) len); + d_iov_set(&iov, (void*)buf, (size_t)len); sgl.sg_iovs = &iov; DAOS_CALL(daos_array_write(oh_, DAOS_TX_NONE, &iod, &sgl, NULL)); return len; - } -/// @note: a buffer (buf) and its length (len) must be provided. A read from the +/// @note: a buffer (buf) and its length (len) must be provided. A read from the /// DAOS array of the full buffer length will be attempted. /// @note: daos_array_read does not fail if requested len is larger than object, /// and does not report back the actual amount of bytes read. The user must -/// call DaosArray::size() (i.e. daos_array_get_size) first and request +/// call DaosArray::size() (i.e. daos_array_get_size) first and request /// read of a correspondingly sized buffer. /// @todo: implement the note above in dummy DAOS. /// @note: therefore, this method won't be compatible with the DataHandle::read -/// interface (which is expected to return actual read bytes) unless the +/// interface (which is expected to return actual read bytes) unless the /// implementation of such method checks the actual size and returns it /// if smaller than the provided buffer - which would entail an additional -/// RPC and reduce performance. If the read interface is not properly +/// RPC and reduce performance. If the read interface is not properly /// implemented, at least DataHandle::copyTo breaks. /// @todo: fix this issue by using the "short_read" feature in daos_array_read /// and returning actual read size, rather han having DataHandle::read @@ -270,11 +244,11 @@ uint64_t DaosArray::read(void* buf, uint64_t len, const eckit::Offset& off) { iod.arr_nr = 1; rg.rg_len = len; - rg.rg_idx = (daos_off_t) off; + rg.rg_idx = (daos_off_t)off; iod.arr_rgs = &rg; sgl.sg_nr = 1; - d_iov_set(&iov, buf, (size_t) len); + d_iov_set(&iov, buf, (size_t)len); sgl.sg_iovs = &iov; iod.arr_nr_short_read = 1; @@ -282,7 +256,6 @@ uint64_t DaosArray::read(void* buf, uint64_t len, const eckit::Offset& off) { DAOS_CALL(daos_array_read(oh_, DAOS_TX_NONE, &iod, &sgl, NULL)); return len; - } uint64_t DaosArray::size() { @@ -294,7 +267,6 @@ uint64_t DaosArray::size() { DAOS_CALL(daos_array_get_size(oh_, DAOS_TX_NONE, &array_size, NULL)); return array_size; - } DaosKeyValue::DaosKeyValue(DaosKeyValue&& other) noexcept : DaosObject::DaosObject(std::move(other)) {} @@ -304,62 +276,57 @@ DaosKeyValue::DaosKeyValue(fdb5::DaosContainer& cont, const fdb5::DaosOID& oid, ASSERT(oid_.otype() == type()); if (verify && !exists()) { - throw fdb5::DaosEntityNotFoundException( - "KeyValue with oid " + oid.asString() + " not found", - Here()); + throw fdb5::DaosEntityNotFoundException("KeyValue with oid " + oid.asString() + " not found", Here()); } - } DaosKeyValue::DaosKeyValue(fdb5::DaosContainer& cont, const fdb5::DaosOID& oid) : DaosKeyValue(cont, oid, true) {} -DaosKeyValue::DaosKeyValue(fdb5::DaosSession& session, const fdb5::DaosNameBase& name) : - DaosKeyValue(name_to_cont_ref(session, name), name.OID()) {} +DaosKeyValue::DaosKeyValue(fdb5::DaosSession& session, const fdb5::DaosNameBase& name) + : DaosKeyValue(name_to_cont_ref(session, name), name.OID()) {} DaosKeyValue::DaosKeyValue(fdb5::DaosSession& session, const eckit::URI& uri) : DaosKeyValue(session, DaosName(uri)) {} DaosKeyValue::~DaosKeyValue() { - if (open_) close(); - + if (open_) + close(); } bool DaosKeyValue::exists() { - open(); /// @note: creates it if not exists - - return true; + open(); /// @note: creates it if not exists + return true; } void DaosKeyValue::create() { - if (open_) throw eckit::SeriousBug("Attempted create() on an open DaosKeyValue"); + if (open_) + throw eckit::SeriousBug("Attempted create() on an open DaosKeyValue"); const daos_handle_t& coh = cont_.getOpenHandle(); DAOS_CALL(daos_kv_open(coh, oid_.asDaosObjIdT(), DAOS_OO_RW, &oh_, NULL)); open_ = true; - } void DaosKeyValue::destroy() { - open(); + open(); DAOS_CALL(daos_kv_destroy(oh_, DAOS_TX_NONE, NULL)); close(); - } void DaosKeyValue::open() { - if (open_) return; + if (open_) + return; create(); - } void DaosKeyValue::close() { @@ -373,14 +340,13 @@ void DaosKeyValue::close() { int code = daos_obj_close(oh_, NULL); - if (code < 0) eckit::Log::warning() << "DAOS error in call to daos_obj_close(), file " - << __FILE__ << ", line " << __LINE__ << ", function " << __func__ << " [" << code << "] (" - << code << ")" << std::endl; - + if (code < 0) + eckit::Log::warning() << "DAOS error in call to daos_obj_close(), file " << __FILE__ << ", line " << __LINE__ + << ", function " << __func__ << " [" << code << "] (" << code << ")" << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "DAOS_CALL <= daos_obj_close()" << std::endl; open_ = false; - } uint64_t DaosKeyValue::size(const std::string& key) { @@ -392,13 +358,11 @@ uint64_t DaosKeyValue::size(const std::string& key) { DAOS_CALL(daos_kv_get(oh_, DAOS_TX_NONE, 0, key.c_str(), &res, nullptr, NULL)); return res; - } bool DaosKeyValue::has(const std::string& key) { return size(key) != 0; - } /// @note: daos_kv_put fails if written len is smaller than requested len @@ -409,7 +373,6 @@ uint64_t DaosKeyValue::put(const std::string& key, const void* buf, const uint64 DAOS_CALL(daos_kv_put(oh_, DAOS_TX_NONE, 0, key.c_str(), len, buf, NULL)); return len; - } /// @note: daos_kv_get fails if requested value does not fit in buffer @@ -421,10 +384,10 @@ uint64_t DaosKeyValue::get(const std::string& key, void* buf, const uint64_t& le DAOS_CALL(daos_kv_get(oh_, DAOS_TX_NONE, 0, key.c_str(), &res, buf, NULL)); - if (res == 0) throw fdb5::DaosEntityNotFoundException("Key '" + key + "' not found in KeyValue with OID " + oid_.asString()); + if (res == 0) + throw fdb5::DaosEntityNotFoundException("Key '" + key + "' not found in KeyValue with OID " + oid_.asString()); return res; - } void DaosKeyValue::remove(const std::string& key) { @@ -432,19 +395,18 @@ void DaosKeyValue::remove(const std::string& key) { open(); DAOS_CALL(daos_kv_remove(oh_, DAOS_TX_NONE, 0, key.c_str(), NULL)); - } std::vector DaosKeyValue::keys() { /// @todo: proper memory management - int max_keys_per_rpc = 1024; /// @todo: take from config + int max_keys_per_rpc = 1024; /// @todo: take from config daos_key_desc_t key_sizes[max_keys_per_rpc]; d_sg_list_t sgl; d_iov_t sg_iov; size_t bufsize = 1024; eckit::Buffer list_buf{bufsize}; - d_iov_set(&sg_iov, (char*) list_buf, bufsize); + d_iov_set(&sg_iov, (char*)list_buf, bufsize); sgl.sg_nr = 1; sgl.sg_nr_out = 0; sgl.sg_iovs = &sg_iov; @@ -455,30 +417,30 @@ std::vector DaosKeyValue::keys() { uint32_t nkeys_found = max_keys_per_rpc; int rc; list_buf.zero(); - + DAOS_CALL(daos_kv_list(oh_, DAOS_TX_NONE, &nkeys_found, key_sizes, &sgl, &listing_status, NULL)); size_t key_start = 0; for (int i = 0; i < nkeys_found; i++) { - listed_keys.push_back(std::string((char*) list_buf + key_start, key_sizes[i].kd_key_len)); + listed_keys.push_back(std::string((char*)list_buf + key_start, key_sizes[i].kd_key_len)); key_start += key_sizes[i].kd_key_len; } } return listed_keys; - } -eckit::MemoryStream DaosKeyValue::getMemoryStream(std::vector& v, const std::string& key, const std::string& kvTitle) { +eckit::MemoryStream DaosKeyValue::getMemoryStream(std::vector& v, const std::string& key, + const std::string& kvTitle) { uint64_t sz = size(key); - if (sz == 0) throw fdb5::DaosEntityNotFoundException(std::string("Key '") + key + "' not found in " + kvTitle); + if (sz == 0) + throw fdb5::DaosEntityNotFoundException(std::string("Key '") + key + "' not found in " + kvTitle); v.resize(sz); get(key, &v[0], sz); return eckit::MemoryStream(&v[0], sz); - } //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosObject.h b/src/fdb5/daos/DaosObject.h index b623788a9..4221b8493 100644 --- a/src/fdb5/daos/DaosObject.h +++ b/src/fdb5/daos/DaosObject.h @@ -17,10 +17,10 @@ #include -#include "eckit/utils/Optional.h" -#include "eckit/filesystem/URI.h" #include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" #include "eckit/serialisation/MemoryStream.h" +#include "eckit/utils/Optional.h" #include "fdb5/daos/DaosOID.h" @@ -37,9 +37,8 @@ class DaosSession; class DaosObject { friend DaosContainer; - -public: // methods +public: // methods DaosObject(DaosObject&&) noexcept; DaosObject(const DaosObject&) = default; @@ -59,20 +58,16 @@ class DaosObject { fdb5::DaosContainer& getContainer() const; protected: // methods - DaosObject(fdb5::DaosContainer&, const fdb5::DaosOID&); private: // methods - virtual void create() = 0; protected: // members - fdb5::DaosContainer& cont_; fdb5::DaosOID oid_; daos_handle_t oh_; bool open_; - }; class DaosArray : public DaosObject { @@ -80,7 +75,6 @@ class DaosArray : public DaosObject { friend DaosContainer; public: // methods - DaosArray(DaosArray&&) noexcept; DaosArray(fdb5::DaosContainer&, const fdb5::DaosOID&); @@ -100,11 +94,9 @@ class DaosArray : public DaosObject { uint64_t read(void*, uint64_t, const eckit::Offset&); private: // methods - DaosArray(fdb5::DaosContainer&, const fdb5::DaosOID&, bool verify); void create() override; - }; class DaosKeyValue : public DaosObject { @@ -112,7 +104,6 @@ class DaosKeyValue : public DaosObject { friend DaosContainer; public: // methods - DaosKeyValue(DaosKeyValue&&) noexcept; DaosKeyValue(fdb5::DaosContainer&, const fdb5::DaosOID&); @@ -136,16 +127,15 @@ class DaosKeyValue : public DaosObject { std::vector keys(); /// @note: expects empty vector - eckit::MemoryStream getMemoryStream(std::vector& v, const std::string& key, const std::string& kvTitle = "kv"); + eckit::MemoryStream getMemoryStream(std::vector& v, const std::string& key, + const std::string& kvTitle = "kv"); private: // methods - DaosKeyValue(fdb5::DaosContainer&, const fdb5::DaosOID&, bool verify); void create() override; - }; //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosPool.cc b/src/fdb5/daos/DaosPool.cc index 05d5d0f5a..64da7f106 100644 --- a/src/fdb5/daos/DaosPool.cc +++ b/src/fdb5/daos/DaosPool.cc @@ -11,9 +11,9 @@ #include "eckit/exception/Exceptions.h" #include "fdb5/LibFdb5.h" +#include "fdb5/daos/DaosException.h" #include "fdb5/daos/DaosPool.h" #include "fdb5/daos/DaosSession.h" -#include "fdb5/daos/DaosException.h" #ifdef fdb5_HAVE_DAOS_ADMIN extern "C" { @@ -25,13 +25,15 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -DaosPool::DaosPool(DaosPool&& other) noexcept : uuid_(std::move(other.uuid_)), - known_uuid_(other.known_uuid_), label_(std::move(other.label_)), - poh_(std::move(other.poh_)), open_(other.open_), - cont_cache_(std::move(other.cont_cache_)) { +DaosPool::DaosPool(DaosPool&& other) noexcept + : uuid_(std::move(other.uuid_)) + , known_uuid_(other.known_uuid_) + , label_(std::move(other.label_)) + , poh_(std::move(other.poh_)) + , open_(other.open_) + , cont_cache_(std::move(other.cont_cache_)) { other.open_ = false; - } DaosPool::DaosPool() : known_uuid_(false), open_(false) {} @@ -40,16 +42,18 @@ DaosPool::DaosPool(const fdb5::UUID& uuid) : uuid_(uuid), known_uuid_(true), ope DaosPool::DaosPool(const std::string& label) : known_uuid_(false), label_(label), open_(false) {} - -DaosPool::DaosPool(const fdb5::UUID& uuid, const std::string& label) : - uuid_(uuid), known_uuid_(true), label_(label), open_(false) {} +DaosPool::DaosPool(const fdb5::UUID& uuid, const std::string& label) + : uuid_(uuid) + , known_uuid_(true) + , label_(label) + , open_(false) {} DaosPool::~DaosPool() { cont_cache_.clear(); - if (open_) close(); - + if (open_) + close(); } DaosPool& DaosPool::operator=(DaosPool&& other) noexcept { @@ -64,7 +68,6 @@ DaosPool& DaosPool::operator=(DaosPool&& other) noexcept { other.open_ = false; return *this; - } #ifdef fdb5_HAVE_DAOS_ADMIN @@ -80,7 +83,7 @@ void DaosPool::create(const uint64_t& scmSize, const uint64_t& nvmeSize) { D_ALLOC_ARRAY(svcl.rl_ranks, svcl.rl_nr); ASSERT(svcl.rl_ranks); - daos_prop_t *prop = NULL; + daos_prop_t* prop = NULL; if (label_.size() > 0) { @@ -88,32 +91,27 @@ void DaosPool::create(const uint64_t& scmSize, const uint64_t& nvmeSize) { // Throw exception if that fails. prop = daos_prop_alloc(1); prop->dpp_entries[0].dpe_type = DAOS_PROP_PO_LABEL; - D_STRNDUP(prop->dpp_entries[0].dpe_str, label_.c_str(), DAOS_PROP_LABEL_MAX_LEN); - + D_STRNDUP(prop->dpp_entries[0].dpe_str, label_.c_str(), DAOS_PROP_LABEL_MAX_LEN); } - DAOS_CALL( - dmg_pool_create( - fdb5::DaosSession().dmgConfigFile().c_str(), geteuid(), getegid(), NULL, NULL, - scmSize, nvmeSize, - prop, &svcl, uuid_.internal - ) - ); + DAOS_CALL(dmg_pool_create(fdb5::DaosSession().dmgConfigFile().c_str(), geteuid(), getegid(), NULL, NULL, scmSize, + nvmeSize, prop, &svcl, uuid_.internal)); /// @todo: query the pool to ensure it's ready - if (prop != NULL) daos_prop_free(prop); + if (prop != NULL) + daos_prop_free(prop); D_FREE(svcl.rl_ranks); known_uuid_ = true; - } #endif void DaosPool::open() { - if (open_) return; + if (open_) + return; ASSERT(known_uuid_ || label_.size() > 0); @@ -124,9 +122,8 @@ void DaosPool::open() { uuid_unparse(uuid_.internal, uuid_cstr); DAOS_CALL(daos_pool_connect(uuid_cstr, NULL, DAOS_PC_RW, &poh_, NULL, NULL)); } - - open_ = true; + open_ = true; } void DaosPool::close() { @@ -142,44 +139,40 @@ void DaosPool::close() { int code = daos_pool_disconnect(poh_, NULL); - if (code < 0) eckit::Log::warning() << "DAOS error in call to daos_pool_disconnect(), file " - << __FILE__ << ", line " << __LINE__ << ", function " << __func__ << " [" << code << "] (" - << code << ")" << std::endl; - + if (code < 0) + eckit::Log::warning() << "DAOS error in call to daos_pool_disconnect(), file " << __FILE__ << ", line " + << __LINE__ << ", function " << __func__ << " [" << code << "] (" << code << ")" + << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "DAOS_CALL <= daos_pool_disconnect()" << std::endl; open_ = false; - } std::map::iterator DaosPool::getCachedContainer(const std::string& label) { return cont_cache_.find(label); - } fdb5::DaosContainer& DaosPool::getContainer(const std::string& label, bool verify) { std::map::iterator it = getCachedContainer(label); - if (it != cont_cache_.end()) return it->second; - + if (it != cont_cache_.end()) + return it->second; + fdb5::DaosContainer c{*this, label}; if (verify && !c.exists()) { - throw fdb5::DaosEntityNotFoundException( - "Container with label " + label + " not found", - Here()); + throw fdb5::DaosEntityNotFoundException("Container with label " + label + " not found", Here()); } return cont_cache_.emplace(label, std::move(c)).first->second; - } fdb5::DaosContainer& DaosPool::getContainer(const std::string& label) { return getContainer(label, true); - } fdb5::DaosContainer& DaosPool::createContainer(const std::string& label) { @@ -191,7 +184,6 @@ fdb5::DaosContainer& DaosPool::createContainer(const std::string& label) { c.create(); return c; - } fdb5::DaosContainer& DaosPool::ensureContainer(const std::string& label) { @@ -200,12 +192,12 @@ fdb5::DaosContainer& DaosPool::ensureContainer(const std::string& label) { /// @todo: if the container is found in the cache, it can be assumed /// it exists and this check could be skipped - if (c.exists()) return c; + if (c.exists()) + return c; c.create(); return c; - } void DaosPool::destroyContainer(const std::string& label) { @@ -213,7 +205,7 @@ void DaosPool::destroyContainer(const std::string& label) { ASSERT(label.size() > 0); /// @todo: consider cases where DaosContainer instances may be cached - /// which only have a uuid and no label, but correspond to the same + /// which only have a uuid and no label, but correspond to the same /// container being destroyed here bool found = false; @@ -226,25 +218,21 @@ void DaosPool::destroyContainer(const std::string& label) { } else { - throw fdb5::DaosEntityNotFoundException( - "Container with label " + label + " not found", - Here()); - + throw fdb5::DaosEntityNotFoundException("Container with label " + label + " not found", Here()); } DAOS_CALL(daos_cont_destroy(poh_, label.c_str(), 1, NULL)); /// @todo: whenever a container is destroyed and the user owns open objects, their handles may not /// be possible to close anymore, and any actions on such objects will fail - } /// @note: indended for DaosPool::close() void DaosPool::closeContainers() { std::map::iterator it; - for (it = cont_cache_.begin(); it != cont_cache_.end(); ++it) it->second.close(); - + for (it = cont_cache_.begin(); it != cont_cache_.end(); ++it) + it->second.close(); } std::vector DaosPool::listContainers() { @@ -254,22 +242,22 @@ std::vector DaosPool::listContainers() { daos_size_t ncont{0}; DAOS_CALL(daos_pool_list_cont(getOpenHandle(), &ncont, NULL, NULL)); - if (ncont == 0) return res; + if (ncont == 0) + return res; std::vector info(ncont); DAOS_CALL(daos_pool_list_cont(getOpenHandle(), &ncont, info.data(), NULL)); - for (const auto& c : info) res.push_back(c.pci_label); + for (const auto& c : info) + res.push_back(c.pci_label); return res; - } const daos_handle_t& DaosPool::getOpenHandle() { - + open(); return poh_; - } bool DaosPool::exists() { @@ -279,40 +267,33 @@ bool DaosPool::exists() { open(); - } catch (eckit::SeriousBug& e) { - - return false; - - } + } catch (eckit::SeriousBug& e) { return false; } return true; - } std::string DaosPool::name() const { - /// @note: cannot generate a name for an unidentified pool. + /// @note: cannot generate a name for an unidentified pool. /// Either create it or provide a label upon construction. ASSERT(label_.size() > 0 || known_uuid_); - if (label_.size() > 0) return label_; + if (label_.size() > 0) + return label_; char name_cstr[37]; uuid_unparse(uuid_.internal, name_cstr); return std::string(name_cstr); - } const fdb5::UUID& DaosPool::uuid() const { return uuid_; - } std::string DaosPool::label() const { return label_; - } //---------------------------------------------------------------------------------------------------------------------- @@ -333,10 +314,9 @@ AutoPoolDestroy::~AutoPoolDestroy() noexcept(false) { eckit::Log::error() << "** An exception is already in progress" << std::endl; eckit::Log::error() << "** Exception is ignored" << std::endl; } - } #endif //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosPool.h b/src/fdb5/daos/DaosPool.h index f9fb24008..fec97e0b6 100644 --- a/src/fdb5/daos/DaosPool.h +++ b/src/fdb5/daos/DaosPool.h @@ -15,11 +15,11 @@ #include -#include #include +#include -#include "fdb5/daos/UUID.h" #include "fdb5/daos/DaosContainer.h" +#include "fdb5/daos/UUID.h" #include "fdb5/fdb5_config.h" @@ -34,7 +34,6 @@ class DaosSession; class DaosPool { public: // methods - DaosPool(DaosPool&&) noexcept; ~DaosPool(); @@ -54,13 +53,12 @@ class DaosPool { std::vector listContainers(); const daos_handle_t& getOpenHandle(); - + std::string name() const; const fdb5::UUID& uuid() const; std::string label() const; private: // methods - friend class DaosSession; DaosPool(); @@ -74,14 +72,13 @@ class DaosPool { fdb5::DaosContainer& getContainer(const std::string&, bool); - void closeContainers(); + void closeContainers(); bool exists(); ContainerCache::iterator getCachedContainer(const std::string&); private: // members - fdb5::UUID uuid_; bool known_uuid_; std::string label_ = std::string(); @@ -89,25 +86,21 @@ class DaosPool { bool open_; ContainerCache cont_cache_; - }; #ifdef fdb5_HAVE_DAOS_ADMIN class AutoPoolDestroy { public: // methods - AutoPoolDestroy(fdb5::DaosPool& pool) : pool_(pool) {} ~AutoPoolDestroy() noexcept(false); private: // members - fdb5::DaosPool& pool_; - }; #endif //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosSession.cc b/src/fdb5/daos/DaosSession.cc index 98221bcc9..1acca043c 100644 --- a/src/fdb5/daos/DaosSession.cc +++ b/src/fdb5/daos/DaosSession.cc @@ -13,8 +13,8 @@ #include "eckit/runtime/Main.h" #include "eckit/utils/Translator.h" -#include "fdb5/daos/DaosSession.h" #include "fdb5/daos/DaosException.h" +#include "fdb5/daos/DaosSession.h" #ifdef fdb5_HAVE_DAOS_ADMIN extern "C" { @@ -26,21 +26,15 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -DaosManager::DaosManager() : - containerOidsPerAlloc_(100), - objectCreateCellSize_(1), - objectCreateChunkSize_(1048576) { +DaosManager::DaosManager() : containerOidsPerAlloc_(100), objectCreateCellSize_(1), objectCreateChunkSize_(1048576) { #ifdef fdb5_HAVE_DAOS_ADMIN - dmgConfigFile_ = eckit::Resource( - "fdbDaosDmgConfigFile;$FDB_DAOS_DMG_CONFIG_FILE", dmgConfigFile_ - ); + dmgConfigFile_ = eckit::Resource("fdbDaosDmgConfigFile;$FDB_DAOS_DMG_CONFIG_FILE", dmgConfigFile_); #endif - /// @note: daos_init can be called multiple times. An internal reference + /// @note: daos_init can be called multiple times. An internal reference /// count is maintained by the library DAOS_CALL(daos_init()); - } DaosManager::~DaosManager() { @@ -50,13 +44,12 @@ DaosManager::~DaosManager() { LOG_DEBUG_LIB(LibFdb5) << "DAOS_CALL => daos_fini()" << std::endl; int code = daos_fini(); - - if (code < 0) eckit::Log::warning() << "DAOS error in call to daos_fini(), file " - << __FILE__ << ", line " << __LINE__ << ", function " << __func__ << " [" << code << "] (" - << code << ")" << std::endl; - LOG_DEBUG_LIB(LibFdb5) << "DAOS_CALL <= daos_fini()" << std::endl; + if (code < 0) + eckit::Log::warning() << "DAOS error in call to daos_fini(), file " << __FILE__ << ", line " << __LINE__ + << ", function " << __func__ << " [" << code << "] (" << code << ")" << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "DAOS_CALL <= daos_fini()" << std::endl; } void DaosManager::error(int code, const char* msg, const char* file, int line, const char* func) { @@ -65,7 +58,6 @@ void DaosManager::error(int code, const char* msg, const char* file, int line, c oss << "DAOS error " << msg << ", file " << file << ", line " << line << ", function " << func << " [" << code << "] (" << code << ")"; throw eckit::SeriousBug(oss.str()); - } void DaosManager::configure(const eckit::LocalConfiguration& config) { @@ -78,13 +70,12 @@ void DaosManager::configure(const eckit::LocalConfiguration& config) { #ifdef fdb5_HAVE_DAOS_ADMIN dmgConfigFile_ = config.getString("dmg_config_file", dmgConfigFile_); #endif - }; //---------------------------------------------------------------------------------------------------------------------- -DaosSession::DaosSession() : - // mutex_(DaosManager::instance().mutex_), +DaosSession::DaosSession() + : // mutex_(DaosManager::instance().mutex_), pool_cache_(DaosManager::instance().pool_cache_) {} /// @todo: make getCachedPool return a *DaosPool rather than an iterator? @@ -94,12 +85,11 @@ std::deque::iterator DaosSession::getCachedPool(const fdb5::UUID std::deque::iterator it; for (it = pool_cache_.begin(); it != pool_cache_.end(); ++it) { - if (uuid_compare(uuid.internal, it->uuid().internal) == 0) break; - + if (uuid_compare(uuid.internal, it->uuid().internal) == 0) + break; } return it; - } std::deque::iterator DaosSession::getCachedPool(const std::string& label) { @@ -107,12 +97,11 @@ std::deque::iterator DaosSession::getCachedPool(const std::strin std::deque::iterator it; for (it = pool_cache_.begin(); it != pool_cache_.end(); ++it) { - if (it->label() == label) break; - + if (it->label() == label) + break; } return it; - } #ifdef fdb5_HAVE_DAOS_ADMIN @@ -125,19 +114,17 @@ fdb5::DaosPool& DaosSession::createPool(const uint64_t& scmSize, const uint64_t& p.create(scmSize, nvmeSize); return p; - } fdb5::DaosPool& DaosSession::createPool(const std::string& label, const uint64_t& scmSize, const uint64_t& nvmeSize) { pool_cache_.push_front(fdb5::DaosPool(label)); - + fdb5::DaosPool& p = pool_cache_.at(0); - + p.create(scmSize, nvmeSize); return p; - } #endif @@ -145,7 +132,8 @@ fdb5::DaosPool& DaosSession::getPool(const fdb5::UUID& uuid) { std::deque::iterator it = getCachedPool(uuid); - if (it != pool_cache_.end()) return *it; + if (it != pool_cache_.end()) + return *it; fdb5::DaosPool p(uuid); @@ -153,62 +141,58 @@ fdb5::DaosPool& DaosSession::getPool(const fdb5::UUID& uuid) { char uuid_cstr[37]; uuid_unparse(uuid.internal, uuid_cstr); std::string uuid_str(uuid_cstr); - throw fdb5::DaosEntityNotFoundException( - "Pool with uuid " + uuid_str + " not found", - Here()); + throw fdb5::DaosEntityNotFoundException("Pool with uuid " + uuid_str + " not found", Here()); } pool_cache_.push_front(std::move(p)); return pool_cache_.at(0); - } fdb5::DaosPool& DaosSession::getPool(const std::string& label) { std::deque::iterator it = getCachedPool(label); - if (it != pool_cache_.end()) return *it; + if (it != pool_cache_.end()) + return *it; fdb5::DaosPool p(label); if (!p.exists()) { - throw fdb5::DaosEntityNotFoundException( - "Pool with label " + label + " not found", - Here()); + throw fdb5::DaosEntityNotFoundException("Pool with label " + label + " not found", Here()); } pool_cache_.push_front(std::move(p)); return pool_cache_.at(0); - } DaosPool& DaosSession::getPool(const fdb5::UUID& uuid, const std::string& label) { - /// @note: When both pool uuid and label are known, using this method - /// to declare a pool is preferred to avoid the following + /// @note: When both pool uuid and label are known, using this method + /// to declare a pool is preferred to avoid the following /// inconsistencies and/or inefficiencies: - /// - when a user declares a pool by label in a process where that pool - /// has not been created, that pool will live in the cache with only a - /// label and no uuid. If the user later declares the same pool from its - /// uuid, two DaosPool instances will exist in the cache for the same + /// - when a user declares a pool by label in a process where that pool + /// has not been created, that pool will live in the cache with only a + /// label and no uuid. If the user later declares the same pool from its + /// uuid, two DaosPool instances will exist in the cache for the same /// DAOS pool, each with their connection handle. - /// - these two instances will be incomplete and the user may not be able + /// - these two instances will be incomplete and the user may not be able /// to retrieve the label/uuid information. std::deque::iterator it = getCachedPool(uuid); if (it != pool_cache_.end()) { - if (it->label() == label) return *it; + if (it->label() == label) + return *it; pool_cache_.push_front(fdb5::DaosPool(uuid, label)); return pool_cache_.at(0); - } it = getCachedPool(label); - if (it != pool_cache_.end()) return *it; + if (it != pool_cache_.end()) + return *it; fdb5::DaosPool p(uuid, label); @@ -216,15 +200,13 @@ DaosPool& DaosSession::getPool(const fdb5::UUID& uuid, const std::string& label) char uuid_cstr[37]; uuid_unparse(uuid.internal, uuid_cstr); std::string uuid_str(uuid_cstr); - throw fdb5::DaosEntityNotFoundException( - "Pool with uuid " + uuid_str + " or label " + label + " not found", - Here()); + throw fdb5::DaosEntityNotFoundException("Pool with uuid " + uuid_str + " or label " + label + " not found", + Here()); } pool_cache_.push_front(std::move(p)); return pool_cache_.at(0); - } #ifdef fdb5_HAVE_DAOS_ADMIN @@ -245,7 +227,6 @@ void DaosSession::destroyPool(const fdb5::UUID& uuid, const int& force) { } else { ++it; } - } if (!found) { @@ -253,20 +234,16 @@ void DaosSession::destroyPool(const fdb5::UUID& uuid, const int& force) { char uuid_cstr[37]; uuid_unparse(uuid.internal, uuid_cstr); std::string uuid_str(uuid_cstr); - throw fdb5::DaosEntityNotFoundException( - "Pool with uuid " + uuid_str + " not found", - Here()); - + throw fdb5::DaosEntityNotFoundException("Pool with uuid " + uuid_str + " not found", Here()); } DAOS_CALL(dmg_pool_destroy(dmgConfigFile().c_str(), uuid.internal, NULL, force)); /// @todo: cached DaosPools declared with a label only, pointing to the pool // being destroyed may still exist and should be closed and removed - } #endif //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosSession.h b/src/fdb5/daos/DaosSession.h index f8f5ae1f1..07e0a40d6 100644 --- a/src/fdb5/daos/DaosSession.h +++ b/src/fdb5/daos/DaosSession.h @@ -15,18 +15,18 @@ #include -#include #include #include +#include -#include "eckit/exception/Exceptions.h" #include "eckit/config/LocalConfiguration.h" #include "eckit/config/Resource.h" +#include "eckit/exception/Exceptions.h" #include "eckit/log/Timer.h" #include "fdb5/LibFdb5.h" -#include "fdb5/daos/DaosPool.h" #include "fdb5/daos/DaosException.h" +#include "fdb5/daos/DaosPool.h" #include "fdb5/fdb5_config.h" @@ -42,7 +42,6 @@ using PoolCache = std::deque; class DaosManager : private eckit::NonCopyable { public: // methods - static DaosManager& instance() { static DaosManager instance; return instance; @@ -53,45 +52,42 @@ class DaosManager : private eckit::NonCopyable { void configure(const eckit::LocalConfiguration&); private: // methods - DaosManager(); ~DaosManager(); private: // members - friend class DaosSession; std::mutex mutex_; PoolCache pool_cache_; /// @note: sets number of OIDs allocated in a single daos_cont_alloc_oids call - int containerOidsPerAlloc_; + int containerOidsPerAlloc_; /// @note: number of bytes per cell in a DAOS object /// @note: cell size is the unit of atomicity / update. If the object /// will always be updated or read in elements of e.g. 64k, then 64k /// can be used as the cell size. Then, that 64k element cannot be - /// partially updated anymore. + /// partially updated anymore. uint64_t objectCreateCellSize_; /// @note: number of cells of per dkey in a DAOS object - /// @note: the chunk size maps to how many cells to put under 1 dkey. - /// So it also controls the RPC size. It should not really be something - /// very small otherwise it might create a lot of RPCs. If not + /// @note: the chunk size maps to how many cells to put under 1 dkey. + /// So it also controls the RPC size. It should not really be something + /// very small otherwise it might create a lot of RPCs. If not /// using redundancy (SX), setting it to something as equal or a multiple - /// of the most common transfer size is OK. the default is 1 MiB which is - /// usually OK. If using EC, it gets more tricky as the EC cell size and - /// transfer size come into play and break that even more and can cause - /// overhead on the client side. Ideally, set both to the same as the IO - /// size, but that is not always possible because the EC cell size is - /// only changed per container, vs the chunk and transfer size that can - /// vary per object / file. That may be changed in DAOS to allow more + /// of the most common transfer size is OK. the default is 1 MiB which is + /// usually OK. If using EC, it gets more tricky as the EC cell size and + /// transfer size come into play and break that even more and can cause + /// overhead on the client side. Ideally, set both to the same as the IO + /// size, but that is not always possible because the EC cell size is + /// only changed per container, vs the chunk and transfer size that can + /// vary per object / file. That may be changed in DAOS to allow more /// flexibility uint64_t objectCreateChunkSize_; #ifdef fdb5_HAVE_DAOS_ADMIN std::string dmgConfigFile_; #endif - }; #define DAOS_CALL(a) fdb5::daos_call(a, #a, __FILE__, __LINE__, __func__) @@ -102,8 +98,10 @@ static inline int daos_call(int code, const char* msg, const char* file, int lin if (code < 0) { eckit::Log::error() << "DAOS_FAIL !! " << msg << std::endl; - if (code == -DER_NONEXIST) throw fdb5::DaosEntityNotFoundException(msg); - if (code == -DER_EXIST) throw fdb5::DaosEntityAlreadyExistsException(msg); + if (code == -DER_NONEXIST) + throw fdb5::DaosEntityNotFoundException(msg); + if (code == -DER_EXIST) + throw fdb5::DaosEntityAlreadyExistsException(msg); DaosManager::error(code, msg, file, line, func); } @@ -116,26 +114,21 @@ static inline int daos_call(int code, const char* msg, const char* file, int lin /// @note: DaosSession acts as a mere wrapper for DaosManager such that DaosManager::instance does not need /// to be called in so many places -/// @note: DaosSession no longer performs daos_init on creation and daos_fini on destroy. This is because -/// any pool handles obtained within a session are cached in DaosManager beyond DaosSession lifetime, +/// @note: DaosSession no longer performs daos_init on creation and daos_fini on destroy. This is because +/// any pool handles obtained within a session are cached in DaosManager beyond DaosSession lifetime, /// and the pool handles may become invalid if daos_fini is called for all sessions class DaosSession : eckit::NonCopyable { public: // methods - DaosSession(); ~DaosSession() {}; #ifdef fdb5_HAVE_DAOS_ADMIN // administrative - fdb5::DaosPool& createPool( - const uint64_t& scmSize = 10ULL << 30, - const uint64_t& nvmeSize = 40ULL << 30); - fdb5::DaosPool& createPool( - const std::string& label, - const uint64_t& scmSize = 10ULL << 30, - const uint64_t& nvmeSize = 40ULL << 30); + fdb5::DaosPool& createPool(const uint64_t& scmSize = 10ULL << 30, const uint64_t& nvmeSize = 40ULL << 30); + fdb5::DaosPool& createPool(const std::string& label, const uint64_t& scmSize = 10ULL << 30, + const uint64_t& nvmeSize = 40ULL << 30); void destroyPool(const fdb5::UUID&, const int& force = 1); #endif @@ -152,18 +145,15 @@ class DaosSession : eckit::NonCopyable { #endif private: // methods - PoolCache::iterator getCachedPool(const fdb5::UUID&); PoolCache::iterator getCachedPool(const std::string&); private: // members - /// @todo: add lock_guards in all DaosSession methods using pool_cache_. Same for cont_cache_ in DaosPool. // std::mutex& mutex_; PoolCache& pool_cache_; - }; //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosStore.cc b/src/fdb5/daos/DaosStore.cc index 3536a1b31..03be538fc 100644 --- a/src/fdb5/daos/DaosStore.cc +++ b/src/fdb5/daos/DaosStore.cc @@ -10,36 +10,34 @@ #include "eckit/config/Resource.h" -#include "fdb5/daos/DaosFieldLocation.h" -#include "fdb5/daos/DaosStore.h" #include "fdb5/daos/DaosArrayHandle.h" -#include "fdb5/daos/DaosObject.h" #include "fdb5/daos/DaosContainer.h" +#include "fdb5/daos/DaosException.h" +#include "fdb5/daos/DaosFieldLocation.h" +#include "fdb5/daos/DaosObject.h" #include "fdb5/daos/DaosPool.h" #include "fdb5/daos/DaosSession.h" -#include "fdb5/daos/DaosException.h" +#include "fdb5/daos/DaosStore.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -DaosStore::DaosStore(const Key& key, const Config& config) : - Store(), DaosCommon(config, "store", key), db_str_(db_cont_) {} +DaosStore::DaosStore(const Key& key, const Config& config) + : Store() + , DaosCommon(config, "store", key) + , db_str_(db_cont_) {} eckit::URI DaosStore::uri() const { return fdb5::DaosName(pool_, db_str_).URI(); - } bool DaosStore::uriBelongs(const eckit::URI& uri) const { /// @todo: avoid building a DaosName as it makes uriBelongs expensive /// @todo: assert uri points to a (not necessarily existing) array object - return ( - (uri.scheme() == type()) && - (fdb5::DaosName(uri).containerName().rfind(db_str_, 0) == 0)); - + return ((uri.scheme() == type()) && (fdb5::DaosName(uri).containerName().rfind(db_str_, 0) == 0)); } bool DaosStore::uriExists(const eckit::URI& uri) const { @@ -54,7 +52,6 @@ bool DaosStore::uriExists(const eckit::URI& uri) const { ASSERT(n.hasOID()); return n.exists(); - } std::vector DaosStore::collocatedDataURIs() const { @@ -62,47 +59,44 @@ std::vector DaosStore::collocatedDataURIs() const { std::vector collocated_data_uris; fdb5::DaosName db_cont{pool_, db_str_}; - - if (!db_cont.exists()) return collocated_data_uris; - + + if (!db_cont.exists()) + return collocated_data_uris; + for (const auto& oid : db_cont.listOIDs()) { - + if (oid.otype() != DAOS_OT_KV_HASHED && oid.otype() != DAOS_OT_ARRAY_BYTE) throw eckit::SeriousBug("Found non-KV non-ByteArray objects in DB container " + db_cont.URI().asString()); - - if (oid.otype() == DAOS_OT_KV_HASHED) continue; - collocated_data_uris.push_back(fdb5::DaosArrayName(pool_, db_str_, oid).URI()); + if (oid.otype() == DAOS_OT_KV_HASHED) + continue; + collocated_data_uris.push_back(fdb5::DaosArrayName(pool_, db_str_, oid).URI()); } return collocated_data_uris; - } std::set DaosStore::asCollocatedDataURIs(const std::vector& uris) const { std::set res; - for (auto& uri : uris) + for (auto& uri : uris) /// @note: seems redundant, but intends to check validity of input URIs res.insert(fdb5::DaosName(uri).URI()); return res; - } bool DaosStore::exists() const { return fdb5::DaosName(pool_).exists(); - } /// @todo: never used in actual fdb-read? eckit::DataHandle* DaosStore::retrieve(Field& field) const { return field.dataHandle(); - } std::unique_ptr DaosStore::archive(const Key&, const void* data, eckit::Length length) { @@ -110,10 +104,12 @@ std::unique_ptr DaosStore::archive(const Key&, const void* /// @note: performed RPCs: /// - open pool if not cached (daos_pool_connect) -- always skipped as it is cached after selectDatabase. /// If the cat backend is toc, then it is performed but only on first write. - /// - check if container exists if not cached (daos_cont_open) -- always skipped as it is cached after selectDatabase. + /// - check if container exists if not cached (daos_cont_open) -- always skipped as it is cached after + /// selectDatabase. /// If the cat backend is toc, then it is performed but only on first write. /// - allocate oid (daos_cont_alloc_oids) -- skipped most of the times as oids per alloc is set to 100 - fdb5::DaosArrayName n = fdb5::DaosName(pool_, db_str_).createArrayName(OC_S1, false); // TODO: pass oclass from config + fdb5::DaosArrayName n = + fdb5::DaosName(pool_, db_str_).createArrayName(OC_S1, false); // TODO: pass oclass from config std::unique_ptr h(n.dataHandle()); @@ -149,7 +145,7 @@ size_t DaosStore::flush() { void DaosStore::remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const { fdb5::DaosName n{uri}; - + ASSERT(n.hasContainerName()); ASSERT(n.poolName() == pool_); ASSERT(n.containerName() == db_str_); @@ -162,14 +158,13 @@ void DaosStore::remove(const eckit::URI& uri, std::ostream& logAlways, std::ostr } logAlways << n.asString() << std::endl; - if (doit) n.destroy(); - + if (doit) + n.destroy(); } -void DaosStore::print(std::ostream &out) const { +void DaosStore::print(std::ostream& out) const { out << "DaosStore(" << pool_ << ")"; - } static StoreBuilder builder("daos"); diff --git a/src/fdb5/daos/DaosStore.h b/src/fdb5/daos/DaosStore.h index 151bca309..82853a21d 100644 --- a/src/fdb5/daos/DaosStore.h +++ b/src/fdb5/daos/DaosStore.h @@ -25,7 +25,6 @@ namespace fdb5 { class DaosStore : public Store, public DaosCommon { public: // methods - DaosStore(const Key& key, const Config& config); ~DaosStore() override {} @@ -47,7 +46,6 @@ class DaosStore : public Store, public DaosCommon { bool auxiliaryURIExists(const eckit::URI&) const override { return false; } protected: // methods - std::string type() const override { return "daos"; } bool exists() const override; @@ -57,13 +55,11 @@ class DaosStore : public Store, public DaosCommon { void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override; - void print(std::ostream &out) const override; + void print(std::ostream& out) const override; private: // members - std::string db_str_; size_t archivedFields_; - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/daos/DaosWipeVisitor.cc b/src/fdb5/daos/DaosWipeVisitor.cc index dd7ebbb4b..433f7e373 100644 --- a/src/fdb5/daos/DaosWipeVisitor.cc +++ b/src/fdb5/daos/DaosWipeVisitor.cc @@ -11,10 +11,10 @@ #include "eckit/log/Log.h" #include "eckit/serialisation/MemoryStream.h" -#include "fdb5/database/Store.h" -#include "fdb5/daos/DaosWipeVisitor.h" #include "fdb5/daos/DaosName.h" #include "fdb5/daos/DaosSession.h" +#include "fdb5/daos/DaosWipeVisitor.h" +#include "fdb5/database/Store.h" #include @@ -24,17 +24,13 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -DaosWipeVisitor::DaosWipeVisitor(const DaosCatalogue& catalogue, - const Store& store, - const metkit::mars::MarsRequest& request, - std::ostream& out, - bool doit, - bool porcelain, - bool unsafeWipeAll) : - WipeVisitor(request, out, doit, porcelain, unsafeWipeAll), - catalogue_(catalogue), - store_(store), - dbKvName_("") {} +DaosWipeVisitor::DaosWipeVisitor(const DaosCatalogue& catalogue, const Store& store, + const metkit::mars::MarsRequest& request, std::ostream& out, bool doit, bool porcelain, + bool unsafeWipeAll) + : WipeVisitor(request, out, doit, porcelain, unsafeWipeAll) + , catalogue_(catalogue) + , store_(store) + , dbKvName_("") {} DaosWipeVisitor::~DaosWipeVisitor() {} @@ -79,8 +75,7 @@ bool DaosWipeVisitor::visitIndex(const Index& index) { bool include = index.key().match(indexRequest_); // If we have cross fdb-mounted another DB, ensure we can't delete another DBs data. - if (!(location.containerName() == db_kv.containerName() && - location.poolName() == db_kv.poolName())) { + if (!(location.containerName() == db_kv.containerName() && location.poolName() == db_kv.poolName())) { include = false; } // ASSERT(location.dirName().sameAs(basePath) || !include); @@ -89,7 +84,7 @@ bool DaosWipeVisitor::visitIndex(const Index& index) { std::set axes; fdb5::DaosSession s{}; - fdb5::DaosKeyValue index_kv{s, location}; /// @todo: asserts that kv exists + fdb5::DaosKeyValue index_kv{s, location}; /// @todo: asserts that kv exists uint64_t size = index_kv.size("axes"); if (size > 0) { std::vector axes_data(size); @@ -116,16 +111,18 @@ bool DaosWipeVisitor::visitIndex(const Index& index) { // Enumerate data files. - /// @note: although a daos index will point to only one daos store container if using a daos store, + /// @note: although a daos index will point to only one daos store container if using a daos store, /// a daos index can point to multiple posix store files (one per IO server process) if using a posix store. std::vector indexDataURIs(index.dataURIs()); for (const eckit::URI& uri : store_.asCollocatedDataURIs(indexDataURIs)) { if (include) { if (!store_.uriBelongs(uri)) { - Log::error() << "Index to be deleted has pointers to fields that don't belong to the configured store." << std::endl; + Log::error() << "Index to be deleted has pointers to fields that don't belong to the configured store." + << std::endl; Log::error() << "Configured Store URI: " << store_.uri().asString() << std::endl; Log::error() << "Pointed Store unit URI: " << uri.asString() << std::endl; - Log::error() << "Impossible to delete such fields. Index deletion aborted to avoid leaking fields." << std::endl; + Log::error() << "Impossible to delete such fields. Index deletion aborted to avoid leaking fields." + << std::endl; throw eckit::SeriousBug{"Index deletion aborted to avoid leaking fields."}; } storeURIs_.insert(uri); @@ -135,7 +132,6 @@ bool DaosWipeVisitor::visitIndex(const Index& index) { } return true; // Explore contained entries - } void DaosWipeVisitor::ensureSafeURIs() { @@ -144,7 +140,6 @@ void DaosWipeVisitor::ensureSafeURIs() { for (const auto& p : safeStoreURIs_) storeURIs_.erase(p); - } void DaosWipeVisitor::calculateResidualURIs() { @@ -160,7 +155,8 @@ void DaosWipeVisitor::calculateResidualURIs() { std::set deleteKvNames; deleteKvNames.insert(indexNames_.begin(), indexNames_.end()); deleteKvNames.insert(axisNames_.begin(), axisNames_.end()); - if (dbKvName_.poolName().size()) deleteKvNames.insert(dbKvName_.URI()); + if (dbKvName_.poolName().size()) + deleteKvNames.insert(dbKvName_.URI()); std::set allKvNames; /// @note: given a database container, list DB kv, index kvs and axis kvs @@ -181,8 +177,7 @@ void DaosWipeVisitor::calculateResidualURIs() { // First we check if there are names marked to delete that don't exist. This is an error std::set names; - std::set_difference(deleteKvNames.begin(), deleteKvNames.end(), - allKvNames.begin(), allKvNames.end(), + std::set_difference(deleteKvNames.begin(), deleteKvNames.end(), allKvNames.begin(), allKvNames.end(), std::inserter(names, names.begin())); if (!names.empty()) { @@ -190,25 +185,26 @@ void DaosWipeVisitor::calculateResidualURIs() { for (const auto& n : names) { Log::error() << " - " << n.URI() << std::endl; } - throw SeriousBug("KV names to delete in deleteKvNames should should be in existing name set. Are multiple wipe commands running simultaneously?", Here()); + throw SeriousBug( + "KV names to delete in deleteKvNames should should be in existing name set. Are multiple wipe commands " + "running simultaneously?", + Here()); } - std::set_difference(allKvNames.begin(), allKvNames.end(), - deleteKvNames.begin(), deleteKvNames.end(), + std::set_difference(allKvNames.begin(), allKvNames.end(), deleteKvNames.begin(), deleteKvNames.end(), std::inserter(residualKvNames_, residualKvNames_.begin())); } // repeat the algorithm for store units (store files or containers) for (std::set* uriset : {&storeURIs_}) { - for (std::set::iterator it = uriset->begin(); it != uriset->end(); ) { + for (std::set::iterator it = uriset->begin(); it != uriset->end();) { if (store_.uriExists(*it)) { ++it; } else { uriset->erase(it++); } - } } @@ -228,8 +224,7 @@ void DaosWipeVisitor::calculateResidualURIs() { // First we check if there are URIs marked to delete that don't exist. This is an error std::set uris; - std::set_difference(storeURIs_.begin(), storeURIs_.end(), - allStoreURIs.begin(), allStoreURIs.end(), + std::set_difference(storeURIs_.begin(), storeURIs_.end(), allStoreURIs.begin(), allStoreURIs.end(), std::inserter(uris, uris.begin())); if (!uris.empty()) { @@ -237,15 +232,15 @@ void DaosWipeVisitor::calculateResidualURIs() { for (const auto& u : uris) { Log::error() << " - " << u << std::endl; } - throw SeriousBug("Store unit to delete should be in existing URI set. Are multiple wipe commands running simultaneously?", Here()); + throw SeriousBug( + "Store unit to delete should be in existing URI set. Are multiple wipe commands running " + "simultaneously?", + Here()); } - std::set_difference(allStoreURIs.begin(), allStoreURIs.end(), - storeURIs_.begin(), storeURIs_.end(), + std::set_difference(allStoreURIs.begin(), allStoreURIs.end(), storeURIs_.begin(), storeURIs_.end(), std::inserter(residualStoreURIs_, residualStoreURIs_.begin())); - } - } bool DaosWipeVisitor::anythingToWipe() const { @@ -259,7 +254,7 @@ void DaosWipeVisitor::report(bool wipeAll) { if (wipeAll) { out_ << "DB container to delete:" << std::endl; - const fdb5::DaosKeyValueName& db_kv = catalogue_.dbKeyValue(); + const fdb5::DaosKeyValueName& db_kv = catalogue_.dbKeyValue(); out_ << " " << fdb5::DaosName{db_kv.poolName(), db_kv.containerName()}.URI() << std::endl; out_ << std::endl; @@ -286,30 +281,31 @@ void DaosWipeVisitor::report(bool wipeAll) { out_ << " - NONE -" << std::endl; out_ << std::endl; } - } out_ << "Index KVs to delete: " << std::endl; - if (indexNames_.empty()) out_ << " - NONE -" << std::endl; + if (indexNames_.empty()) + out_ << " - NONE -" << std::endl; for (const auto& n : indexNames_) { out_ << " " << n.URI() << std::endl; } out_ << std::endl; out_ << "Axis KVs to delete: " << std::endl; - if (axisNames_.empty()) out_ << " - NONE -" << std::endl; + if (axisNames_.empty()) + out_ << " - NONE -" << std::endl; for (const auto& n : axisNames_) { out_ << " " << n.URI() << std::endl; } out_ << std::endl; out_ << "Store units (store files or arrays) to delete: " << std::endl; - if (storeURIs_.empty()) out_ << " - NONE -" << std::endl; + if (storeURIs_.empty()) + out_ << " - NONE -" << std::endl; for (const auto& f : storeURIs_) { out_ << " " << f << std::endl; } out_ << std::endl; - } void DaosWipeVisitor::wipe(bool wipeAll) { @@ -357,7 +353,8 @@ void DaosWipeVisitor::wipe(bool wipeAll) { catalogue_.remove(name, logAlways, logVerbose, doit_); fdb5::DaosKeyValue db_kv{s, catalogue_.dbKeyValue()}; - if (doit_) db_kv.remove(idx); + if (doit_) + db_kv.remove(idx); } else { NOTIMP; @@ -370,7 +367,7 @@ void DaosWipeVisitor::wipe(bool wipeAll) { if (wipeAll) { if (store_.type() != "daos") - /// @todo: if the store is holding catalogue information (e.g. index files) it + /// @todo: if the store is holding catalogue information (e.g. index files) it /// should not be removed store_.remove(store_.uri(), logAlways, logVerbose, doit_); @@ -379,8 +376,9 @@ void DaosWipeVisitor::wipe(bool wipeAll) { fdb5::DaosName db_cont{db_kv.poolName(), db_kv.containerName()}; - if (db_cont.exists() && doit_) db_cont.destroy(); - + if (db_cont.exists() && doit_) + db_cont.destroy(); + std::string db_key = db_kv.containerName(); fdb5::DaosSession s{}; fdb5::DaosKeyValue root{s, root_kv}; @@ -388,10 +386,8 @@ void DaosWipeVisitor::wipe(bool wipeAll) { root.remove(db_key); } } - } - void DaosWipeVisitor::catalogueComplete(const Catalogue& catalogue) { WipeVisitor::catalogueComplete(catalogue); @@ -412,24 +408,26 @@ void DaosWipeVisitor::catalogueComplete(const Catalogue& catalogue) { if (anythingToWipe()) { - if (wipeAll) calculateResidualURIs(); + if (wipeAll) + calculateResidualURIs(); - if (!porcelain_) report(wipeAll); + if (!porcelain_) + report(wipeAll); // This is here as it needs to run whatever combination of doit/porcelain/... if (wipeAll && !residualKvNames_.empty()) { out_ << "Unexpected KVs present in DB container: " << std::endl; - for (const auto& n : residualKvNames_) out_ << " " << n.URI() << std::endl; + for (const auto& n : residualKvNames_) + out_ << " " << n.URI() << std::endl; out_ << std::endl; - } if (wipeAll && !residualStoreURIs_.empty()) { out_ << "Unexpected store units (store files or arrays) present in store: " << std::endl; - for (const auto& u : residualStoreURIs_) out_ << " " << store_.type() << "://" << u << std::endl; + for (const auto& u : residualStoreURIs_) + out_ << " " << store_.type() << "://" << u << std::endl; out_ << std::endl; - } if (wipeAll && (!residualKvNames_.empty() || !residualStoreURIs_.empty())) { if (!unsafeWipeAll_) { @@ -439,11 +437,11 @@ void DaosWipeVisitor::catalogueComplete(const Catalogue& catalogue) { } } - if (doit_ || porcelain_) wipe(wipeAll); + if (doit_ || porcelain_) + wipe(wipeAll); } } - //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 diff --git a/src/fdb5/daos/DaosWipeVisitor.h b/src/fdb5/daos/DaosWipeVisitor.h index d5ebebf63..f68171ac2 100644 --- a/src/fdb5/daos/DaosWipeVisitor.h +++ b/src/fdb5/daos/DaosWipeVisitor.h @@ -13,8 +13,8 @@ #pragma once -#include "fdb5/database/WipeVisitor.h" #include "fdb5/daos/DaosCatalogue.h" +#include "fdb5/database/WipeVisitor.h" namespace fdb5 { @@ -23,18 +23,11 @@ namespace fdb5 { class DaosWipeVisitor : public WipeVisitor { public: - - DaosWipeVisitor(const DaosCatalogue& catalogue, - const Store& store, - const metkit::mars::MarsRequest& request, - std::ostream& out, - bool doit, - bool porcelain, - bool unsafeWipeAll); + DaosWipeVisitor(const DaosCatalogue& catalogue, const Store& store, const metkit::mars::MarsRequest& request, + std::ostream& out, bool doit, bool porcelain, bool unsafeWipeAll); ~DaosWipeVisitor() override; private: // methods - bool visitDatabase(const Catalogue& catalogue) override; bool visitIndex(const Index& index) override; void catalogueComplete(const Catalogue& catalogue) override; @@ -48,7 +41,6 @@ class DaosWipeVisitor : public WipeVisitor { void wipe(bool wipeAll); private: // members - // What are the parameters of the wipe operation const DaosCatalogue& catalogue_; const Store& store_; @@ -63,10 +55,9 @@ class DaosWipeVisitor : public WipeVisitor { std::set storeURIs_; std::set safeStoreURIs_; - + std::set residualKvNames_; std::set residualStoreURIs_; - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/daos/UUID.cc b/src/fdb5/daos/UUID.cc index bf72f4307..5ac228135 100644 --- a/src/fdb5/daos/UUID.cc +++ b/src/fdb5/daos/UUID.cc @@ -20,9 +20,8 @@ UUID::UUID(const std::string& uuid) { if (uuid_parse(uuid.c_str(), internal) != 0) throw eckit::BadParameter("The provided string is not a uuid."); - } //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/daos/UUID.h b/src/fdb5/daos/UUID.h index 0e30f5d42..50a25f390 100644 --- a/src/fdb5/daos/UUID.h +++ b/src/fdb5/daos/UUID.h @@ -24,17 +24,14 @@ namespace fdb5 { class UUID { public: // methods - UUID() : internal{0} {} UUID(const std::string&); public: // members - uuid_t internal; - }; //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 \ No newline at end of file +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/database/ArchiveVisitor.cc b/src/fdb5/database/ArchiveVisitor.cc index 338897359..81749e3ae 100644 --- a/src/fdb5/database/ArchiveVisitor.cc +++ b/src/fdb5/database/ArchiveVisitor.cc @@ -7,23 +7,25 @@ * granted to it by virtue of its status as an intergovernmental organisation nor * does it submit to any jurisdiction. */ -#include +#include "fdb5/database/ArchiveVisitor.h" #include "eckit/exception/Exceptions.h" #include "fdb5/database/Archiver.h" -#include "fdb5/database/ArchiveVisitor.h" #include "fdb5/database/Catalogue.h" #include "fdb5/database/Store.h" +#include namespace fdb5 { -ArchiveVisitor::ArchiveVisitor(Archiver& owner, const Key& initialFieldKey, const void *data, size_t size, const ArchiveCallback& callback) : - BaseArchiveVisitor(owner, initialFieldKey), - data_(data), - size_(size), - callback_(callback){ -} +ArchiveVisitor::ArchiveVisitor(Archiver& owner, const Key& initialFieldKey, const void* data, size_t size, + const ArchiveCallback& callback) + : BaseArchiveVisitor(owner, initialFieldKey) + , data_(data) + , size_(size) + , callback_(callback) {} -void ArchiveVisitor::callbacks(fdb5::CatalogueWriter* catalogue, const Key& idxKey, const Key& datumKey, std::shared_ptr>> p, std::shared_ptr fieldLocation) { +void ArchiveVisitor::callbacks(fdb5::CatalogueWriter* catalogue, const Key& idxKey, const Key& datumKey, + std::shared_ptr>> p, + std::shared_ptr fieldLocation) { p->set_value(fieldLocation); catalogue->archive(idxKey, datumKey, std::move(fieldLocation)); } @@ -33,19 +35,21 @@ bool ArchiveVisitor::selectDatum(const TypedKey& datumKey, const TypedKey& fullC checkMissingKeys(fullComputedKey); const Key idxKey = catalogue()->currentIndexKey(); - std::shared_ptr>> p = std::make_shared>>(std::promise>()); + std::shared_ptr>> p = + std::make_shared>>( + std::promise>()); store()->archive(idxKey, data_, size_, - std::bind(&ArchiveVisitor::callbacks, this, catalogue(), idxKey, datumKey.canonical(), p, std::placeholders::_1)); + std::bind(&ArchiveVisitor::callbacks, this, catalogue(), idxKey, datumKey.canonical(), p, + std::placeholders::_1)); callback_(initialFieldKey(), data_, size_, p->get_future()); return true; } -void ArchiveVisitor::print(std::ostream &out) const { +void ArchiveVisitor::print(std::ostream& out) const { out << "ArchiveVisitor[" - << "size=" << size_ - << "]"; + << "size=" << size_ << "]"; } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/ArchiveVisitor.h b/src/fdb5/database/ArchiveVisitor.h index 7f2d9c142..4301e6d1d 100644 --- a/src/fdb5/database/ArchiveVisitor.h +++ b/src/fdb5/database/ArchiveVisitor.h @@ -19,7 +19,9 @@ #include "fdb5/api/helpers/Callback.h" #include "fdb5/database/BaseArchiveVisitor.h" -namespace metkit { class MarsRequest; } +namespace metkit { +class MarsRequest; +} namespace fdb5 { @@ -30,26 +32,24 @@ class Archiver; class ArchiveVisitor : public BaseArchiveVisitor { public: // methods - - ArchiveVisitor(Archiver& owner, const Key& dataKey, const void* data, size_t size, const ArchiveCallback& callback = CALLBACK_ARCHIVE_NOOP); + ArchiveVisitor(Archiver& owner, const Key& dataKey, const void* data, size_t size, + const ArchiveCallback& callback = CALLBACK_ARCHIVE_NOOP); protected: // methods - bool selectDatum(const TypedKey& datumKey, const TypedKey& fullComputedKey) override; - void print( std::ostream &out ) const override; + void print(std::ostream& out) const override; private: // methods - - void callbacks(fdb5::CatalogueWriter* catalogue, const Key& idxKey, const Key& datumKey, std::shared_ptr>> p, std::shared_ptr fieldLocation); + void callbacks(fdb5::CatalogueWriter* catalogue, const Key& idxKey, const Key& datumKey, + std::shared_ptr>> p, + std::shared_ptr fieldLocation); private: // members - - const void *data_; + const void* data_; size_t size_; const ArchiveCallback& callback_; - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/Archiver.cc b/src/fdb5/database/Archiver.cc index 3b1840546..d4832067f 100644 --- a/src/fdb5/database/Archiver.cc +++ b/src/fdb5/database/Archiver.cc @@ -8,7 +8,6 @@ * does it submit to any jurisdiction. */ - #include "fdb5/database/Archiver.h" #include @@ -18,18 +17,18 @@ #include "fdb5/LibFdb5.h" #include "fdb5/database/ArchiveVisitor.h" #include "fdb5/database/BaseArchiveVisitor.h" -#include "fdb5/rules/Schema.h" -#include "fdb5/rules/Rule.h" #include "fdb5/database/Store.h" +#include "fdb5/rules/Rule.h" +#include "fdb5/rules/Schema.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -Archiver::Archiver(const Config& dbConfig, const ArchiveCallback& callback) : - dbConfig_(dbConfig), - db_(nullptr), - callback_(callback) {} +Archiver::Archiver(const Config& dbConfig, const ArchiveCallback& callback) + : dbConfig_(dbConfig) + , db_(nullptr) + , callback_(callback) {} Archiver::~Archiver() { flush(); // certify that all sessions are flushed before closing them @@ -44,7 +43,7 @@ void Archiver::archive(const Key& key, BaseArchiveVisitor& visitor) { std::lock_guard lock(flushMutex_); visitor.rule(nullptr); - + dbConfig_.schema().expand(key, visitor); const Rule* rule = visitor.rule(); @@ -69,7 +68,7 @@ void Archiver::selectDatabase(const Key& dbKey) { auto i = databases_.find(dbKey); - if (i != databases_.end() ) { + if (i != databases_.end()) { db_ = &(i->second); i->second.time_ = ::time(0); return; @@ -95,7 +94,7 @@ void Archiver::selectDatabase(const Key& dbKey) { std::lock_guard lock(flushMutex_); databases_[oldK].catalogue_->flush(databases_[oldK].store_->flush()); - + eckit::Log::info() << "Closing database " << *databases_[oldK].catalogue_ << std::endl; databases_.erase(oldK); } @@ -118,8 +117,7 @@ void Archiver::selectDatabase(const Key& dbKey) { void Archiver::print(std::ostream& out) const { out << "Archiver[" - << "]" - << std::endl; + << "]" << std::endl; } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/Archiver.h b/src/fdb5/database/Archiver.h index ac6826b8e..2611b5cdf 100644 --- a/src/fdb5/database/Archiver.h +++ b/src/fdb5/database/Archiver.h @@ -24,7 +24,7 @@ #include "fdb5/config/Config.h" #include "fdb5/database/Catalogue.h" -namespace eckit { +namespace eckit { class DataHandle; } @@ -39,14 +39,13 @@ class Schema; //---------------------------------------------------------------------------------------------------------------------- struct Database { - time_t time_; - std::unique_ptr catalogue_; - std::unique_ptr store_; + time_t time_; + std::unique_ptr catalogue_; + std::unique_ptr store_; }; class Archiver : public eckit::NonCopyable { public: // methods - Archiver(const Config& dbConfig = Config().expandConfig(), const ArchiveCallback& callback = CALLBACK_ARCHIVE_NOOP); virtual ~Archiver(); @@ -64,7 +63,6 @@ class Archiver : public eckit::NonCopyable { } private: // methods - void print(std::ostream& out) const; void selectDatabase(const Key& key); diff --git a/src/fdb5/database/AxisRegistry.cc b/src/fdb5/database/AxisRegistry.cc index c25e965a1..c3f2447fa 100644 --- a/src/fdb5/database/AxisRegistry.cc +++ b/src/fdb5/database/AxisRegistry.cc @@ -10,8 +10,8 @@ #include -#include "eckit/thread/AutoLock.h" #include "eckit/log/Log.h" +#include "eckit/thread/AutoLock.h" #include "fdb5/LibFdb5.h" #include "fdb5/database/AxisRegistry.h" @@ -47,19 +47,17 @@ void AxisRegistry::deduplicate(const keyword_t& keyword, std::shared_ptr eckit::AutoLock lock(mutex_); -// static std::size_t dedups = 0; + // static std::size_t dedups = 0; axis_store_t& axis = axes_[keyword]; axis_store_t::iterator it = axis.find(ptr); if (it == axis.end()) { axis.insert(ptr); - } - else { -// dedups++; -// LOG_DEBUG_LIB(LibFdb5) << dedups << " deduped axis [" << *ptr << "]" << std::endl; + } else { + // dedups++; + // LOG_DEBUG_LIB(LibFdb5) << dedups << " deduped axis [" << *ptr << "]" << std::endl; ptr = *it; } } -} - +} // namespace fdb5 diff --git a/src/fdb5/database/AxisRegistry.h b/src/fdb5/database/AxisRegistry.h index 722b2b66a..ad3f10ae1 100644 --- a/src/fdb5/database/AxisRegistry.h +++ b/src/fdb5/database/AxisRegistry.h @@ -16,10 +16,10 @@ #ifndef fdb5_AxisRegistry_H #define fdb5_AxisRegistry_H +#include #include #include #include -#include #include "eckit/container/DenseSet.h" #include "eckit/thread/Mutex.h" @@ -30,17 +30,14 @@ namespace fdb5 { class AxisRegistry { public: // types - typedef std::string keyword_t; typedef eckit::DenseSet axis_t; typedef std::shared_ptr ptr_axis_t; - struct HashDenseSet - { - std::size_t operator()(ptr_axis_t const& p) const noexcept - { + struct HashDenseSet { + std::size_t operator()(ptr_axis_t const& p) const noexcept { std::size_t h = 0; - for(const auto& s: *p) { + for (const auto& s : *p) { // this hash combine is inspired in the boost::hash_combine // 0x9e3779b9 is the reciprocal of the golden ratio to ensure random bit distribution h ^= std::hash{}(s) + 0x9e3779b9 + (h << 6) + (h >> 2); @@ -49,34 +46,27 @@ class AxisRegistry { } }; - struct EqualsDenseSet - { - bool operator()(ptr_axis_t const& left, ptr_axis_t const& right) const noexcept { - return *left == *right; - } + struct EqualsDenseSet { + bool operator()(ptr_axis_t const& left, ptr_axis_t const& right) const noexcept { return *left == *right; } }; private: // types - typedef std::string axis_key_t; - typedef std::unordered_set axis_store_t; + typedef std::unordered_set axis_store_t; typedef std::map axis_map_t; public: // methods - static AxisRegistry& instance(); void deduplicate(const keyword_t& key, std::shared_ptr& ptr); void release(const keyword_t& key, std::shared_ptr& ptr); private: // members - axis_map_t axes_; mutable eckit::Mutex mutex_; }; -} +} // namespace fdb5 #endif - diff --git a/src/fdb5/database/BaseArchiveVisitor.cc b/src/fdb5/database/BaseArchiveVisitor.cc index 9be4c2096..d40a0da29 100644 --- a/src/fdb5/database/BaseArchiveVisitor.cc +++ b/src/fdb5/database/BaseArchiveVisitor.cc @@ -13,15 +13,15 @@ #include "fdb5/LibFdb5.h" #include "fdb5/database/Archiver.h" #include "fdb5/database/BaseArchiveVisitor.h" -#include "fdb5/rules/Rule.h" #include "fdb5/database/Store.h" +#include "fdb5/rules/Rule.h" namespace fdb5 { -BaseArchiveVisitor::BaseArchiveVisitor(Archiver &owner, const Key& initialFieldKey) : - WriteVisitor(owner.prev_), - owner_(owner), - initialFieldKey_(initialFieldKey) { +BaseArchiveVisitor::BaseArchiveVisitor(Archiver& owner, const Key& initialFieldKey) + : WriteVisitor(owner.prev_) + , owner_(owner) + , initialFieldKey_(initialFieldKey) { checkMissingKeysOnWrite_ = eckit::Resource("checkMissingKeysOnWrite", true); } diff --git a/src/fdb5/database/BaseArchiveVisitor.h b/src/fdb5/database/BaseArchiveVisitor.h index c313162f2..133ed702a 100644 --- a/src/fdb5/database/BaseArchiveVisitor.h +++ b/src/fdb5/database/BaseArchiveVisitor.h @@ -18,7 +18,9 @@ #include "fdb5/database/WriteVisitor.h" -namespace metkit { class MarsRequest; } +namespace metkit { +class MarsRequest; +} namespace fdb5 { @@ -32,11 +34,9 @@ class Schema; class BaseArchiveVisitor : public WriteVisitor { public: // methods - BaseArchiveVisitor(Archiver& owner, const Key& initialFieldKey); protected: // methods - bool selectDatabase(const Key& dbKey, const TypedKey& fullComputedKey) override; bool selectIndex(const Key& idxKey, const TypedKey& fullComputedKey) override; @@ -48,10 +48,9 @@ class BaseArchiveVisitor : public WriteVisitor { fdb5::CatalogueWriter* catalogue() const; fdb5::Store* store() const; - const Key& initialFieldKey() const { return initialFieldKey_; } + const Key& initialFieldKey() const { return initialFieldKey_; } private: // members - Archiver& owner_; const Key initialFieldKey_; diff --git a/src/fdb5/database/Catalogue.cc b/src/fdb5/database/Catalogue.cc index 573fe6c87..79e48fbd9 100644 --- a/src/fdb5/database/Catalogue.cc +++ b/src/fdb5/database/Catalogue.cc @@ -25,7 +25,7 @@ namespace fdb5 { -std::ostream &operator<<(std::ostream &s, const Catalogue &x) { +std::ostream& operator<<(std::ostream& s, const Catalogue& x) { x.print(s); return s; } @@ -59,7 +59,6 @@ void Catalogue::visitEntries(EntryVisitor& visitor, bool sorted) { } visitor.catalogueComplete(*this); - } const Key CatalogueWriter::currentIndexKey() { @@ -106,7 +105,8 @@ bool CatalogueReaderFactory::has(const std::string& name) { void CatalogueReaderFactory::list(std::ostream& out) { eckit::AutoLock lock(mutex_); const char* sep = ""; - for (std::map::const_iterator j = builders_.begin(); j != builders_.end(); ++j) { + for (std::map::const_iterator j = builders_.begin(); j != builders_.end(); + ++j) { out << sep << (*j).first; sep = ", "; } @@ -159,11 +159,11 @@ CatalogueReaderBuilderBase::CatalogueReaderBuilderBase(const std::string& name) } CatalogueReaderBuilderBase::~CatalogueReaderBuilderBase() { - if(LibFdb5::instance().dontDeregisterFactories()) return; + if (LibFdb5::instance().dontDeregisterFactories()) + return; CatalogueReaderFactory::instance().remove(name_); } - //---------------------------------------------------------------------------------------------------------------------- CatalogueWriterFactory::CatalogueWriterFactory() {} @@ -200,7 +200,8 @@ bool CatalogueWriterFactory::has(const std::string& name) { void CatalogueWriterFactory::list(std::ostream& out) { eckit::AutoLock lock(mutex_); const char* sep = ""; - for (std::map::const_iterator j = builders_.begin(); j != builders_.end(); ++j) { + for (std::map::const_iterator j = builders_.begin(); j != builders_.end(); + ++j) { out << sep << (*j).first; sep = ", "; } @@ -253,10 +254,11 @@ CatalogueWriterBuilderBase::CatalogueWriterBuilderBase(const std::string& name) } CatalogueWriterBuilderBase::~CatalogueWriterBuilderBase() { - if(LibFdb5::instance().dontDeregisterFactories()) return; + if (LibFdb5::instance().dontDeregisterFactories()) + return; CatalogueWriterFactory::instance().remove(name_); } //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/database/Catalogue.h b/src/fdb5/database/Catalogue.h index 9c798035e..8290de49a 100644 --- a/src/fdb5/database/Catalogue.h +++ b/src/fdb5/database/Catalogue.h @@ -25,13 +25,12 @@ #include "fdb5/database/Catalogue.h" #include "fdb5/database/Field.h" #include "fdb5/database/FieldLocation.h" -#include "fdb5/database/Key.h" #include "fdb5/database/Index.h" -#include "fdb5/api/helpers/ControlIterator.h" +#include "fdb5/database/Key.h" +#include "fdb5/database/MoveVisitor.h" #include "fdb5/database/PurgeVisitor.h" #include "fdb5/database/StatsReportVisitor.h" #include "fdb5/database/WipeVisitor.h" -#include "fdb5/database/MoveVisitor.h" #include "fdb5/rules/Schema.h" namespace fdb5 { @@ -42,7 +41,6 @@ typedef std::map IndexStore; class Catalogue { public: - Catalogue() {} virtual ~Catalogue() {} @@ -62,18 +60,21 @@ class Catalogue { virtual void hideContents() = 0; - virtual void dump(std::ostream& out, bool simple=false, const eckit::Configuration& conf = eckit::LocalConfiguration()) const = 0; + virtual void dump(std::ostream& out, bool simple = false, + const eckit::Configuration& conf = eckit::LocalConfiguration()) const = 0; virtual StatsReportVisitor* statsReportVisitor() const = 0; virtual PurgeVisitor* purgeVisitor(const Store& store) const = 0; - virtual WipeVisitor* wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, bool doit, bool porcelain, bool unsafeWipeAll) const = 0; - virtual MoveVisitor* moveVisitor(const Store& store, const metkit::mars::MarsRequest& request, const eckit::URI& dest, eckit::Queue& queue) const = 0; + virtual WipeVisitor* wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, + bool doit, bool porcelain, bool unsafeWipeAll) const = 0; + virtual MoveVisitor* moveVisitor(const Store& store, const metkit::mars::MarsRequest& request, + const eckit::URI& dest, eckit::Queue& queue) const = 0; virtual void control(const ControlAction& action, const ControlIdentifiers& identifiers) const = 0; virtual bool enabled(const ControlIdentifier& controlIdentifier) const = 0; - virtual std::vector indexes(bool sorted=false) const = 0; + virtual std::vector indexes(bool sorted = false) const = 0; /// For use by the WipeVisitor virtual void maskIndexEntry(const Index& index) const = 0; @@ -82,8 +83,8 @@ class Catalogue { virtual void allMasked(std::set>& metadata, std::set& data) const = 0; - friend std::ostream &operator<<(std::ostream &s, const Catalogue &x); - virtual void print( std::ostream &out ) const = 0; + friend std::ostream& operator<<(std::ostream& s, const Catalogue& x); + virtual void print(std::ostream& out) const = 0; virtual std::string type() const = 0; virtual bool open() = 0; @@ -97,16 +98,15 @@ class Catalogue { virtual eckit::URI uri() const = 0; protected: // methods - virtual void loadSchema() = 0; - }; class CatalogueImpl : virtual public Catalogue { public: - CatalogueImpl(const Key& key, ControlIdentifiers controlIdentifiers, const fdb5::Config& config) - : dbKey_(key), config_(config), controlIdentifiers_(controlIdentifiers) {} + : dbKey_(key) + , config_(config) + , controlIdentifiers_(controlIdentifiers) {} virtual ~CatalogueImpl() {} @@ -121,23 +121,19 @@ class CatalogueImpl : virtual public Catalogue { bool enabled(const ControlIdentifier& controlIdentifier) const override; protected: // methods - CatalogueImpl() : dbKey_(Key()), config_(Config()), controlIdentifiers_(ControlIdentifiers()) {} protected: // members - Key dbKey_; Config config_; ControlIdentifiers controlIdentifiers_; - }; class CatalogueReader : virtual public Catalogue { public: - CatalogueReader() {} - + virtual ~CatalogueReader() {} virtual DbStats stats() const = 0; @@ -145,18 +141,18 @@ class CatalogueReader : virtual public Catalogue { virtual bool retrieve(const Key& key, Field& field) const = 0; }; - -class CatalogueWriter : virtual public Catalogue { +class CatalogueWriter : virtual public Catalogue { public: - CatalogueWriter() {} virtual ~CatalogueWriter() {} virtual const Index& currentIndex() = 0; virtual const Key currentIndexKey(); - virtual void archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) = 0; - virtual void overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) = 0; + virtual void archive(const Key& idxKey, const Key& datumKey, + std::shared_ptr fieldLocation) = 0; + virtual void overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, + bool unmount) = 0; virtual void index(const Key& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) = 0; virtual void reconsolidate() = 0; }; @@ -175,8 +171,12 @@ class CatalogueReaderBuilderBase { template class CatalogueReaderBuilder : public CatalogueReaderBuilderBase { - virtual std::unique_ptr make(const fdb5::Key& key, const fdb5::Config& config) override { return std::unique_ptr(new T(key, config)); } - virtual std::unique_ptr make(const eckit::URI& uri, const fdb5::Config& config) override { return std::unique_ptr(new T(uri, config)); } + virtual std::unique_ptr make(const fdb5::Key& key, const fdb5::Config& config) override { + return std::unique_ptr(new T(key, config)); + } + virtual std::unique_ptr make(const eckit::URI& uri, const fdb5::Config& config) override { + return std::unique_ptr(new T(uri, config)); + } public: CatalogueReaderBuilder(const std::string& name) : CatalogueReaderBuilderBase(name) {} @@ -219,8 +219,12 @@ class CatalogueWriterBuilderBase { template class CatalogueWriterBuilder : public CatalogueWriterBuilderBase { - virtual std::unique_ptr make(const fdb5::Key& key, const fdb5::Config& config) override { return std::unique_ptr(new T(key, config)); } - virtual std::unique_ptr make(const eckit::URI& uri, const fdb5::Config& config) override { return std::unique_ptr(new T(uri, config)); } + virtual std::unique_ptr make(const fdb5::Key& key, const fdb5::Config& config) override { + return std::unique_ptr(new T(key, config)); + } + virtual std::unique_ptr make(const eckit::URI& uri, const fdb5::Config& config) override { + return std::unique_ptr(new T(uri, config)); + } public: CatalogueWriterBuilder(const std::string& name) : CatalogueWriterBuilderBase(name) {} @@ -251,7 +255,6 @@ class CatalogueWriterFactory { class NullCatalogue : public Catalogue { public: - const Key& key() const override { NOTIMP; } const Key& indexKey() const override { NOTIMP; } const Config& config() const override { NOTIMP; } @@ -267,27 +270,38 @@ class NullCatalogue : public Catalogue { void hideContents() override { NOTIMP; } - void dump(std::ostream& out, bool simple=false, const eckit::Configuration& conf = eckit::LocalConfiguration()) const override { NOTIMP; } + void dump(std::ostream& out, bool simple = false, + const eckit::Configuration& conf = eckit::LocalConfiguration()) const override { + NOTIMP; + } bool enabled(const ControlIdentifier& controlIdentifier) const override { NOTIMP; } StatsReportVisitor* statsReportVisitor() const override { NOTIMP; } PurgeVisitor* purgeVisitor(const Store& store) const override { NOTIMP; } - WipeVisitor* wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, bool doit, bool porcelain, bool unsafeWipeAll) const override { NOTIMP; } - MoveVisitor* moveVisitor(const Store& store, const metkit::mars::MarsRequest& request, const eckit::URI& dest, eckit::Queue& queue) const override { NOTIMP; } + WipeVisitor* wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, bool doit, + bool porcelain, bool unsafeWipeAll) const override { + NOTIMP; + } + MoveVisitor* moveVisitor(const Store& store, const metkit::mars::MarsRequest& request, const eckit::URI& dest, + eckit::Queue& queue) const override { + NOTIMP; + } void control(const ControlAction& action, const ControlIdentifiers& identifiers) const override { NOTIMP; } - std::vector indexes(bool sorted=false) const override { NOTIMP; } + std::vector indexes(bool sorted = false) const override { NOTIMP; } /// For use by the WipeVisitor void maskIndexEntry(const Index& index) const override { NOTIMP; } /// For use by purge/wipe void allMasked(std::set>& metadata, - std::set& data) const override { NOTIMP; } + std::set& data) const override { + NOTIMP; + } - friend std::ostream &operator<<(std::ostream &s, const Catalogue &x); - void print( std::ostream &out ) const override { NOTIMP; } + friend std::ostream& operator<<(std::ostream& s, const Catalogue& x); + void print(std::ostream& out) const override { NOTIMP; } std::string type() const override { NOTIMP; } bool open() override { NOTIMP; } @@ -301,9 +315,7 @@ class NullCatalogue : public Catalogue { eckit::URI uri() const override { NOTIMP; } protected: // methods - void loadSchema() override { NOTIMP; } - }; -} +} // namespace fdb5 diff --git a/src/fdb5/database/DataStats.cc b/src/fdb5/database/DataStats.cc index c503c8102..f4c488c84 100644 --- a/src/fdb5/database/DataStats.cc +++ b/src/fdb5/database/DataStats.cc @@ -18,30 +18,23 @@ namespace fdb5 { class NullDataStats : public DataStatsContent { public: + virtual void add(const DataStatsContent&) { NOTIMP; } - virtual void add(const DataStatsContent&) { - NOTIMP; - } - - virtual void report(std::ostream& out, const char* indent) const { - NOTIMP; - } + virtual void report(std::ostream& out, const char* indent) const { NOTIMP; } }; //---------------------------------------------------------------------------------------------------------------------- -DataStats::DataStats() : - content_(new NullDataStats()) { +DataStats::DataStats() : content_(new NullDataStats()) { content_->attach(); } -DataStats::DataStats(DataStatsContent* p) : - content_(p) { +DataStats::DataStats(DataStatsContent* p) : content_(p) { content_->attach(); } DataStats::~DataStats() { - content_->detach(); + content_->detach(); } DataStats::DataStats(const DataStats& s) : content_(s.content_) { @@ -55,19 +48,15 @@ DataStats& DataStats::operator=(const DataStats& s) { return *this; } -void DataStats::add(const DataStats& s) -{ +void DataStats::add(const DataStats& s) { content_->add(*s.content_); } -void DataStats::report(std::ostream& out, const char* indent) const -{ +void DataStats::report(std::ostream& out, const char* indent) const { content_->report(out, indent); } -DataStatsContent::~DataStatsContent() -{ -} +DataStatsContent::~DataStatsContent() {} //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/DataStats.h b/src/fdb5/database/DataStats.h index 9c99932c3..67bf9608e 100644 --- a/src/fdb5/database/DataStats.h +++ b/src/fdb5/database/DataStats.h @@ -25,16 +25,13 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -class DataStatsContent : public eckit::Counted, - public eckit::Statistics { +class DataStatsContent : public eckit::Counted, public eckit::Statistics { public: - ~DataStatsContent() override; virtual void add(const DataStatsContent&) = 0; virtual void report(std::ostream& out, const char* indent) const = 0; - }; //---------------------------------------------------------------------------------------------------------------------- @@ -44,7 +41,6 @@ class DataStatsContent : public eckit::Counted, class DataStats { public: // methods - DataStats(); DataStats(DataStatsContent*); @@ -58,21 +54,20 @@ class DataStats { void report(std::ostream& out, const char* indent = "") const; -// template -// T& as() { -// return dynamic_cast(*content_); -// } + // template + // T& as() { + // return dynamic_cast(*content_); + // } private: // methods - void print(std::ostream&) const; friend std::ostream& operator<<(std::ostream& s, const DataStats& o) { - o.print(s); return s; + o.print(s); + return s; } private: // members - DataStatsContent* content_; }; diff --git a/src/fdb5/database/DatabaseNotFoundException.cc b/src/fdb5/database/DatabaseNotFoundException.cc index 6be6a5709..41d579bd0 100644 --- a/src/fdb5/database/DatabaseNotFoundException.cc +++ b/src/fdb5/database/DatabaseNotFoundException.cc @@ -16,9 +16,9 @@ namespace fdb5 { DatabaseNotFoundException::DatabaseNotFoundException(const std::string& w) : Exception(w) {} -DatabaseNotFoundException::DatabaseNotFoundException(const std::string& w, const eckit::CodeLocation& l) : - Exception(w, l) {} +DatabaseNotFoundException::DatabaseNotFoundException(const std::string& w, const eckit::CodeLocation& l) + : Exception(w, l) {} //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/database/DatabaseNotFoundException.h b/src/fdb5/database/DatabaseNotFoundException.h index 66d40b280..ba565dd27 100644 --- a/src/fdb5/database/DatabaseNotFoundException.h +++ b/src/fdb5/database/DatabaseNotFoundException.h @@ -27,4 +27,4 @@ class DatabaseNotFoundException : public eckit::Exception { //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/database/DbStats.cc b/src/fdb5/database/DbStats.cc old mode 100755 new mode 100644 index d1495eb83..fa26f87c3 --- a/src/fdb5/database/DbStats.cc +++ b/src/fdb5/database/DbStats.cc @@ -14,40 +14,34 @@ namespace fdb5 { -::eckit::ClassSpec DbStatsContent::classSpec_ = {&Streamable::classSpec(), "DbStatsContent",}; +::eckit::ClassSpec DbStatsContent::classSpec_ = { + &Streamable::classSpec(), + "DbStatsContent", +}; //---------------------------------------------------------------------------------------------------------------------- class NullDbStats : public DbStatsContent { public: + virtual void add(const DbStatsContent&) { NOTIMP; } - virtual void add(const DbStatsContent&) { - NOTIMP; - } - - virtual void report(std::ostream& out, const char* indent) const { - NOTIMP; - } + virtual void report(std::ostream& out, const char* indent) const { NOTIMP; } - virtual void encode(eckit::Stream& s) const { - NOTIMP; - } + virtual void encode(eckit::Stream& s) const { NOTIMP; } }; //---------------------------------------------------------------------------------------------------------------------- -DbStats::DbStats() : - content_(new NullDbStats()) { +DbStats::DbStats() : content_(new NullDbStats()) { content_->attach(); } -DbStats::DbStats(DbStatsContent* p) : - content_(p) { +DbStats::DbStats(DbStatsContent* p) : content_(p) { content_->attach(); } DbStats::~DbStats() { - content_->detach(); + content_->detach(); } DbStats::DbStats(const DbStats& s) : content_(s.content_) { @@ -66,13 +60,11 @@ DbStats& DbStats::operator+=(const DbStats& s) { return *this; } -void DbStats::add(const DbStats& s) -{ +void DbStats::add(const DbStats& s) { content_->add(*s.content_); } -void DbStats::report(std::ostream& out, const char* indent) const -{ +void DbStats::report(std::ostream& out, const char* indent) const { content_->report(out, indent); } @@ -80,9 +72,7 @@ void DbStats::encode(eckit::Stream& s) const { s << *content_; } -DbStatsContent::~DbStatsContent() -{ -} +DbStatsContent::~DbStatsContent() {} //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/DbStats.h b/src/fdb5/database/DbStats.h old mode 100755 new mode 100644 index 695ca533e..df56d162c --- a/src/fdb5/database/DbStats.h +++ b/src/fdb5/database/DbStats.h @@ -26,11 +26,8 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -class DbStatsContent : public eckit::Counted, - public eckit::Statistics, - public eckit::Streamable { +class DbStatsContent : public eckit::Counted, public eckit::Statistics, public eckit::Streamable { public: - ~DbStatsContent() override; virtual void add(const DbStatsContent&) = 0; @@ -38,12 +35,10 @@ class DbStatsContent : public eckit::Counted, virtual void report(std::ostream& out, const char* indent) const = 0; public: // For Streamable - void encode(eckit::Stream& s) const override = 0; protected: // For Streamable - - static eckit::ClassSpec classSpec_; + static eckit::ClassSpec classSpec_; }; //---------------------------------------------------------------------------------------------------------------------- @@ -53,7 +48,6 @@ class DbStatsContent : public eckit::Counted, class DbStats { public: // methods - DbStats(); DbStats(DbStatsContent*); @@ -63,24 +57,24 @@ class DbStats { DbStats& operator=(const DbStats&); - DbStats& operator+= (const DbStats& rhs); + DbStats& operator+=(const DbStats& rhs); void add(const DbStats&); void report(std::ostream& out, const char* indent = "") const; -// template -// T& as() { -// return dynamic_cast(*content_); -// } + // template + // T& as() { + // return dynamic_cast(*content_); + // } private: // methods - void print(std::ostream&) const; void encode(eckit::Stream& s) const; friend std::ostream& operator<<(std::ostream& s, const DbStats& o) { - o.print(s); return s; + o.print(s); + return s; } friend eckit::Stream& operator<<(eckit::Stream& s, const DbStats& r) { @@ -89,7 +83,6 @@ class DbStats { } private: // members - DbStatsContent* content_; }; diff --git a/src/fdb5/database/Engine.cc b/src/fdb5/database/Engine.cc index 7fd00dbc4..60766a8e3 100644 --- a/src/fdb5/database/Engine.cc +++ b/src/fdb5/database/Engine.cc @@ -8,10 +8,10 @@ * does it submit to any jurisdiction. */ +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/PathName.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/Mutex.h" -#include "eckit/filesystem/PathName.h" -#include "eckit/exception/Exceptions.h" #include "fdb5/database/Engine.h" @@ -22,7 +22,7 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -eckit::Mutex *local_mutex = 0; +eckit::Mutex* local_mutex = 0; pthread_once_t once = PTHREAD_ONCE_INIT; static std::map* m; @@ -32,8 +32,7 @@ void init() { m = new std::map(); } -bool EngineRegistry::has(const std::string& name) -{ +bool EngineRegistry::has(const std::string& name) { pthread_once(&once, init); AutoLock lock(*local_mutex); @@ -46,7 +45,7 @@ Engine& EngineRegistry::engine(const std::string& name) { AutoLock lock(*local_mutex); std::map::iterator i = m->find(name); - if(i != m->end()) { + if (i != m->end()) { return *(i->second); } @@ -56,8 +55,7 @@ Engine& EngineRegistry::engine(const std::string& name) { throw eckit::BadParameter(oss.str(), Here()); } -std::vector EngineRegistry::engines() -{ +std::vector EngineRegistry::engines() { pthread_once(&once, init); AutoLock lock(*local_mutex); @@ -68,8 +66,7 @@ std::vector EngineRegistry::engines() return res; } -void EngineRegistry::add(Engine* e) -{ +void EngineRegistry::add(Engine* e) { pthread_once(&once, init); eckit::AutoLock lock(*local_mutex); @@ -79,8 +76,7 @@ void EngineRegistry::add(Engine* e) (*m)[name] = e; } -Engine* EngineRegistry::remove(const std::string& name) -{ +Engine* EngineRegistry::remove(const std::string& name) { pthread_once(&once, init); eckit::AutoLock lock(*local_mutex); @@ -91,9 +87,7 @@ Engine* EngineRegistry::remove(const std::string& name) return e; } - -std::vector EngineRegistry::list() -{ +std::vector EngineRegistry::list() { pthread_once(&once, init); eckit::AutoLock lock(*local_mutex); @@ -104,12 +98,12 @@ std::vector EngineRegistry::list() return res; } -void EngineRegistry::list(std::ostream &out) { +void EngineRegistry::list(std::ostream& out) { pthread_once(&once, init); eckit::AutoLock lock(local_mutex); - const char *sep = ""; + const char* sep = ""; for (std::map::const_iterator j = m->begin(); j != m->end(); ++j) { out << sep << (*j).first; sep = ", "; @@ -118,13 +112,11 @@ void EngineRegistry::list(std::ostream &out) { //---------------------------------------------------------------------------------------------------------------------- -Engine& Engine::backend(const std::string& name) -{ +Engine& Engine::backend(const std::string& name) { return EngineRegistry::engine(name); } -Engine::~Engine() { -} +Engine::~Engine() {} //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/Engine.h b/src/fdb5/database/Engine.h index 193b4a90b..1eb9feedd 100644 --- a/src/fdb5/database/Engine.h +++ b/src/fdb5/database/Engine.h @@ -17,14 +17,13 @@ #define fdb5_Engine_H #include -#include #include +#include #include "metkit/mars/MarsRequest.h" -#include "eckit/memory/NonCopyable.h" #include "eckit/filesystem/URI.h" - +#include "eckit/memory/NonCopyable.h" namespace fdb5 { @@ -36,11 +35,9 @@ class Config; class Engine : private eckit::NonCopyable { public: // methods - static Engine& backend(const std::string& name); public: // methods - virtual ~Engine(); /// @returns the named identifier of this engine @@ -55,14 +52,13 @@ class Engine : private eckit::NonCopyable { /// Lists the roots that can be visited given a DB key virtual std::vector visitableLocations(const Key& key, const Config& config) const = 0; - virtual std::vector visitableLocations(const metkit::mars::MarsRequest& rq, const Config& config) const = 0; + virtual std::vector visitableLocations(const metkit::mars::MarsRequest& rq, + const Config& config) const = 0; - friend std::ostream &operator<<(std::ostream &s, const Engine& x); + friend std::ostream& operator<<(std::ostream& s, const Engine& x); protected: // methods - - virtual void print( std::ostream &out ) const = 0; - + virtual void print(std::ostream& out) const = 0; }; //---------------------------------------------------------------------------------------------------------------------- @@ -72,7 +68,6 @@ class Engine : private eckit::NonCopyable { class EngineRegistry : private eckit::NonCopyable { public: // methods - static bool has(const std::string& name); static Engine& engine(const std::string& name); @@ -81,22 +76,18 @@ class EngineRegistry : private eckit::NonCopyable { static std::vector list(); - static void list(std::ostream &); + static void list(std::ostream&); protected: // methods - static void add(Engine*); static Engine* remove(const std::string&); - }; - /// Templated for self-registering engines that does the self-registration into the registry -template< class T> +template class EngineBuilder : public EngineRegistry { public: - EngineBuilder() { Engine* e = new T(); name_ = e->name(); @@ -109,7 +100,6 @@ class EngineBuilder : public EngineRegistry { } std::string name_; - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/EntryVisitMechanism.cc b/src/fdb5/database/EntryVisitMechanism.cc index bc2b8d328..f0f710093 100644 --- a/src/fdb5/database/EntryVisitMechanism.cc +++ b/src/fdb5/database/EntryVisitMechanism.cc @@ -12,16 +12,15 @@ #include "eckit/io/AutoCloser.h" +#include "fdb5/LibFdb5.h" #include "fdb5/api/helpers/FDBToolRequest.h" -#include "fdb5/database/Manager.h" #include "fdb5/database/Key.h" -#include "fdb5/LibFdb5.h" -#include "fdb5/rules/Schema.h" +#include "fdb5/database/Manager.h" #include "fdb5/database/Store.h" +#include "fdb5/rules/Schema.h" using namespace eckit; - namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- @@ -83,16 +82,13 @@ void EntryVisitor::visitDatum(const Field& field, const std::string& keyFingerpr visitDatum(field, key); } - time_t EntryVisitor::indexTimestamp() const { return currentIndex_ == nullptr ? 0 : currentIndex_->timestamp(); } //---------------------------------------------------------------------------------------------------------------------- -EntryVisitMechanism::EntryVisitMechanism(const Config& config) : - dbConfig_(config), - fail_(true) {} +EntryVisitMechanism::EntryVisitMechanism(const Config& config) : dbConfig_(config), fail_(true) {} void EntryVisitMechanism::visit(const FDBToolRequest& request, EntryVisitor& visitor) { @@ -122,14 +118,10 @@ void EntryVisitMechanism::visit(const FDBToolRequest& request, EntryVisitor& vis std::unique_ptr catalogue; try { - - catalogue = CatalogueReaderFactory::instance().build(uri, dbConfig_); - } catch (fdb5::DatabaseNotFoundException& e) { - - visitor.onDatabaseNotFound(e); + catalogue = CatalogueReaderFactory::instance().build(uri, dbConfig_); - } + } catch (fdb5::DatabaseNotFoundException& e) { visitor.onDatabaseNotFound(e); } ASSERT(catalogue->open()); @@ -138,15 +130,13 @@ void EntryVisitMechanism::visit(const FDBToolRequest& request, EntryVisitor& vis catalogue->visitEntries(visitor, /* *store, */ false); } - } catch (eckit::UserError&) { - throw; - } catch (eckit::Exception& e) { + } catch (eckit::UserError&) { throw; } catch (eckit::Exception& e) { Log::warning() << e.what() << std::endl; - if (fail_) throw; + if (fail_) + throw; } - } //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/database/EntryVisitMechanism.h b/src/fdb5/database/EntryVisitMechanism.h index 92139c8ca..c58dae95d 100644 --- a/src/fdb5/database/EntryVisitMechanism.h +++ b/src/fdb5/database/EntryVisitMechanism.h @@ -32,8 +32,7 @@ class TypedKey; class EntryVisitor : public eckit::NonCopyable { -public: // methods - +public: // methods EntryVisitor(); virtual ~EntryVisitor(); @@ -41,8 +40,8 @@ class EntryVisitor : public eckit::NonCopyable { virtual bool visitIndexes() { return true; } virtual bool visitEntries() { return true; } - virtual bool visitDatabase(const Catalogue& catalogue); // return true if Catalogue should be explored - virtual bool visitIndex(const Index& index); // return true if index should be explored + virtual bool visitDatabase(const Catalogue& catalogue); // return true if Catalogue should be explored + virtual bool visitIndex(const Index& index); // return true if index should be explored virtual void catalogueComplete(const Catalogue& catalogue); virtual void visitDatum(const Field& field, const std::string& keyFingerprint); @@ -51,14 +50,12 @@ class EntryVisitor : public eckit::NonCopyable { time_t indexTimestamp() const; protected: - Store& store() const; private: // methods - virtual void visitDatum(const Field& field, const Key& datumKey) = 0; -protected: // members +protected: // members /// Non-owning const Catalogue* currentCatalogue_ = nullptr; /// Owned store @@ -73,14 +70,12 @@ class EntryVisitor : public eckit::NonCopyable { class EntryVisitMechanism : public eckit::NonCopyable { -public: // methods - +public: // methods EntryVisitMechanism(const Config& config); void visit(const FDBToolRequest& request, EntryVisitor& visitor); -private: // members - +private: // members const Config& dbConfig_; // Fail on error @@ -89,6 +84,6 @@ class EntryVisitMechanism : public eckit::NonCopyable { //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 #endif diff --git a/src/fdb5/database/Field.cc b/src/fdb5/database/Field.cc index b8eaa1053..4108dc3d9 100644 --- a/src/fdb5/database/Field.cc +++ b/src/fdb5/database/Field.cc @@ -16,21 +16,21 @@ namespace fdb5 { Field::Field() {} -Field::Field(std::shared_ptr location, time_t timestamp, const FieldDetails& details): - location_(std::move(location)), - timestamp_(timestamp), - details_(details) { -} +Field::Field(std::shared_ptr location, time_t timestamp, const FieldDetails& details) + : location_(std::move(location)) + , timestamp_(timestamp) + , details_(details) {} -Field::Field(const FieldLocation&& location, time_t timestamp, const FieldDetails& details): - location_(location.make_shared()), - timestamp_(timestamp), - details_(details) { -} +Field::Field(const FieldLocation&& location, time_t timestamp, const FieldDetails& details) + : location_(location.make_shared()) + , timestamp_(timestamp) + , details_(details) {} void Field::print(std::ostream& out) const { out << "Field(location=" << location_; - if(details_) { out << ",details=" << details_; } + if (details_) { + out << ",details=" << details_; + } out << ")"; } diff --git a/src/fdb5/database/Field.h b/src/fdb5/database/Field.h index c02af6635..204849738 100644 --- a/src/fdb5/database/Field.h +++ b/src/fdb5/database/Field.h @@ -21,13 +21,13 @@ #include "eckit/io/Length.h" #include "eckit/io/Offset.h" #include "eckit/memory/NonCopyable.h" -#include "eckit/types/Types.h" #include "eckit/types/FixedString.h" +#include "eckit/types/Types.h" +#include "fdb5/database/FieldDetails.h" +#include "fdb5/database/FieldLocation.h" #include "fdb5/database/IndexAxis.h" #include "fdb5/database/Key.h" -#include "fdb5/database/FieldLocation.h" -#include "fdb5/database/FieldDetails.h" namespace eckit { class DataHandle; @@ -40,10 +40,10 @@ namespace fdb5 { class Field { public: // methods - Field(); - Field(std::shared_ptr location, time_t timestamp, const FieldDetails& details = FieldDetails()); + Field(std::shared_ptr location, time_t timestamp, + const FieldDetails& details = FieldDetails()); Field(const FieldLocation&& location, time_t timestamp, const FieldDetails& details = FieldDetails()); eckit::DataHandle* dataHandle() const { return location_->dataHandle(); } @@ -54,26 +54,22 @@ class Field { /// stableLocation is an object with validity that extends longer than that of the /// owning DB. May need converting to a more static form --- or not. - std::shared_ptr stableLocation() const { - return location_->stableLocation(); - } + std::shared_ptr stableLocation() const { return location_->stableLocation(); } const FieldDetails& details() const { return details_; } private: // members - - std::shared_ptr location_; + std::shared_ptr location_; time_t timestamp_; FieldDetails details_; - void print( std::ostream &out ) const; + void print(std::ostream& out) const; - friend std::ostream &operator<<(std::ostream &s, const Field &x) { + friend std::ostream& operator<<(std::ostream& s, const Field& x) { x.print(s); return s; } - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/FieldDetails.cc b/src/fdb5/database/FieldDetails.cc index 1e17dac50..cd255fb5c 100644 --- a/src/fdb5/database/FieldDetails.cc +++ b/src/fdb5/database/FieldDetails.cc @@ -15,16 +15,11 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- void FieldDetails::print(std::ostream& out) const { - out - << "(referenceValue=" << referenceValue_ - << ",binaryScaleFactor=" << binaryScaleFactor_ - << ",decimalScaleFactor=" << decimalScaleFactor_ - << ",bitsPerValue=" << bitsPerValue_ - << ",offsetBeforeData=" << offsetBeforeData_ - << ",offsetBeforeBitmap=" << offsetBeforeBitmap_ - << ",numberOfValues=" << numberOfValues_ - << ",numberOfDataPoints=" << numberOfDataPoints_ - << ",sphericalHarmonics=" << sphericalHarmonics_ << ")"; + out << "(referenceValue=" << referenceValue_ << ",binaryScaleFactor=" << binaryScaleFactor_ + << ",decimalScaleFactor=" << decimalScaleFactor_ << ",bitsPerValue=" << bitsPerValue_ + << ",offsetBeforeData=" << offsetBeforeData_ << ",offsetBeforeBitmap=" << offsetBeforeBitmap_ + << ",numberOfValues=" << numberOfValues_ << ",numberOfDataPoints=" << numberOfDataPoints_ + << ",sphericalHarmonics=" << sphericalHarmonics_ << ")"; } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/FieldDetails.h b/src/fdb5/database/FieldDetails.h index 124ec0522..69450b1cf 100644 --- a/src/fdb5/database/FieldDetails.h +++ b/src/fdb5/database/FieldDetails.h @@ -21,8 +21,8 @@ #include "eckit/io/Length.h" #include "eckit/io/Offset.h" #include "eckit/memory/NonCopyable.h" -#include "eckit/types/Types.h" #include "eckit/types/FixedString.h" +#include "eckit/types/Types.h" #include "fdb5/database/IndexAxis.h" #include "fdb5/database/Key.h" @@ -33,36 +33,34 @@ namespace fdb5 { struct FieldDetails { - FieldDetails(): - referenceValue_(0), - binaryScaleFactor_(0), - decimalScaleFactor_(0), - bitsPerValue_(0), - offsetBeforeData_(0), - offsetBeforeBitmap_(0), - numberOfValues_(0), - numberOfDataPoints_(0), - sphericalHarmonics_(0) - { - } + FieldDetails() + : referenceValue_(0) + , binaryScaleFactor_(0) + , decimalScaleFactor_(0) + , bitsPerValue_(0) + , offsetBeforeData_(0) + , offsetBeforeBitmap_(0) + , numberOfValues_(0) + , numberOfDataPoints_(0) + , sphericalHarmonics_(0) {} operator bool() const { return bitsPerValue_ && !gridMD5_.empty(); } - double referenceValue_; - long binaryScaleFactor_; - long decimalScaleFactor_; + double referenceValue_; + long binaryScaleFactor_; + long decimalScaleFactor_; unsigned long bitsPerValue_; unsigned long offsetBeforeData_; unsigned long offsetBeforeBitmap_; unsigned long numberOfValues_; unsigned long numberOfDataPoints_; - long sphericalHarmonics_; + long sphericalHarmonics_; - eckit::FixedString<32> gridMD5_; ///< md5 of the grid geometry section in GRIB + eckit::FixedString<32> gridMD5_; ///< md5 of the grid geometry section in GRIB - void print( std::ostream &out ) const; + void print(std::ostream& out) const; - friend std::ostream &operator<<(std::ostream &s, const FieldDetails &x) { + friend std::ostream& operator<<(std::ostream& s, const FieldDetails& x) { x.print(s); return s; } diff --git a/src/fdb5/database/FieldLocation.cc b/src/fdb5/database/FieldLocation.cc index 4365c7f55..4e57834ba 100644 --- a/src/fdb5/database/FieldLocation.cc +++ b/src/fdb5/database/FieldLocation.cc @@ -18,10 +18,12 @@ namespace fdb5 { -::eckit::ClassSpec FieldLocation::classSpec_ = {&Streamable::classSpec(), "FieldLocation",}; +::eckit::ClassSpec FieldLocation::classSpec_ = { + &Streamable::classSpec(), + "FieldLocation", +}; -FieldLocationFactory::FieldLocationFactory() { -} +FieldLocationFactory::FieldLocationFactory() {} FieldLocationFactory& FieldLocationFactory::instance() { static FieldLocationFactory theOne; @@ -30,7 +32,7 @@ FieldLocationFactory& FieldLocationFactory::instance() { void FieldLocationFactory::add(const std::string& name, FieldLocationBuilderBase* builder) { eckit::AutoLock lock(mutex_); - if(has(name)) { + if (has(name)) { throw eckit::SeriousBug("Duplicate entry in FieldLocationFactory: " + name, Here()); } builders_[name] = builder; @@ -48,15 +50,17 @@ bool FieldLocationFactory::has(const std::string& name) { void FieldLocationFactory::list(std::ostream& out) { eckit::AutoLock lock(mutex_); const char* sep = ""; - for (std::map::const_iterator j = builders_.begin(); j != builders_.end(); ++j) { + for (std::map::const_iterator j = builders_.begin(); j != builders_.end(); + ++j) { out << sep << (*j).first; sep = ", "; } } -FieldLocation* FieldLocationFactory::build(const std::string& name, const eckit::URI &uri, eckit::Offset offset, eckit::Length length, const Key& remapKey) { +FieldLocation* FieldLocationFactory::build(const std::string& name, const eckit::URI& uri, eckit::Offset offset, + eckit::Length length, const Key& remapKey) { - ASSERT (length != 0); + ASSERT(length != 0); eckit::AutoLock lock(mutex_); @@ -75,7 +79,7 @@ FieldLocation* FieldLocationFactory::build(const std::string& name, const eckit: return (*j).second->make(uri, offset, length, remapKey); } -FieldLocation* FieldLocationFactory::build(const std::string& name, const eckit::URI &uri) { +FieldLocation* FieldLocationFactory::build(const std::string& name, const eckit::URI& uri) { eckit::AutoLock lock(mutex_); @@ -101,7 +105,8 @@ FieldLocationBuilderBase::FieldLocationBuilderBase(const std::string& name) : na } FieldLocationBuilderBase::~FieldLocationBuilderBase() { - if(LibFdb5::instance().dontDeregisterFactories()) return; + if (LibFdb5::instance().dontDeregisterFactories()) + return; FieldLocationFactory::instance().remove(name_); } @@ -110,17 +115,13 @@ FieldLocationBuilderBase::~FieldLocationBuilderBase() { FieldLocation::FieldLocation(const eckit::URI& uri) : uri_(uri) { try { offset_ = eckit::Offset(std::stoll(uri.fragment())); - } catch (std::invalid_argument& e) { - offset_ = eckit::Offset(0); - } + } catch (std::invalid_argument& e) { offset_ = eckit::Offset(0); } std::string lengthStr = uri.query("length"); if (!lengthStr.empty()) { try { length_ = eckit::Length(std::stoll(lengthStr)); - } catch (std::invalid_argument& e) { - length_ = eckit::Length(0); - } + } catch (std::invalid_argument& e) { length_ = eckit::Length(0); } } else { length_ = eckit::Length(0); } @@ -143,9 +144,11 @@ eckit::URI FieldLocation::fullUri() const { return full; } - FieldLocation::FieldLocation(const eckit::URI& uri, eckit::Offset offset, eckit::Length length, const Key& remapKey) - : uri_(uri), offset_(offset), length_(length), remapKey_(remapKey) {} + : uri_(uri) + , offset_(offset) + , length_(length) + , remapKey_(remapKey) {} void FieldLocation::encode(eckit::Stream& s) const { s << uri_; @@ -161,21 +164,21 @@ FieldLocation::FieldLocation(eckit::Stream& s) { s >> remapKey_; } -//void FieldLocation::remapKey(const Key& key) { NOTIMP; } -//const Key& FieldLocation::remapKey() const { NOTIMP; } +// void FieldLocation::remapKey(const Key& key) { NOTIMP; } +// const Key& FieldLocation::remapKey() const { NOTIMP; } void FieldLocation::dump(std::ostream& out) const { out << " uri: " << uri().asRawString(); } void FieldLocation::print(std::ostream& out) const { - out << " FieldLocation[uri=" << uri_ << ",offset=" << offset() << ",length=" << length() << ",remapKey=" << remapKey_ << "]"; + out << " FieldLocation[uri=" << uri_ << ",offset=" << offset() << ",length=" << length() + << ",remapKey=" << remapKey_ << "]"; } //---------------------------------------------------------------------------------------------------------------------- -FieldLocationVisitor::~FieldLocationVisitor() -{} +FieldLocationVisitor::~FieldLocationVisitor() {} void FieldLocationPrinter::operator()(const FieldLocation& location) { location.dump(out_); diff --git a/src/fdb5/database/FieldLocation.h b/src/fdb5/database/FieldLocation.h index 2e07ae4e8..68e944892 100644 --- a/src/fdb5/database/FieldLocation.h +++ b/src/fdb5/database/FieldLocation.h @@ -16,8 +16,8 @@ #ifndef fdb5_FieldLocation_H #define fdb5_FieldLocation_H -#include #include +#include #include "eckit/filesystem/PathName.h" #include "eckit/io/Length.h" @@ -27,7 +27,7 @@ #include "fdb5/database/Key.h" namespace eckit { - class DataHandle; +class DataHandle; } namespace fdb5 { @@ -39,7 +39,6 @@ class FieldLocationVisitor; class FieldLocation : public eckit::OwnedLock, public eckit::Streamable { public: // methods - FieldLocation() : offset_(eckit::Offset(0)), length_(eckit::Length(0)), remapKey_(Key()) {} FieldLocation(const eckit::URI& uri); FieldLocation(const eckit::URI& uri, eckit::Offset offset, eckit::Length length, const Key& remapKey); @@ -55,7 +54,7 @@ class FieldLocation : public eckit::OwnedLock, public eckit::Streamable { virtual eckit::Length length() const { return length_; } const Key& remapKey() const { return remapKey_; } - virtual eckit::DataHandle *dataHandle() const = 0; + virtual eckit::DataHandle* dataHandle() const = 0; /// Create a (shared) copy of the current object, for storage in a general container. virtual std::shared_ptr make_shared() const = 0; @@ -64,28 +63,24 @@ class FieldLocation : public eckit::OwnedLock, public eckit::Streamable { virtual void visit(FieldLocationVisitor& visitor) const = 0; - virtual void dump(std::ostream &out) const; + virtual void dump(std::ostream& out) const; private: // methods - - virtual void print( std::ostream &out ) const; + virtual void print(std::ostream& out) const; protected: // For Streamable - void encode(eckit::Stream&) const override; - static eckit::ClassSpec classSpec_; + static eckit::ClassSpec classSpec_; protected: // members - eckit::URI uri_; eckit::Offset offset_; eckit::Length length_; Key remapKey_; private: // friends - - friend std::ostream &operator<<(std::ostream &s, const FieldLocation &x) { + friend std::ostream& operator<<(std::ostream& s, const FieldLocation& x) { x.print(s); return s; } @@ -97,68 +92,67 @@ class FieldLocation : public eckit::OwnedLock, public eckit::Streamable { //---------------------------------------------------------------------------------------------------------------------- - class FieldLocationBuilderBase { - std::string name_; - public: - FieldLocationBuilderBase(const std::string &); - virtual ~FieldLocationBuilderBase(); - virtual FieldLocation* make(const eckit::URI &uri) = 0; - virtual FieldLocation* make(const eckit::URI &uri, eckit::Offset offset, eckit::Length length, const Key& remapKey) = 0; - }; +class FieldLocationBuilderBase { + std::string name_; - template< class T> - class FieldLocationBuilder : public FieldLocationBuilderBase { - FieldLocation* make(const eckit::URI &uri) override { - return new T(uri); - } - FieldLocation* make(const eckit::URI &uri, eckit::Offset offset, eckit::Length length, const Key& remapKey) override { - return new T(uri, offset, length, remapKey); - } - public: - FieldLocationBuilder(const std::string &name) : FieldLocationBuilderBase(name) {} - virtual ~FieldLocationBuilder() = default; - }; +public: + FieldLocationBuilderBase(const std::string&); + virtual ~FieldLocationBuilderBase(); + virtual FieldLocation* make(const eckit::URI& uri) = 0; + virtual FieldLocation* make(const eckit::URI& uri, eckit::Offset offset, eckit::Length length, + const Key& remapKey) = 0; +}; - class FieldLocationFactory { - public: +template +class FieldLocationBuilder : public FieldLocationBuilderBase { + FieldLocation* make(const eckit::URI& uri) override { return new T(uri); } + FieldLocation* make(const eckit::URI& uri, eckit::Offset offset, eckit::Length length, + const Key& remapKey) override { + return new T(uri, offset, length, remapKey); + } - static FieldLocationFactory& instance(); +public: + FieldLocationBuilder(const std::string& name) : FieldLocationBuilderBase(name) {} + virtual ~FieldLocationBuilder() = default; +}; - void add(const std::string& name, FieldLocationBuilderBase* builder); - void remove(const std::string& name); +class FieldLocationFactory { +public: + static FieldLocationFactory& instance(); - bool has(const std::string& name); - void list(std::ostream &); + void add(const std::string& name, FieldLocationBuilderBase* builder); + void remove(const std::string& name); - /// @returns a specialized FieldLocation built by specified builder - FieldLocation* build(const std::string &, const eckit::URI &); - FieldLocation* build(const std::string &, const eckit::URI &, eckit::Offset offset, eckit::Length length, const Key& remapKey); + bool has(const std::string& name); + void list(std::ostream&); - private: + /// @returns a specialized FieldLocation built by specified builder + FieldLocation* build(const std::string&, const eckit::URI&); + FieldLocation* build(const std::string&, const eckit::URI&, eckit::Offset offset, eckit::Length length, + const Key& remapKey); - FieldLocationFactory(); +private: + FieldLocationFactory(); - std::map builders_; - eckit::Mutex mutex_; - }; + std::map builders_; + eckit::Mutex mutex_; +}; //---------------------------------------------------------------------------------------------------------------------- - class FieldLocationVisitor : private eckit::NonCopyable { public: // methods - virtual ~FieldLocationVisitor(); - virtual void operator() (const FieldLocation& location) = 0; - + virtual void operator()(const FieldLocation& location) = 0; }; class FieldLocationPrinter : public FieldLocationVisitor { public: FieldLocationPrinter(std::ostream& out) : out_(out) {} - virtual void operator() (const FieldLocation& location); + virtual void operator()(const FieldLocation& location); + private: std::ostream& out_; }; diff --git a/src/fdb5/database/Index.cc b/src/fdb5/database/Index.cc old mode 100755 new mode 100644 index a08db81a6..6011afe0c --- a/src/fdb5/database/Index.cc +++ b/src/fdb5/database/Index.cc @@ -8,43 +8,37 @@ * does it submit to any jurisdiction. */ -#include "fdb5/LibFdb5.h" #include "fdb5/database/Index.h" -#include "fdb5/rules/Schema.h" -#include "fdb5/rules/Rule.h" +#include "fdb5/LibFdb5.h" #include "fdb5/database/EntryVisitMechanism.h" +#include "fdb5/rules/Rule.h" +#include "fdb5/rules/Schema.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -IndexBase::IndexBase(const Key& key, const std::string& type, const Catalogue& catalogue) : - type_(type), - key_(key), - catalogue_(catalogue) {} +IndexBase::IndexBase(const Key& key, const std::string& type, const Catalogue& catalogue) + : type_(type) + , key_(key) + , catalogue_(catalogue) {} -enum IndexBaseStreamKeys { - IndexKeyUnrecognised, - IndexKey, - IndexType, - IndexTimestamp -}; +enum IndexBaseStreamKeys { IndexKeyUnrecognised, IndexKey, IndexType, IndexTimestamp }; IndexBaseStreamKeys keyId(const std::string& s) { - static const std::map keys { - {"key" , IndexKey}, + static const std::map keys{ + {"key", IndexKey}, {"type", IndexType}, {"time", IndexTimestamp}, }; auto it = keys.find(s); - if( it != keys.end() ) { + if (it != keys.end()) { return it->second; } return IndexKeyUnrecognised; } - void IndexBase::decodeCurrent(eckit::Stream& s, const int version) { ASSERT(version >= 3); @@ -65,7 +59,7 @@ void IndexBase::decodeCurrent(eckit::Stream& s, const int version) { s >> timestamp_; break; default: - throw eckit::SeriousBug("IndexBase de-serialization error: "+k+" field is not recognized"); + throw eckit::SeriousBug("IndexBase de-serialization error: " + k + " field is not recognized"); } } ASSERT(!key_.empty()); @@ -73,12 +67,12 @@ void IndexBase::decodeCurrent(eckit::Stream& s, const int version) { ASSERT(timestamp_); } -void IndexBase::decodeLegacy(eckit::Stream& s, const int version) { // decoding of old Stream format, for backward compatibility +void IndexBase::decodeLegacy(eckit::Stream& s, + const int version) { // decoding of old Stream format, for backward compatibility ASSERT(version <= 2); axes_.decode(s, version); - std::string dummy; s >> key_; s >> dummy; ///< legacy entry, no longer used but stays here so we can read existing indexes @@ -86,17 +80,14 @@ void IndexBase::decodeLegacy(eckit::Stream& s, const int version) { // decoding timestamp_ = 0; } - -IndexBase::IndexBase(eckit::Stream& s, const int version, const Catalogue& catalogue) : - catalogue_(catalogue) { +IndexBase::IndexBase(eckit::Stream& s, const int version, const Catalogue& catalogue) : catalogue_(catalogue) { if (version >= 3) decodeCurrent(s, version); else decodeLegacy(s, version); } -IndexBase::~IndexBase() { -} +IndexBase::~IndexBase() {} void IndexBase::encode(eckit::Stream& s, const int version) const { if (version >= 3) { @@ -145,9 +136,11 @@ const TypesRegistry& IndexBase::registry() const { bool IndexBase::partialMatch(const metkit::mars::MarsRequest& request) const { - if (!key_.partialMatch(request)) return false; + if (!key_.partialMatch(request)) + return false; - if (!axes_.partialMatch(request, registry())) return false; + if (!axes_.partialMatch(request, registry())) + return false; return true; } @@ -172,11 +165,9 @@ const IndexAxis& IndexBase::axes() const { class NullIndex : public IndexBase { public: // methods - NullIndex() : IndexBase(Key{}, "null", NullCatalogue{}) {} private: // methods - const IndexLocation& location() const override { NOTIMP; } bool dirty() const override { NOTIMP; } @@ -193,33 +184,28 @@ class NullIndex : public IndexBase { void encode(eckit::Stream&, const int version) const override { NOTIMP; } void entries(EntryVisitor&) const override { NOTIMP; } - void print( std::ostream& s) const override { s << "NullIndex()"; } + void print(std::ostream& s) const override { s << "NullIndex()"; } void dump(std::ostream&, const char*, bool, bool) const override { NOTIMP; } void flock() const override { NOTIMP; } void funlock() const override { NOTIMP; } IndexStats statistics() const override { NOTIMP; } - }; //---------------------------------------------------------------------------------------------------------------------- -Index::Index() : - content_(new NullIndex()), - null_(true) { +Index::Index() : content_(new NullIndex()), null_(true) { content_->attach(); } -Index::Index(IndexBase* p) : - content_(p), - null_(false) { +Index::Index(IndexBase* p) : content_(p), null_(false) { ASSERT(p); content_->attach(); } Index::~Index() { - content_->detach(); + content_->detach(); } Index::Index(const Index& s) : content_(s.content_), null_(s.null_) { @@ -229,7 +215,7 @@ Index::Index(const Index& s) : content_(s.content_), null_(s.null_) { Index& Index::operator=(const Index& s) { content_->detach(); content_ = s.content_; - null_ = s.null_; + null_ = s.null_; content_->attach(); return *this; } diff --git a/src/fdb5/database/Index.h b/src/fdb5/database/Index.h old mode 100755 new mode 100644 index 1d38114f8..bace047ed --- a/src/fdb5/database/Index.h +++ b/src/fdb5/database/Index.h @@ -15,27 +15,26 @@ #ifndef fdb5_Index_H #define fdb5_Index_H +#include #include #include -#include #include "eckit/eckit.h" #include "eckit/io/Length.h" #include "eckit/io/Offset.h" +#include "eckit/memory/Counted.h" #include "eckit/memory/NonCopyable.h" #include "eckit/types/FixedString.h" #include "eckit/types/Types.h" -#include "eckit/memory/Counted.h" #include "fdb5/database/EntryVisitMechanism.h" #include "fdb5/database/Field.h" -#include "fdb5/database/IndexStats.h" #include "fdb5/database/IndexAxis.h" #include "fdb5/database/IndexLocation.h" +#include "fdb5/database/IndexStats.h" #include "fdb5/database/Indexer.h" - namespace eckit { class Stream; } @@ -55,7 +54,6 @@ class Schema; class IndexBase : public eckit::Counted { public: // methods - IndexBase(const Key& key, const std::string& type, const Catalogue& catalogue); IndexBase(eckit::Stream& s, const int version, const Catalogue& catalogue); @@ -83,8 +81,8 @@ class IndexBase : public eckit::Counted { time_t timestamp() const { return timestamp_; } - virtual bool get(const Key& key, const Key& remapKey, Field &field) const = 0; - virtual void put(const Key& key, const Field &field); + virtual bool get(const Key& key, const Key& remapKey, Field& field) const = 0; + virtual void put(const Key& key, const Field& field); virtual void encode(eckit::Stream& s, const int version) const; virtual void entries(EntryVisitor& visitor) const = 0; @@ -95,7 +93,7 @@ class IndexBase : public eckit::Counted { virtual IndexStats statistics() const = 0; - virtual void print( std::ostream &out ) const = 0; + virtual void print(std::ostream& out) const = 0; virtual void flock() const = 0; virtual void funlock() const = 0; @@ -104,37 +102,34 @@ class IndexBase : public eckit::Counted { void takeTimestamp() { time(×tamp_); } private: // methods - void encodeCurrent(eckit::Stream& s, const int version) const; void encodeLegacy(eckit::Stream& s, const int version) const; void decodeCurrent(eckit::Stream& s, const int version); void decodeLegacy(eckit::Stream& s, const int version); - virtual void add(const Key& key, const Field &field) = 0; + virtual void add(const Key& key, const Field& field) = 0; const TypesRegistry& registry() const; protected: // members - std::string type_; /// @note Order of members is important here ... - IndexAxis axes_; ///< This Index spans along these axis - Key key_; ///< key that selected this index - time_t timestamp_; ///< timestamp when this Index was flushed + IndexAxis axes_; ///< This Index spans along these axis + Key key_; ///< key that selected this index + time_t timestamp_; ///< timestamp when this Index was flushed - Indexer indexer_; + Indexer indexer_; friend std::ostream& operator<<(std::ostream& s, const IndexBase& o) { - o.print(s); return s; + o.print(s); + return s; } private: // members - const Catalogue& catalogue_; mutable std::optional> registry_; - }; //---------------------------------------------------------------------------------------------------------------------- @@ -142,7 +137,6 @@ class IndexBase : public eckit::Counted { class Index { public: // methods - Index(); Index(IndexBase* i); @@ -157,10 +151,10 @@ class Index { bool dirty() const { return content_->dirty(); } - void open() { return content_->open(); } + void open() { return content_->open(); } void reopen() { return content_->reopen(); } - void close() { return content_->close(); } - void flush() { return content_->flush(); } + void close() { return content_->close(); } + void flush() { return content_->flush(); } void visit(IndexLocationVisitor& visitor) const { content_->visit(visitor); } @@ -176,7 +170,7 @@ class Index { void encode(eckit::Stream& s, const int version) const { content_->encode(s, version); } void entries(EntryVisitor& v) const { content_->entries(v); } - void dump(std::ostream &out, const char* indent, bool simple = false, bool dumpFields = false) const { + void dump(std::ostream& out, const char* indent, bool simple = false, bool dumpFields = false) const { content_->dump(out, indent, simple, dumpFields); } @@ -190,22 +184,21 @@ class Index { bool null() const { return null_; } - friend bool operator< (const Index& i1, const Index& i2) { return i1.content_ < i2.content_; } - friend bool operator== (const Index& i1, const Index& i2) { return i1.content_ == i2.content_; } + friend bool operator<(const Index& i1, const Index& i2) { return i1.content_ < i2.content_; } + friend bool operator==(const Index& i1, const Index& i2) { return i1.content_ == i2.content_; } void flock() const { content_->flock(); } void funlock() const { content_->funlock(); } private: // methods - void print(std::ostream& s) const { content_->print(s); } friend std::ostream& operator<<(std::ostream& s, const Index& o) { - o.print(s); return s; + o.print(s); + return s; } private: // members - IndexBase* content_; bool null_; }; diff --git a/src/fdb5/database/IndexAxis.cc b/src/fdb5/database/IndexAxis.cc old mode 100755 new mode 100644 index c6b29e669..de86f776a --- a/src/fdb5/database/IndexAxis.cc +++ b/src/fdb5/database/IndexAxis.cc @@ -8,47 +8,41 @@ * does it submit to any jurisdiction. */ - -#include "eckit/log/Log.h" #include "eckit/exception/Exceptions.h" +#include "eckit/log/Log.h" #include "metkit/mars/MarsRequest.h" #include "fdb5/database/AxisRegistry.h" #include "fdb5/database/IndexAxis.h" #include "fdb5/database/Key.h" -#include "fdb5/types/TypesRegistry.h" #include "fdb5/types/Type.h" +#include "fdb5/types/TypesRegistry.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -IndexAxis::IndexAxis() : - readOnly_(false), - dirty_(false) { -} +IndexAxis::IndexAxis() : readOnly_(false), dirty_(false) {} IndexAxis::~IndexAxis() { - if (!readOnly_) - return; + if (!readOnly_) + return; for (AxisMap::iterator it = axis_.begin(); it != axis_.end(); ++it) { - AxisRegistry::instance().release(it->first, it->second); + AxisRegistry::instance().release(it->first, it->second); } } -IndexAxis::IndexAxis(eckit::Stream &s, const int version) : - readOnly_(true), - dirty_(false) { +IndexAxis::IndexAxis(eckit::Stream& s, const int version) : readOnly_(true), dirty_(false) { decode(s, version); } -IndexAxis::IndexAxis(IndexAxis&& rhs) noexcept : - axis_(std::move(rhs.axis_)), - readOnly_(rhs.readOnly_), - dirty_(rhs.dirty_) {} +IndexAxis::IndexAxis(IndexAxis&& rhs) noexcept + : axis_(std::move(rhs.axis_)) + , readOnly_(rhs.readOnly_) + , dirty_(rhs.dirty_) {} IndexAxis& IndexAxis::operator=(IndexAxis&& rhs) noexcept { axis_ = std::move(rhs.axis_); @@ -59,12 +53,15 @@ IndexAxis& IndexAxis::operator=(IndexAxis&& rhs) noexcept { bool IndexAxis::operator==(const IndexAxis& rhs) const { - if (axis_.size() != rhs.axis_.size()) return false; + if (axis_.size() != rhs.axis_.size()) + return false; for (const auto& kv : axis_) { auto it = rhs.axis_.find(kv.first); - if (it == rhs.axis_.end()) return false; - if (*kv.second != *it->second) return false; + if (it == rhs.axis_.end()) + return false; + if (*kv.second != *it->second) + return false; } return true; @@ -79,7 +76,7 @@ size_t encodeString(size_t len) { } size_t IndexAxis::encodeSize(const int version) const { - size_t size=2; + size_t size = 2; size += encodeString(4) + 5; size += encodeString(4); for (const auto& [key, vals] : axis_) { @@ -91,7 +88,7 @@ size_t IndexAxis::encodeSize(const int version) const { return size; } -void IndexAxis::encode(eckit::Stream &s, const int version) const { +void IndexAxis::encode(eckit::Stream& s, const int version) const { if (version >= 3) { encodeCurrent(s, version); } else { @@ -99,7 +96,7 @@ void IndexAxis::encode(eckit::Stream &s, const int version) const { } } -void IndexAxis::encodeCurrent(eckit::Stream &s, const int version) const { +void IndexAxis::encodeCurrent(eckit::Stream& s, const int version) const { ASSERT(version >= 3); s.startObject(); @@ -107,7 +104,7 @@ void IndexAxis::encodeCurrent(eckit::Stream &s, const int version) const { s << "axes"; for (AxisMap::const_iterator i = axis_.begin(); i != axis_.end(); ++i) { s << (*i).first; - const eckit::DenseSet &values = *(*i).second; + const eckit::DenseSet& values = *(*i).second; s << values.size(); for (eckit::DenseSet::const_iterator j = values.begin(); j != values.end(); ++j) { s << (*j); @@ -116,13 +113,13 @@ void IndexAxis::encodeCurrent(eckit::Stream &s, const int version) const { s.endObject(); } -void IndexAxis::encodeLegacy(eckit::Stream &s, const int version) const { +void IndexAxis::encodeLegacy(eckit::Stream& s, const int version) const { ASSERT(version <= 2); s << axis_.size(); for (AxisMap::const_iterator i = axis_.begin(); i != axis_.end(); ++i) { s << (*i).first; - const eckit::DenseSet &values = *(*i).second; + const eckit::DenseSet& values = *(*i).second; s << values.size(); for (eckit::DenseSet::const_iterator j = values.begin(); j != values.end(); ++j) { s << (*j); @@ -130,34 +127,29 @@ void IndexAxis::encodeLegacy(eckit::Stream &s, const int version) const { } } - -void IndexAxis::decode(eckit::Stream &s, const int version) { +void IndexAxis::decode(eckit::Stream& s, const int version) { if (version >= 3) decodeCurrent(s, version); else decodeLegacy(s, version); } -enum IndexAxisStreamKeys { - IndexAxisKeyUnrecognised, - IndexAxisSize, - IndexAxes -}; +enum IndexAxisStreamKeys { IndexAxisKeyUnrecognised, IndexAxisSize, IndexAxes }; IndexAxisStreamKeys indexAxiskeyId(const std::string& s) { - static const std::map keys { + static const std::map keys{ {"size", IndexAxisSize}, {"axes", IndexAxes}, }; auto it = keys.find(s); - if( it != keys.end() ) { + if (it != keys.end()) { return it->second; } - return IndexAxisKeyUnrecognised; + return IndexAxisKeyUnrecognised; } -void IndexAxis::decodeCurrent(eckit::Stream &s, const int version) { +void IndexAxis::decodeCurrent(eckit::Stream& s, const int version) { ASSERT(version >= 3); ASSERT(s.next()); @@ -189,7 +181,7 @@ void IndexAxis::decodeCurrent(eckit::Stream &s, const int version) { } break; default: - throw eckit::SeriousBug("IndexBase de-serialization error: "+k+" field is not recognized"); + throw eckit::SeriousBug("IndexBase de-serialization error: " + k + " field is not recognized"); } } ASSERT(!axis_.empty()); @@ -220,23 +212,22 @@ void IndexAxis::decodeLegacy(eckit::Stream& s, const int version) { } } -void IndexAxis::dump(std::ostream &out, const char* indent) const { +void IndexAxis::dump(std::ostream& out, const char* indent) const { out << indent << "Axes:" << std::endl; - for (AxisMap::const_iterator i = axis_.begin(); i != axis_.end(); ++i) { + for (AxisMap::const_iterator i = axis_.begin(); i != axis_.end(); ++i) { out << indent << indent << (*i).first << std::endl; - const eckit::DenseSet &values = *(*i).second; + const eckit::DenseSet& values = *(*i).second; for (eckit::DenseSet::const_iterator j = values.begin(); j != values.end(); ++j) { out << indent << indent << indent; if ((*j).empty()) { out << ""; - } - else { + } else { out << (*j); } - out << std::endl; + out << std::endl; } } - // out << std::endl; + // out << std::endl; } bool IndexAxis::partialMatch(const metkit::mars::MarsRequest& request, const TypesRegistry& registry) const { @@ -256,16 +247,19 @@ bool IndexAxis::partialMatch(const metkit::mars::MarsRequest& request, const Typ for (const auto& rqval : request.values(kv.first)) { if (kv.second->contains(rqval)) { found = true; - break;; + break; + ; } std::string canonical_rqval = registry.lookupType(kv.first).toKey(rqval); if (kv.second->contains(canonical_rqval)) { found = true; - break;; + break; + ; } } - if (!found) return false; + if (!found) + return false; } } @@ -288,7 +282,7 @@ void IndexAxis::insert(const Key& key) { for (const auto& k : key) { - std::shared_ptr >& axis_set = axis_[k.first]; + std::shared_ptr>& axis_set = axis_[k.first]; if (!axis_set) axis_set.reset(new eckit::DenseSet); @@ -298,35 +292,34 @@ void IndexAxis::insert(const Key& key) { } } -/// @note: this method inserts key-value pairs into an axis in memory. +/// @note: this method inserts key-value pairs into an axis in memory. /// Intended for importing axis information from storage in the DAOS backend. /// Input values are required to be cannoicalised. void IndexAxis::insert(const std::string& axis, const std::vector& values) { ASSERT(!readOnly_); - std::shared_ptr >& axis_set = axis_[axis]; + std::shared_ptr>& axis_set = axis_[axis]; if (!axis_set) axis_set.reset(new eckit::DenseSet()); - for (const auto& value : values) axis_set->insert(value); + for (const auto& value : values) + axis_set->insert(value); dirty_ = true; - } bool IndexAxis::dirty() const { return dirty_; } - void IndexAxis::clean() { dirty_ = false; } void IndexAxis::sort() { for (AxisMap::iterator i = axis_.begin(); i != axis_.end(); ++i) - i->second->sort(); + i->second->sort(); } void IndexAxis::wipe() { @@ -337,12 +330,12 @@ void IndexAxis::wipe() { clean(); } -bool IndexAxis::has(const std::string &keyword) const { +bool IndexAxis::has(const std::string& keyword) const { AxisMap::const_iterator i = axis_.find(keyword); return (i != axis_.end()); } -const eckit::DenseSet &IndexAxis::values(const std::string &keyword) const { +const eckit::DenseSet& IndexAxis::values(const std::string& keyword) const { // If an Index is empty, this is bad, but is not strictly an error. Nothing will // be found... @@ -371,9 +364,9 @@ std::map> IndexAxis::map() const { return result; } -void IndexAxis::print(std::ostream &out) const { +void IndexAxis::print(std::ostream& out) const { out << "IndexAxis[" - << "axis="; + << "axis="; const char* sep = ""; out << "{"; @@ -388,7 +381,7 @@ void IndexAxis::print(std::ostream &out) const { sep = ","; } out << "}"; - out << "]"; + out << "]"; } void IndexAxis::json(eckit::JSON& json) const { diff --git a/src/fdb5/database/IndexAxis.h b/src/fdb5/database/IndexAxis.h index d57c3b51c..301f978fb 100644 --- a/src/fdb5/database/IndexAxis.h +++ b/src/fdb5/database/IndexAxis.h @@ -21,8 +21,8 @@ #include #include "eckit/container/DenseSet.h" -#include "eckit/memory/NonCopyable.h" #include "eckit/filesystem/PathName.h" +#include "eckit/memory/NonCopyable.h" #include "eckit/types/Types.h" namespace eckit { @@ -43,9 +43,8 @@ class TypesRegistry; class IndexAxis : private eckit::NonCopyable { public: // methods - IndexAxis(); - IndexAxis(eckit::Stream &s, const int version); + IndexAxis(eckit::Stream& s, const int version); IndexAxis(IndexAxis&& rhs) noexcept; IndexAxis& operator=(IndexAxis&& rhs) noexcept; @@ -59,7 +58,7 @@ class IndexAxis : private eckit::NonCopyable { /// @note: the values are required to be cannonicalised void insert(const std::string& axis, const std::vector& values); size_t encodeSize(const int version) const; - void encode(eckit::Stream &s, const int version) const; + void encode(eckit::Stream& s, const int version) const; static int currentVersion() { return 3; } void merge(const IndexAxis& other); @@ -67,12 +66,12 @@ class IndexAxis : private eckit::NonCopyable { // Decode can be used for two-stage initialisation (IndexAxis a; a.decode(s);) void decode(eckit::Stream& s, const int version); - bool has(const std::string &keyword) const; - const eckit::DenseSet &values(const std::string &keyword) const; + bool has(const std::string& keyword) const; + const eckit::DenseSet& values(const std::string& keyword) const; std::map> map() const; - void dump(std::ostream &out, const char* indent) const; + void dump(std::ostream& out, const char* indent) const; bool partialMatch(const metkit::mars::MarsRequest& request, const TypesRegistry& registry) const; bool contains(const Key& key) const; @@ -88,7 +87,7 @@ class IndexAxis : private eckit::NonCopyable { /// Reset the axis to a default state. void wipe(); - friend std::ostream &operator<<(std::ostream &s, const IndexAxis &x) { + friend std::ostream& operator<<(std::ostream& s, const IndexAxis& x) { x.print(s); return s; } @@ -99,24 +98,21 @@ class IndexAxis : private eckit::NonCopyable { } private: // methods - - void encodeCurrent(eckit::Stream &s, const int version) const; - void encodeLegacy(eckit::Stream &s, const int version) const; + void encodeCurrent(eckit::Stream& s, const int version) const; + void encodeLegacy(eckit::Stream& s, const int version) const; void decodeCurrent(eckit::Stream& s, const int version); void decodeLegacy(eckit::Stream& s, const int version); - void print(std::ostream &out) const; + void print(std::ostream& out) const; void json(eckit::JSON& j) const; private: // members - typedef std::map>> AxisMap; AxisMap axis_; bool readOnly_; bool dirty_; - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/IndexFactory.cc b/src/fdb5/database/IndexFactory.cc index cf7b7cddb..4414d84ad 100644 --- a/src/fdb5/database/IndexFactory.cc +++ b/src/fdb5/database/IndexFactory.cc @@ -8,8 +8,8 @@ * does it submit to any jurisdiction. */ -#include "eckit/thread/AutoLock.h" #include "eckit/exception/Exceptions.h" +#include "eckit/thread/AutoLock.h" #include "fdb5/LibFdb5.h" #include "fdb5/database/IndexFactory.h" @@ -18,18 +18,17 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -static eckit::Mutex *local_mutex = 0; -static std::map *m = 0; +static eckit::Mutex* local_mutex = 0; +static std::map* m = 0; static pthread_once_t once = PTHREAD_ONCE_INIT; static void init() { local_mutex = new eckit::Mutex(); - m = new std::map(); + m = new std::map(); } //---------------------------------------------------------------------------------------------------------------------- -BTreeIndexFactory::BTreeIndexFactory(const std::string &name) : - name_(name) { +BTreeIndexFactory::BTreeIndexFactory(const std::string& name) : name_(name) { pthread_once(&once, init); eckit::AutoLock lock(local_mutex); @@ -38,21 +37,23 @@ BTreeIndexFactory::BTreeIndexFactory(const std::string &name) : } BTreeIndexFactory::~BTreeIndexFactory() { - if(LibFdb5::instance().dontDeregisterFactories()) return; + if (LibFdb5::instance().dontDeregisterFactories()) + return; eckit::AutoLock lock(local_mutex); m->erase(name_); } -BTreeIndex* BTreeIndexFactory::build(const std::string &name, const eckit::PathName &path, bool readOnly, off_t offset) { +BTreeIndex* BTreeIndexFactory::build(const std::string& name, const eckit::PathName& path, bool readOnly, + off_t offset) { pthread_once(&once, init); eckit::AutoLock lock(local_mutex); - std::map::const_iterator j = m->find(name); + std::map::const_iterator j = m->find(name); if (j == m->end()) { eckit::Log::error() << "No IndexFactory for [" << name << "]" << std::endl; eckit::Log::error() << "Values are:" << std::endl; - for (j = m->begin() ; j != m->end() ; ++j) + for (j = m->begin(); j != m->end(); ++j) eckit::Log::error() << " " << (*j).first << std::endl; throw eckit::SeriousBug(std::string("No IndexFactory called ") + name); } diff --git a/src/fdb5/database/IndexFactory.h b/src/fdb5/database/IndexFactory.h index 2eae719b5..17d143921 100644 --- a/src/fdb5/database/IndexFactory.h +++ b/src/fdb5/database/IndexFactory.h @@ -21,7 +21,9 @@ #include "eckit/memory/NonCopyable.h" #include "eckit/types/Types.h" -namespace eckit { class PathName; } +namespace eckit { +class PathName; +} namespace fdb5 { @@ -33,34 +35,31 @@ class BTreeIndex; class BTreeIndexFactory { - virtual BTreeIndex *make(const eckit::PathName &path, bool readOnly, off_t offset) const = 0 ; + virtual BTreeIndex* make(const eckit::PathName& path, bool readOnly, off_t offset) const = 0; protected: - - BTreeIndexFactory(const std::string &); + BTreeIndexFactory(const std::string&); virtual ~BTreeIndexFactory(); std::string name_; public: - - static void list(std::ostream &); - static BTreeIndex *build(const std::string &name, const eckit::PathName &path, bool readOnly, off_t offset); - + static void list(std::ostream&); + static BTreeIndex* build(const std::string& name, const eckit::PathName& path, bool readOnly, off_t offset); }; /// Templated specialisation of the self-registering factory, /// that does the self-registration, and the construction of each object. -template< class T> +template class IndexBuilder : public BTreeIndexFactory { - BTreeIndex *make(const eckit::PathName &path, bool readOnly, off_t offset) const override { + BTreeIndex* make(const eckit::PathName& path, bool readOnly, off_t offset) const override { return new T(path, readOnly, offset); } public: - IndexBuilder(const std::string &name) : BTreeIndexFactory(name) {} + IndexBuilder(const std::string& name) : BTreeIndexFactory(name) {} }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/IndexLocation.cc b/src/fdb5/database/IndexLocation.cc index 8bb87a69b..1ceb555e5 100644 --- a/src/fdb5/database/IndexLocation.cc +++ b/src/fdb5/database/IndexLocation.cc @@ -14,7 +14,10 @@ using namespace eckit; namespace fdb5 { -::eckit::ClassSpec IndexLocation::classSpec_ = {&Streamable::classSpec(), "IndexLocation",}; +::eckit::ClassSpec IndexLocation::classSpec_ = { + &Streamable::classSpec(), + "IndexLocation", +}; //---------------------------------------------------------------------------------------------------------------------- @@ -22,8 +25,8 @@ IndexLocation::IndexLocation() {} IndexLocationVisitor::~IndexLocationVisitor() {} -IndexLocation::~IndexLocation(){} +IndexLocation::~IndexLocation() {} //---------------------------------------------------------------------------------------------------------------------- -} +} // namespace fdb5 diff --git a/src/fdb5/database/IndexLocation.h b/src/fdb5/database/IndexLocation.h index e400a1afa..e3efe1db4 100644 --- a/src/fdb5/database/IndexLocation.h +++ b/src/fdb5/database/IndexLocation.h @@ -8,10 +8,9 @@ * does it submit to any jurisdiction. */ - -#include "eckit/memory/NonCopyable.h" #include "eckit/filesystem/PathName.h" #include "eckit/filesystem/URI.h" +#include "eckit/memory/NonCopyable.h" #include "eckit/serialisation/Streamable.h" #ifndef fdb5_IndexLocation_H @@ -24,28 +23,23 @@ namespace fdb5 { class IndexLocation : public eckit::Streamable { public: // methods - IndexLocation(); ~IndexLocation() override; -// virtual eckit::PathName path() const = 0; + // virtual eckit::PathName path() const = 0; virtual eckit::URI uri() const = 0; virtual IndexLocation* clone() const = 0; private: // methods - virtual void print(std::ostream& out) const = 0; - protected: // For streamable - void encode(eckit::Stream&) const override = 0; static eckit::ClassSpec classSpec_; private: // friends - friend std::ostream& operator<<(std::ostream& s, const IndexLocation& x) { x.print(s); return s; @@ -57,14 +51,13 @@ class IndexLocation : public eckit::Streamable { class IndexLocationVisitor { public: // methods - virtual ~IndexLocationVisitor(); - virtual void operator() (const IndexLocation&) = 0; + virtual void operator()(const IndexLocation&) = 0; }; //---------------------------------------------------------------------------------------------------------------------- -} +} // namespace fdb5 #endif // fdb5_IndexLocation_H diff --git a/src/fdb5/database/IndexStats.cc b/src/fdb5/database/IndexStats.cc index 103b2355a..64cf39391 100644 --- a/src/fdb5/database/IndexStats.cc +++ b/src/fdb5/database/IndexStats.cc @@ -14,18 +14,20 @@ namespace fdb5 { -::eckit::ClassSpec IndexStatsContent::classSpec_ = {&Streamable::classSpec(), "IndexStatsContent",}; +::eckit::ClassSpec IndexStatsContent::classSpec_ = { + &Streamable::classSpec(), + "IndexStatsContent", +}; //---------------------------------------------------------------------------------------------------------------------- class NullIndexStats : public IndexStatsContent { public: - - virtual size_t fieldsCount() const { return 0; } + virtual size_t fieldsCount() const { return 0; } virtual size_t duplicatesCount() const { return 0; } - virtual size_t fieldsSize() const { return 0; } - virtual size_t duplicatesSize() const { return 0; } + virtual size_t fieldsSize() const { return 0; } + virtual size_t duplicatesSize() const { return 0; } virtual size_t addFieldsCount(size_t i) { return i; } virtual size_t addDuplicatesCount(size_t i) { return i; } @@ -33,34 +35,25 @@ class NullIndexStats : public IndexStatsContent { virtual size_t addFieldsSize(size_t i) { return i; } virtual size_t addDuplicatesSize(size_t i) { return i; } - virtual void add(const IndexStatsContent&) { - NOTIMP; - } + virtual void add(const IndexStatsContent&) { NOTIMP; } - virtual void report(std::ostream& out, const char* indent) const { - NOTIMP; - } + virtual void report(std::ostream& out, const char* indent) const { NOTIMP; } - virtual void encode(eckit::Stream& s) const { - NOTIMP; - } + virtual void encode(eckit::Stream& s) const { NOTIMP; } }; //---------------------------------------------------------------------------------------------------------------------- - -IndexStats::IndexStats() : - content_(new NullIndexStats()) { +IndexStats::IndexStats() : content_(new NullIndexStats()) { content_->attach(); } -IndexStats::IndexStats(IndexStatsContent* p) : - content_(p) { +IndexStats::IndexStats(IndexStatsContent* p) : content_(p) { content_->attach(); } IndexStats::~IndexStats() { - content_->detach(); + content_->detach(); } IndexStats::IndexStats(const IndexStats& s) : content_(s.content_) { @@ -79,13 +72,11 @@ IndexStats& IndexStats::operator+=(const IndexStats& s) { return *this; } -void IndexStats::add(const IndexStats& s) -{ +void IndexStats::add(const IndexStats& s) { content_->add(*s.content_); } -void IndexStats::report(std::ostream& out, const char* indent) const -{ +void IndexStats::report(std::ostream& out, const char* indent) const { content_->report(out, indent); } @@ -93,10 +84,7 @@ void IndexStats::encode(eckit::Stream& s) const { s << *content_; } -IndexStatsContent::~IndexStatsContent() -{ -} - +IndexStatsContent::~IndexStatsContent() {} //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/IndexStats.h b/src/fdb5/database/IndexStats.h index 0dcd00987..9a35d2db0 100644 --- a/src/fdb5/database/IndexStats.h +++ b/src/fdb5/database/IndexStats.h @@ -26,11 +26,8 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -class IndexStatsContent : public eckit::Counted, - public eckit::Statistics, - public eckit::Streamable { +class IndexStatsContent : public eckit::Counted, public eckit::Statistics, public eckit::Streamable { public: - ~IndexStatsContent() override; virtual size_t fieldsCount() const = 0; @@ -50,12 +47,10 @@ class IndexStatsContent : public eckit::Counted, virtual void report(std::ostream& out, const char* indent) const = 0; public: // For Streamable - void encode(eckit::Stream& s) const override = 0; protected: // For Streamable - - static eckit::ClassSpec classSpec_; + static eckit::ClassSpec classSpec_; }; //---------------------------------------------------------------------------------------------------------------------- @@ -65,7 +60,6 @@ class IndexStatsContent : public eckit::Counted, class IndexStats { public: // methods - IndexStats(); IndexStats(IndexStatsContent*); @@ -75,7 +69,7 @@ class IndexStats { IndexStats& operator=(const IndexStats&); - IndexStats& operator+= (const IndexStats& rhs); + IndexStats& operator+=(const IndexStats& rhs); size_t fieldsCount() const { return content_->fieldsCount(); } size_t duplicatesCount() const { return content_->duplicatesCount(); } @@ -93,18 +87,18 @@ class IndexStats { void report(std::ostream& out, const char* indent = "") const; -// template -// T& as() { -// return dynamic_cast(*content_); -// } + // template + // T& as() { + // return dynamic_cast(*content_); + // } private: // methods - void print(std::ostream&) const; void encode(eckit::Stream& s) const; friend std::ostream& operator<<(std::ostream& s, const IndexStats& o) { - o.print(s); return s; + o.print(s); + return s; } friend eckit::Stream& operator<<(eckit::Stream& s, const IndexStats& r) { @@ -113,7 +107,6 @@ class IndexStats { } private: // members - IndexStatsContent* content_; }; diff --git a/src/fdb5/database/Indexer.cc b/src/fdb5/database/Indexer.cc index 023a0c041..448739e5a 100644 --- a/src/fdb5/database/Indexer.cc +++ b/src/fdb5/database/Indexer.cc @@ -19,19 +19,12 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -Indexer::Indexer() : - pid_(::getpid()), - thread_(::pthread_self()) -{ +Indexer::Indexer() : pid_(::getpid()), thread_(::pthread_self()) { hostname_ = eckit::Main::hostname(); } -void Indexer::print(std::ostream& out) const -{ - out << "(host=" << hostname_ - << ",pid=" << pid_ - << ",thread=" << thread_ - << ")"; +void Indexer::print(std::ostream& out) const { + out << "(host=" << hostname_ << ",pid=" << pid_ << ",thread=" << thread_ << ")"; } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/Indexer.h b/src/fdb5/database/Indexer.h index 034aa0e80..4115f0a99 100644 --- a/src/fdb5/database/Indexer.h +++ b/src/fdb5/database/Indexer.h @@ -27,9 +27,9 @@ struct Indexer { Indexer(); - void print(std::ostream &out) const; + void print(std::ostream& out) const; - friend std::ostream &operator<<(std::ostream& s, const Indexer& o) { + friend std::ostream& operator<<(std::ostream& s, const Indexer& o) { o.print(s); return s; } diff --git a/src/fdb5/database/Inspector.cc b/src/fdb5/database/Inspector.cc index b25f36cab..536b70840 100644 --- a/src/fdb5/database/Inspector.cc +++ b/src/fdb5/database/Inspector.cc @@ -17,8 +17,8 @@ #include "metkit/mars/MarsRequest.h" #include "fdb5/LibFdb5.h" -#include "fdb5/database/Notifier.h" #include "fdb5/database/MultiRetrieveVisitor.h" +#include "fdb5/database/Notifier.h" #include "fdb5/io/HandleGatherer.h" #include "fdb5/rules/Schema.h" @@ -53,15 +53,13 @@ static void purgeCatalogue(Key& key, CatalogueReader*& db) { delete db; } -Inspector::Inspector(const Config& dbConfig) : - databases_(Resource("fdbMaxOpenDatabases", 16), &purgeCatalogue), - dbConfig_(dbConfig) {} +Inspector::Inspector(const Config& dbConfig) + : databases_(Resource("fdbMaxOpenDatabases", 16), &purgeCatalogue) + , dbConfig_(dbConfig) {} -Inspector::~Inspector() { -} +Inspector::~Inspector() {} -ListIterator Inspector::inspect(const metkit::mars::MarsRequest& request, - const Schema& schema, +ListIterator Inspector::inspect(const metkit::mars::MarsRequest& request, const Schema& schema, const fdb5::Notifier& notifyee) const { InspectIterator* iterator = new InspectIterator(); @@ -89,11 +87,9 @@ ListIterator Inspector::inspect(const metkit::mars::MarsRequest& request, const return inspect(request, dbConfig_.schema(), notifyee); } -void Inspector::visitEntries(const FDBToolRequest &request, EntryVisitor &visitor) const { - -} +void Inspector::visitEntries(const FDBToolRequest& request, EntryVisitor& visitor) const {} -void Inspector::print(std::ostream &out) const { +void Inspector::print(std::ostream& out) const { out << "Inspector[]"; } diff --git a/src/fdb5/database/Inspector.h b/src/fdb5/database/Inspector.h index 7066425d3..989beaead 100644 --- a/src/fdb5/database/Inspector.h +++ b/src/fdb5/database/Inspector.h @@ -16,16 +16,16 @@ #ifndef fdb5_Inspector_H #define fdb5_Inspector_H -#include #include +#include #include -#include "fdb5/config/Config.h" #include "fdb5/api/helpers/ListIterator.h" +#include "fdb5/config/Config.h" -#include "eckit/memory/NonCopyable.h" -#include "eckit/container/CacheLRU.h" #include "eckit/config/LocalConfiguration.h" +#include "eckit/container/CacheLRU.h" +#include "eckit/memory/NonCopyable.h" namespace eckit { class DataHandle; @@ -33,8 +33,9 @@ class DataHandle; namespace metkit { namespace mars { - class MarsRequest; -}} +class MarsRequest; +} +} // namespace metkit namespace fdb5 { @@ -55,6 +56,7 @@ class InspectIterator : public APIIteratorBase { void emplace(ListElement&& elem); bool next(ListElement& elem) override; + private: std::vector queue_; size_t index_; @@ -65,7 +67,6 @@ class InspectIterator : public APIIteratorBase { class Inspector : public eckit::NonCopyable { public: // methods - Inspector(const Config& dbConfig); ~Inspector(); @@ -85,20 +86,19 @@ class Inspector : public eckit::NonCopyable { void visitEntries(const FDBToolRequest& request, EntryVisitor& visitor) const; - friend std::ostream &operator<<(std::ostream &s, const Inspector &x) { + friend std::ostream& operator<<(std::ostream& s, const Inspector& x) { x.print(s); return s; } private: // methods + void print(std::ostream& out) const; - void print(std::ostream &out) const; - - ListIterator inspect(const metkit::mars::MarsRequest& request, const Schema &schema, const Notifier& notifyee) const; + ListIterator inspect(const metkit::mars::MarsRequest& request, const Schema& schema, + const Notifier& notifyee) const; private: // data - - mutable eckit::CacheLRU databases_; + mutable eckit::CacheLRU databases_; Config dbConfig_; }; diff --git a/src/fdb5/database/Key.cc b/src/fdb5/database/Key.cc index bd1abee5c..aedc02b6e 100644 --- a/src/fdb5/database/Key.cc +++ b/src/fdb5/database/Key.cc @@ -14,8 +14,8 @@ #include "eckit/config/Resource.h" #include "eckit/container/DenseSet.h" -#include "eckit/utils/Tokenizer.h" #include "eckit/utils/StringTools.h" +#include "eckit/utils/Tokenizer.h" #include "metkit/mars/MarsRequest.h" @@ -106,7 +106,7 @@ void BaseKey::clear() { names_.clear(); } -void BaseKey::set(const std::string &k, const std::string &v) { +void BaseKey::set(const std::string& k, const std::string& v) { ASSERT(names_.size() == keys_.size()); @@ -119,24 +119,24 @@ void BaseKey::set(const std::string &k, const std::string &v) { } } -void BaseKey::unset(const std::string &k) { +void BaseKey::unset(const std::string& k) { keys_.erase(k); } -void BaseKey::push(const std::string &k, const std::string &v) { +void BaseKey::push(const std::string& k, const std::string& v) { keys_[k] = v; names_.push_back(k); } -void BaseKey::pop(const std::string &k) { +void BaseKey::pop(const std::string& k) { keys_.erase(k); ASSERT(names_.back() == k); names_.pop_back(); } -const std::string &BaseKey::get( const std::string &k ) const { +const std::string& BaseKey::get(const std::string& k) const { eckit::StringDict::const_iterator i = keys_.find(k); - if ( i == keys_.end() ) { + if (i == keys_.end()) { std::ostringstream oss; oss << "BaseKey::get() failed for [" + k + "] in " << *this; throw eckit::SeriousBug(oss.str(), Here()); @@ -157,12 +157,10 @@ bool BaseKey::match(const BaseKey& other) const { if (j->second != i->second && !i->second.empty()) { return false; } - } return true; } - bool BaseKey::match(const metkit::mars::MarsRequest& request) const { for (const auto& k : request.params()) { @@ -172,7 +170,7 @@ bool BaseKey::match(const metkit::mars::MarsRequest& request) const { return false; } - bool found=false; + bool found = false; const auto& values = request.values(k); std::string can = canonicalise(k, j->second); for (auto it = values.cbegin(); !found && it != values.cend(); it++) { @@ -186,7 +184,7 @@ bool BaseKey::match(const metkit::mars::MarsRequest& request) const { return true; } -bool BaseKey::match(const std::string &key, const eckit::DenseSet &values) const { +bool BaseKey::match(const std::string& key, const eckit::DenseSet& values) const { eckit::StringDict::const_iterator i = find(key); if (i == end()) { @@ -222,17 +220,17 @@ std::string BaseKey::canonicalValue(const std::string& keyword) const { std::string BaseKey::valuesToString() const { - if(names_.size() != keys_.size()) { + if (names_.size() != keys_.size()) { std::stringstream ss; ss << "names and keys size mismatch" << std::endl - << " names: " << names_.size() << " " << names_ << std::endl - << " keys: " << keys_.size() << " " << keys_ << std::endl; + << " names: " << names_.size() << " " << names_ << std::endl + << " keys: " << keys_.size() << " " << keys_ << std::endl; throw eckit::SeriousBug(ss.str()); } std::ostringstream oss; - const char *sep = ""; + const char* sep = ""; for (eckit::StringList::const_iterator j = names_.begin(); j != names_.end(); ++j) { eckit::StringDict::const_iterator i = keys_.find(*j); @@ -240,13 +238,12 @@ std::string BaseKey::valuesToString() const { oss << sep; oss << canonicalise(*j, i->second); - + sep = ":"; } return oss.str(); } - const eckit::StringList& BaseKey::names() const { return names_; } @@ -258,7 +255,7 @@ std::string BaseKey::value(const std::string& key) const { return it->second; } -const eckit::StringDict &BaseKey::keyDict() const { +const eckit::StringDict& BaseKey::keyDict() const { return keys_; } @@ -272,14 +269,12 @@ metkit::mars::MarsRequest BaseKey::request(const std::string& verb) const { return req; } - fdb5::BaseKey::operator std::string() const { ASSERT(names_.size() == keys_.size()); return toString(); } -fdb5::BaseKey::operator eckit::StringDict() const -{ +fdb5::BaseKey::operator eckit::StringDict() const { eckit::StringDict res; ASSERT(names_.size() == keys_.size()); @@ -297,7 +292,7 @@ fdb5::BaseKey::operator eckit::StringDict() const return res; } -void BaseKey::print(std::ostream &out) const { +void BaseKey::print(std::ostream& out) const { if (names_.size() == keys_.size()) { out << "{" << toString() << "}"; } else { @@ -307,7 +302,7 @@ void BaseKey::print(std::ostream &out) const { std::string BaseKey::toString() const { std::string res; - const char *sep = ""; + const char* sep = ""; for (eckit::StringList::const_iterator j = names_.begin(); j != names_.end(); ++j) { eckit::StringDict::const_iterator i = keys_.find(*j); ASSERT(i != keys_.end()); @@ -319,21 +314,17 @@ std::string BaseKey::toString() const { return res; } - //---------------------------------------------------------------------------------------------------------------------- -Key::Key(const eckit::StringDict &keys) : - BaseKey(keys) {} +Key::Key(const eckit::StringDict& keys) : BaseKey(keys) {} Key::Key(eckit::Stream& s) { decode(s); } -Key::Key(std::initializer_list> l) : - BaseKey(l) {} +Key::Key(std::initializer_list> l) : BaseKey(l) {} -Key::Key(const std::string& fingerprint, const Rule& rule) : - BaseKey(fingerprint, rule) {} +Key::Key(const std::string& fingerprint, const Rule& rule) : BaseKey(fingerprint, rule) {} Key Key::parseString(const std::string& s) { @@ -363,30 +354,25 @@ std::string Key::type(const std::string& keyword) const { //---------------------------------------------------------------------------------------------------------------------- -TypedKey::TypedKey(const Key& key, const TypesRegistry& reg) : - BaseKey(key), registry_(std::cref(reg)) {} +TypedKey::TypedKey(const Key& key, const TypesRegistry& reg) : BaseKey(key), registry_(std::cref(reg)) {} -TypedKey::TypedKey(const TypesRegistry& reg) : - registry_(std::cref(reg)) {} +TypedKey::TypedKey(const TypesRegistry& reg) : registry_(std::cref(reg)) {} -TypedKey::TypedKey(const std::string &s, const Rule& rule) : - BaseKey(s, rule), - registry_(std::cref(rule.registry())) { -} +TypedKey::TypedKey(const std::string& s, const Rule& rule) : BaseKey(s, rule), registry_(std::cref(rule.registry())) {} -TypedKey::TypedKey(const eckit::StringDict &keys, const TypesRegistry& reg) : - BaseKey(keys), registry_(std::cref(reg)) { -} +TypedKey::TypedKey(const eckit::StringDict& keys, const TypesRegistry& reg) + : BaseKey(keys) + , registry_(std::cref(reg)) {} -TypedKey::TypedKey(eckit::Stream& s, const TypesRegistry& reg) : - registry_(std::cref(reg)) { +TypedKey::TypedKey(eckit::Stream& s, const TypesRegistry& reg) : registry_(std::cref(reg)) { decode(s); } -TypedKey::TypedKey(std::initializer_list> l, const TypesRegistry& reg) : - BaseKey(l), registry_(std::cref(reg)) {} +TypedKey::TypedKey(std::initializer_list> l, const TypesRegistry& reg) + : BaseKey(l) + , registry_(std::cref(reg)) {} -TypedKey TypedKey::parseString(const std::string &s, const TypesRegistry& registry) { +TypedKey TypedKey::parseString(const std::string& s, const TypesRegistry& registry) { eckit::Tokenizer parse1(","); eckit::Tokenizer parse2("="); @@ -422,9 +408,8 @@ void TypedKey::validateKeys(const BaseKey& other, bool checkAlsoValues) const { BaseKey::const_iterator k = find(keyword); if (k == keys_.end()) { missing.insert(keyword); - } - else { - if(checkAlsoValues && !registry_.get().lookupType(keyword).match(keyword, j->second, k->second)) { + } else { + if (checkAlsoValues && !registry_.get().lookupType(keyword).match(keyword, j->second, k->second)) { mismatch.insert((*j).first + "=" + j->second + " and " + k->second); } } @@ -433,11 +418,11 @@ void TypedKey::validateKeys(const BaseKey& other, bool checkAlsoValues) const { if (missing.size() || mismatch.size()) { std::ostringstream oss; - if(missing.size()) { + if (missing.size()) { oss << "Keywords not used: " << missing << " "; } - if(mismatch.size()) { + if (mismatch.size()) { oss << "Values mismatch: " << mismatch << " "; } @@ -470,7 +455,7 @@ std::string TypedKey::type(const std::string& keyword) const { Key TypedKey::canonical() const { Key key{}; - for (const auto& name: names_) { + for (const auto& name : names_) { auto m = keys_.find(name); ASSERT(m != keys_.end()); diff --git a/src/fdb5/database/Key.h b/src/fdb5/database/Key.h index 523a7b015..fb0dce5b8 100644 --- a/src/fdb5/database/Key.h +++ b/src/fdb5/database/Key.h @@ -16,27 +16,28 @@ #ifndef fdb5_Key_H #define fdb5_Key_H +#include #include +#include +#include #include #include -#include -#include -#include #include "eckit/serialisation/Stream.h" #include "eckit/types/Types.h" #include "fdb5/types/TypesRegistry.h" namespace eckit { - class JSON; - template class DenseSet; -} +class JSON; +template +class DenseSet; +} // namespace eckit namespace metkit { namespace mars { - class MarsRequest; -} +class MarsRequest; } +} // namespace metkit namespace fdb5 { @@ -47,11 +48,10 @@ class Rule; class BaseKey { public: // methods - BaseKey() = default; - BaseKey(const BaseKey &key) = default; + BaseKey(const BaseKey& key) = default; - explicit BaseKey(const eckit::StringDict &keys) : keys_(keys) { + explicit BaseKey(const eckit::StringDict& keys) : keys_(keys) { for (const auto& k : keys) { names_.emplace_back(k.first); } @@ -67,13 +67,13 @@ class BaseKey { std::set keys() const; - void set(const std::string &k, const std::string &v); - void unset(const std::string &k); + void set(const std::string& k, const std::string& v); + void unset(const std::string& k); - void push(const std::string &k, const std::string &v); - void pop(const std::string &k); + void push(const std::string& k, const std::string& v); + void pop(const std::string& k); - const std::string& get( const std::string &k ) const; + const std::string& get(const std::string& k) const; void clear(); @@ -86,24 +86,18 @@ class BaseKey { /// keys present in the key. Essentially implements a reject-filter bool partialMatch(const metkit::mars::MarsRequest& request) const; - bool operator< (const BaseKey& other) const { - return keys_ < other.keys_; - } + bool operator<(const BaseKey& other) const { return keys_ < other.keys_; } - bool operator!= (const BaseKey& other) const { - return keys_ != other.keys_; - } + bool operator!=(const BaseKey& other) const { return keys_ != other.keys_; } - bool operator== (const BaseKey& other) const { - return keys_ == other.keys_; - } + bool operator==(const BaseKey& other) const { return keys_ == other.keys_; } - friend std::ostream& operator<<(std::ostream &s, const BaseKey& x) { + friend std::ostream& operator<<(std::ostream& s, const BaseKey& x) { x.print(s); return s; } - friend eckit::Stream& operator<<(eckit::Stream &s, const BaseKey& x) { + friend eckit::Stream& operator<<(eckit::Stream& s, const BaseKey& x) { x.encode(s); return s; } @@ -141,35 +135,30 @@ class BaseKey { size_t encodeSize() const; protected: // members - - //TODO add unit test for each type + // TODO add unit test for each type virtual std::string canonicalise(const std::string& keyword, const std::string& value) const = 0; virtual std::string type(const std::string& keyword) const = 0; - void print( std::ostream &out ) const; + void print(std::ostream& out) const; void decode(eckit::Stream& s); - void encode(eckit::Stream &s) const; + void encode(eckit::Stream& s) const; private: // methods - std::string toString() const; protected: // members - eckit::StringDict keys_; eckit::StringList names_; }; - //---------------------------------------------------------------------------------------------------------------------- class Key : public BaseKey { public: // methods - explicit Key() = default; - explicit Key(eckit::Stream &); - explicit Key(const eckit::StringDict &keys); + explicit Key(eckit::Stream&); + explicit Key(const eckit::StringDict& keys); explicit Key(const std::string& fingerprint, const Rule& rule); Key(std::initializer_list>); @@ -181,10 +170,8 @@ class Key : public BaseKey { } private: // members - std::string canonicalise(const std::string& keyword, const std::string& value) const override; std::string type(const std::string& keyword) const override; - }; //---------------------------------------------------------------------------------------------------------------------- @@ -192,12 +179,11 @@ class Key : public BaseKey { class TypedKey : public BaseKey { public: // methods - explicit TypedKey(const Key& key, const TypesRegistry& reg); explicit TypedKey(const TypesRegistry& reg); - explicit TypedKey(eckit::Stream &, const TypesRegistry& reg); - explicit TypedKey(const std::string &keys, const Rule& rule); - explicit TypedKey(const eckit::StringDict &keys, const TypesRegistry& reg); + explicit TypedKey(eckit::Stream&, const TypesRegistry& reg); + explicit TypedKey(const std::string& keys, const Rule& rule); + explicit TypedKey(const eckit::StringDict& keys, const TypesRegistry& reg); TypedKey(std::initializer_list>, const TypesRegistry& reg); static TypedKey parseString(const std::string&, const TypesRegistry& reg); @@ -211,12 +197,11 @@ class TypedKey : public BaseKey { // Registry is needed before we can stringise/canonicalise. void registry(const TypesRegistry& reg); - [[ nodiscard ]] + [[nodiscard]] const TypesRegistry& registry() const; private: // members - - //TODO add unit test for each type + // TODO add unit test for each type std::string canonicalise(const std::string& keyword, const std::string& value) const override; std::string type(const std::string& keyword) const override; @@ -228,12 +213,10 @@ class TypedKey : public BaseKey { } // namespace fdb5 namespace std { - template <> - struct hash { - size_t operator() (const fdb5::Key& key) const { - return std::hash()(key.valuesToString()); - } - }; -} +template <> +struct hash { + size_t operator()(const fdb5::Key& key) const { return std::hash()(key.valuesToString()); } +}; +} // namespace std #endif diff --git a/src/fdb5/database/Manager.cc b/src/fdb5/database/Manager.cc index 3d888bf84..711f98916 100644 --- a/src/fdb5/database/Manager.cc +++ b/src/fdb5/database/Manager.cc @@ -8,25 +8,23 @@ * does it submit to any jurisdiction. */ - #include "fdb5/database/Manager.h" #include +#include "eckit/config/Resource.h" #include "eckit/log/Log.h" +#include "eckit/thread/AutoLock.h" +#include "eckit/thread/Mutex.h" #include "eckit/utils/Regex.h" -#include "eckit/config/Resource.h" #include "eckit/utils/Tokenizer.h" #include "eckit/utils/Translator.h" -#include "eckit/thread/Mutex.h" -#include "eckit/thread/AutoLock.h" #include "fdb5/LibFdb5.h" -#include "fdb5/database/Key.h" #include "fdb5/database/Engine.h" +#include "fdb5/database/Key.h" #include "fdb5/rules/Schema.h" - using namespace eckit; namespace fdb5 { @@ -35,14 +33,11 @@ namespace fdb5 { struct EngineType { - EngineType(const std::string& name, const std::string& re) : - name_(name), - re_(re) { - } + EngineType(const std::string& name, const std::string& re) : name_(name), re_(re) {} const std::string& engine() const { return name_; } - bool match(const std::string& s) const { return re_.match(s); } + bool match(const std::string& s) const { return re_.match(s); } std::string name_; eckit::Regex re_; @@ -57,7 +52,6 @@ typedef std::map EngineTableMap; static eckit::Mutex engineTypesMutex; static EngineTableMap engineTypes; - static const EngineTable& readEngineTypes(const eckit::PathName enginesFile) { eckit::AutoLock lock(engineTypesMutex); @@ -73,7 +67,7 @@ static const EngineTable& readEngineTypes(const eckit::PathName enginesFile) { // Sensible defaults if not configured - if(!enginesFile.exists()) { + if (!enginesFile.exists()) { LOG_DEBUG_LIB(LibFdb5) << "FDB Engines file not found: assuming Engine 'toc' Regex '.*'" << std::endl; table.push_back(EngineType("toc", ".*")); return table; @@ -113,17 +107,16 @@ static const EngineTable& readEngineTypes(const eckit::PathName enginesFile) { switch (s.size()) { case 2: { - const std::string& regex = s[0]; + const std::string& regex = s[0]; const std::string& engine = s[1]; table.push_back(EngineType(engine, regex)); break; } - default: - eckit::Log::warning() << "FDB Manager: Invalid line ignored: " << line << std::endl; - break; - + default: + eckit::Log::warning() << "FDB Manager: Invalid line ignored: " << line << std::endl; + break; } } @@ -132,10 +125,10 @@ static const EngineTable& readEngineTypes(const eckit::PathName enginesFile) { //---------------------------------------------------------------------------------------------------------------------- -Manager::Manager(const Config& config) : - config_(config) { +Manager::Manager(const Config& config) : config_(config) { - static std::string baseEnginesFile = eckit::Resource("fdbEnginesFile;$FDB_ENGINES_FILE", "~fdb/etc/fdb/engines"); + static std::string baseEnginesFile = + eckit::Resource("fdbEnginesFile;$FDB_ENGINES_FILE", "~fdb/etc/fdb/engines"); enginesFile_ = config.expandPath(baseEnginesFile); @@ -144,11 +137,10 @@ Manager::Manager(const Config& config) : Manager::~Manager() {} - -std::string Manager::engine(const Key& key) -{ +std::string Manager::engine(const Key& key) { // If we have set the engine in the config, use that - if (!explicitEngine_.empty()) return explicitEngine_; + if (!explicitEngine_.empty()) + return explicitEngine_; std::string expanded(key.valuesToString()); @@ -156,13 +148,12 @@ std::string Manager::engine(const Key& key) const EngineTable& engineTypes(readEngineTypes(enginesFile_)); - for (EngineTable::const_iterator i = engineTypes.begin(); i != engineTypes.end() ; ++i) { - if(i->match(expanded)) { + for (EngineTable::const_iterator i = engineTypes.begin(); i != engineTypes.end(); ++i) { + if (i->match(expanded)) { const std::string& engine = i->engine(); if (EngineRegistry::has(engine)) { return engine; - } - else { + } else { Log::warning() << "Configured FDB engine " << engine << " is not registered" << std::endl; } } @@ -174,8 +165,7 @@ std::string Manager::engine(const Key& key) throw eckit::SeriousBug(oss.str()); } -std::set Manager::engines(const Key& key) -{ +std::set Manager::engines(const Key& key) { std::set s; std::string expanded; @@ -189,14 +179,14 @@ std::set Manager::engines(const Key& key) expanded = key.valuesToString(); const EngineTable& engineTypes(readEngineTypes(enginesFile_)); - for (EngineTable::const_iterator i = engineTypes.begin(); i != engineTypes.end() ; ++i) { - if(key.empty() || i->match(expanded)) { + for (EngineTable::const_iterator i = engineTypes.begin(); i != engineTypes.end(); ++i) { + if (key.empty() || i->match(expanded)) { s.insert(i->engine()); } } } - if(s.empty()) { + if (s.empty()) { std::ostringstream oss; oss << "No FDB Engine type found for " << key << " (" << expanded << ")"; throw eckit::SeriousBug(oss.str()); @@ -205,8 +195,7 @@ std::set Manager::engines(const Key& key) return s; } -std::set Manager::engines(const metkit::mars::MarsRequest& rq, bool all) -{ +std::set Manager::engines(const metkit::mars::MarsRequest& rq, bool all) { std::set s; std::string expanded; @@ -218,7 +207,8 @@ std::set Manager::engines(const metkit::mars::MarsRequest& rq, bool const EngineTable& engineTypes(readEngineTypes(enginesFile_)); if (all) { - for (auto e = engineTypes.begin(); e != engineTypes.end(); ++e) s.insert(e->engine()); + for (auto e = engineTypes.begin(); e != engineTypes.end(); ++e) + s.insert(e->engine()); } else { // Match all possible expansions of the first level according to the schema @@ -232,13 +222,14 @@ std::set Manager::engines(const metkit::mars::MarsRequest& rq, bool for (auto e = engineTypes.begin(); e != engineTypes.end(); ++e) { for (auto expanded = expandedKeys.begin(); expanded != expandedKeys.end(); ++expanded) { - if (e->match(*expanded)) s.insert(e->engine()); + if (e->match(*expanded)) + s.insert(e->engine()); } } } } - if(s.empty()) { + if (s.empty()) { std::ostringstream oss; oss << "No FDB Engine type found for " << rq << " (" << expanded << ")"; throw eckit::SeriousBug(oss.str(), Here()); @@ -247,19 +238,19 @@ std::set Manager::engines(const metkit::mars::MarsRequest& rq, bool return s; } -std::string Manager::engine(const URI& uri) -{ +std::string Manager::engine(const URI& uri) { // If we have set the engine in the config, use that - if (!explicitEngine_.empty()) return explicitEngine_; + if (!explicitEngine_.empty()) + return explicitEngine_; // Otherwise, check which engines can handle the given path. std::vector engines = EngineRegistry::engines(); - for(std::vector::const_iterator i = engines.begin(); i != engines.end(); ++i) { + for (std::vector::const_iterator i = engines.begin(); i != engines.end(); ++i) { ASSERT(*i); const Engine& e = **i; - if(e.canHandle(uri, config_)) { + if (e.canHandle(uri, config_)) { return e.dbType(); } } @@ -277,7 +268,7 @@ std::vector Manager::visitableLocations(const metkit::mars::MarsRequ std::vector r; // union of all locations - for(std::set::const_iterator i = engines.begin(); i != engines.end(); ++i) { + for (std::set::const_iterator i = engines.begin(); i != engines.end(); ++i) { LOG_DEBUG_LIB(LibFdb5) << "Selected FDB engine " << *i << std::endl; std::vector p; if (all) { @@ -290,9 +281,8 @@ std::vector Manager::visitableLocations(const metkit::mars::MarsRequ } return r; - } //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/database/Manager.h b/src/fdb5/database/Manager.h index dc7d86c59..cedad47a5 100644 --- a/src/fdb5/database/Manager.h +++ b/src/fdb5/database/Manager.h @@ -16,13 +16,13 @@ #ifndef fdb5_Manager_H #define fdb5_Manager_H -#include #include +#include #include #include "eckit/filesystem/URI.h" -#include "metkit/mars/MarsRequest.h" #include "fdb5/config/Config.h" +#include "metkit/mars/MarsRequest.h" namespace fdb5 { @@ -30,10 +30,9 @@ class Key; //---------------------------------------------------------------------------------------------------------------------- -class Manager { +class Manager { public: // methods - Manager(const Config& config); ~Manager(); @@ -51,7 +50,6 @@ class Manager { std::vector visitableLocations(const metkit::mars::MarsRequest& request, bool all); private: // members - eckit::PathName enginesFile_; // If the engine is specified explicitly in the diff --git a/src/fdb5/database/MoveVisitor.cc b/src/fdb5/database/MoveVisitor.cc index 4d742473b..e6d80eae0 100644 --- a/src/fdb5/database/MoveVisitor.cc +++ b/src/fdb5/database/MoveVisitor.cc @@ -14,11 +14,10 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -MoveVisitor::MoveVisitor(const metkit::mars::MarsRequest& request, - const eckit::URI& dest) : - EntryVisitor(), - request_(request), - dest_(dest) {} +MoveVisitor::MoveVisitor(const metkit::mars::MarsRequest& request, const eckit::URI& dest) + : EntryVisitor() + , request_(request) + , dest_(dest) {} MoveVisitor::~MoveVisitor() {} diff --git a/src/fdb5/database/MoveVisitor.h b/src/fdb5/database/MoveVisitor.h index e6efd693d..981cf0f90 100644 --- a/src/fdb5/database/MoveVisitor.h +++ b/src/fdb5/database/MoveVisitor.h @@ -14,8 +14,8 @@ #pragma once -#include "eckit/exception/Exceptions.h" #include "eckit/distributed/Transport.h" +#include "eckit/exception/Exceptions.h" #include "metkit/mars/MarsRequest.h" @@ -28,9 +28,7 @@ namespace fdb5 { class MoveVisitor : public EntryVisitor { public: // methods - - MoveVisitor(const metkit::mars::MarsRequest& request, - const eckit::URI& dest); + MoveVisitor(const metkit::mars::MarsRequest& request, const eckit::URI& dest); ~MoveVisitor() override; @@ -42,7 +40,6 @@ class MoveVisitor : public EntryVisitor { void visitDatum(const Field& /*field*/, const std::string& /*keyFingerprint*/) override { NOTIMP; } protected: // members - const metkit::mars::MarsRequest& request_; const eckit::URI& dest_; }; @@ -50,4 +47,3 @@ class MoveVisitor : public EntryVisitor { //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 - diff --git a/src/fdb5/database/MultiRetrieveVisitor.cc b/src/fdb5/database/MultiRetrieveVisitor.cc index c5de99e40..f51233d8d 100644 --- a/src/fdb5/database/MultiRetrieveVisitor.cc +++ b/src/fdb5/database/MultiRetrieveVisitor.cc @@ -21,35 +21,29 @@ #include "fdb5/types/Type.h" #include "fdb5/types/TypesRegistry.h" - - namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -MultiRetrieveVisitor::MultiRetrieveVisitor(const Notifier& wind, - InspectIterator& iterator, - eckit::CacheLRU& databases, - const Config& config) : - wind_(wind), - databases_(databases), - iterator_(iterator), - config_(config) { -} +MultiRetrieveVisitor::MultiRetrieveVisitor(const Notifier& wind, InspectIterator& iterator, + eckit::CacheLRU& databases, const Config& config) + : wind_(wind) + , databases_(databases) + , iterator_(iterator) + , config_(config) {} -MultiRetrieveVisitor::~MultiRetrieveVisitor() { -} +MultiRetrieveVisitor::~MultiRetrieveVisitor() {} // From Visitor bool MultiRetrieveVisitor::selectDatabase(const Key& dbKey, const TypedKey& fullComputedKey) { - LOG_DEBUG_LIB(LibFdb5) << "FDB5 selectDatabase " << dbKey << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "FDB5 selectDatabase " << dbKey << std::endl; /* is it the current DB ? */ - if(catalogue_) { - if(dbKey == catalogue_->key()) { + if (catalogue_) { + if (dbKey == catalogue_->key()) { eckit::Log::info() << "This is the current db" << std::endl; return true; } @@ -57,7 +51,7 @@ bool MultiRetrieveVisitor::selectDatabase(const Key& dbKey, const TypedKey& full /* is the DB already open ? */ - if(databases_.exists(dbKey)) { + if (databases_.exists(dbKey)) { LOG_DEBUG_LIB(LibFdb5) << "FDB5 Reusing database " << dbKey << std::endl; catalogue_ = databases_.access(dbKey); return true; @@ -75,7 +69,8 @@ bool MultiRetrieveVisitor::selectDatabase(const Key& dbKey, const TypedKey& full return false; } - LOG_DEBUG_LIB(LibFdb5) << "MultiRetrieveVisitor::selectDatabase opening database " << dbKey << " (type=" << newCatalogue->type() << ")" << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "MultiRetrieveVisitor::selectDatabase opening database " << dbKey + << " (type=" << newCatalogue->type() << ")" << std::endl; if (!newCatalogue->open()) { LOG_DEBUG_LIB(LibFdb5) << "Database does not exist " << dbKey << std::endl; @@ -106,17 +101,16 @@ bool MultiRetrieveVisitor::selectDatum(const TypedKey& datumKey, const TypedKey& simplifiedKey.set(k->first, k->second); } - iterator_.emplace(ListElement({catalogue_->key(), catalogue_->indexKey(), simplifiedKey}, field.stableLocation(), field.timestamp())); + iterator_.emplace(ListElement({catalogue_->key(), catalogue_->indexKey(), simplifiedKey}, + field.stableLocation(), field.timestamp())); return true; } return false; } -void MultiRetrieveVisitor::values(const metkit::mars::MarsRequest &request, - const std::string &keyword, - const TypesRegistry ®istry, - eckit::StringList &values) { +void MultiRetrieveVisitor::values(const metkit::mars::MarsRequest& request, const std::string& keyword, + const TypesRegistry& registry, eckit::StringList& values) { eckit::StringList list; registry.lookupType(keyword).getValues(request, keyword, list, wind_, catalogue_); @@ -126,7 +120,7 @@ void MultiRetrieveVisitor::values(const metkit::mars::MarsRequest &request, toFilter = catalogue_->axis(keyword, filter); } - for(const auto& l: list) { + for (const auto& l : list) { std::string v = registry.lookupType(keyword).toKey(l); if (!toFilter || filter.find(v) != filter.end()) { values.push_back(l); @@ -134,7 +128,7 @@ void MultiRetrieveVisitor::values(const metkit::mars::MarsRequest &request, } } -void MultiRetrieveVisitor::print( std::ostream &out ) const { +void MultiRetrieveVisitor::print(std::ostream& out) const { out << "MultiRetrieveVisitor[]"; } diff --git a/src/fdb5/database/MultiRetrieveVisitor.h b/src/fdb5/database/MultiRetrieveVisitor.h index 98748f2ca..96241caa0 100644 --- a/src/fdb5/database/MultiRetrieveVisitor.h +++ b/src/fdb5/database/MultiRetrieveVisitor.h @@ -36,16 +36,12 @@ class Notifier; class MultiRetrieveVisitor : public ReadVisitor { public: // methods - - MultiRetrieveVisitor(const Notifier& wind, - InspectIterator& queue, - eckit::CacheLRU& databases, - const Config& config); + MultiRetrieveVisitor(const Notifier& wind, InspectIterator& queue, + eckit::CacheLRU& databases, const Config& config); ~MultiRetrieveVisitor(); -private: // methods - +private: // methods // From Visitor bool selectDatabase(const Key& dbKey, const TypedKey& fullComputedKey) override; @@ -54,20 +50,17 @@ class MultiRetrieveVisitor : public ReadVisitor { bool selectDatum(const TypedKey& datumKey, const TypedKey& fullComputedKey) override; - virtual void values(const metkit::mars::MarsRequest& request, - const std::string& keyword, - const TypesRegistry& registry, - eckit::StringList& values) override; + virtual void values(const metkit::mars::MarsRequest& request, const std::string& keyword, + const TypesRegistry& registry, eckit::StringList& values) override; - void print( std::ostream &out ) const override; + void print(std::ostream& out) const override; const Schema& databaseSchema() const override; private: - const Notifier& wind_; - eckit::CacheLRU& databases_; + eckit::CacheLRU& databases_; InspectIterator& iterator_; diff --git a/src/fdb5/database/Notifier.cc b/src/fdb5/database/Notifier.cc index 7e17a7c78..c6a56ce4d 100644 --- a/src/fdb5/database/Notifier.cc +++ b/src/fdb5/database/Notifier.cc @@ -8,18 +8,15 @@ * does it submit to any jurisdiction. */ - #include "fdb5/database/Notifier.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -Notifier::Notifier() { -} +Notifier::Notifier() {} -Notifier::~Notifier() { -} +Notifier::~Notifier() {} //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/Notifier.h b/src/fdb5/database/Notifier.h index de1c73719..cb84c6cb6 100644 --- a/src/fdb5/database/Notifier.h +++ b/src/fdb5/database/Notifier.h @@ -16,17 +16,13 @@ #ifndef fdb5_Notifier_H #define fdb5_Notifier_H - - namespace fdb5 { - //---------------------------------------------------------------------------------------------------------------------- class Notifier { public: // methods - Notifier(); virtual ~Notifier(); diff --git a/src/fdb5/database/ReadVisitor.h b/src/fdb5/database/ReadVisitor.h index e7dbb6612..74139ffef 100644 --- a/src/fdb5/database/ReadVisitor.h +++ b/src/fdb5/database/ReadVisitor.h @@ -19,14 +19,14 @@ #include #include "eckit/memory/NonCopyable.h" -#include "fdb5/database/Catalogue.h" #include "eckit/types/Types.h" +#include "fdb5/database/Catalogue.h" namespace metkit { namespace mars { - class MarsRequest; -} +class MarsRequest; } +} // namespace metkit namespace fdb5 { @@ -41,7 +41,6 @@ class Schema; class ReadVisitor : public eckit::NonCopyable { public: // methods - ReadVisitor() : catalogue_(nullptr) {} virtual ~ReadVisitor() {} @@ -53,26 +52,20 @@ class ReadVisitor : public eckit::NonCopyable { // Once we have selected a database, return its schema. Used for further iteration. virtual const Schema& databaseSchema() const = 0; - virtual void values(const metkit::mars::MarsRequest &request, - const std::string &keyword, - const TypesRegistry ®istry, - eckit::StringList &values) = 0; + virtual void values(const metkit::mars::MarsRequest& request, const std::string& keyword, + const TypesRegistry& registry, eckit::StringList& values) = 0; protected: // methods - - virtual void print( std::ostream &out ) const = 0; + virtual void print(std::ostream& out) const = 0; protected: // members - CatalogueReader* catalogue_; private: // members - - friend std::ostream &operator<<(std::ostream &s, const ReadVisitor &x) { + friend std::ostream& operator<<(std::ostream& s, const ReadVisitor& x) { x.print(s); return s; } - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/Report.cc b/src/fdb5/database/Report.cc index 008024944..a54d11ca7 100644 --- a/src/fdb5/database/Report.cc +++ b/src/fdb5/database/Report.cc @@ -11,31 +11,28 @@ #include #include "eckit/exception/Exceptions.h" -#include "eckit/log/Statistics.h" #include "eckit/log/Log.h" +#include "eckit/log/Statistics.h" #include "fdb5/LibFdb5.h" #include "fdb5/database/Report.h" -using eckit::Statistics; using eckit::Log; +using eckit::Statistics; namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -Report::~Report() { -} +Report::~Report() {} -void Report::append(const dbtype_t& dbType, DbStats stats) -{ +void Report::append(const dbtype_t& dbType, DbStats stats) { dbtypes_.insert(dbType); std::map::iterator itr = dbStats_.find(dbType); - if(itr != dbStats_.end()) { + if (itr != dbStats_.end()) { itr->second.add(stats); - } - else { + } else { dbStats_[dbType] = stats; } } @@ -47,49 +44,43 @@ Report& Report::operator+=(const Report& rhs) { // union of dbtypes std::set join; - std::set_union(dbtypes_.begin(), - dbtypes_.end(), - rhs.dbtypes_.begin(), - rhs.dbtypes_.end(), - std::insert_iterator< std::set>(join, join.begin())); + std::set_union(dbtypes_.begin(), dbtypes_.end(), rhs.dbtypes_.begin(), rhs.dbtypes_.end(), + std::insert_iterator>(join, join.begin())); std::swap(dbtypes_, join); // collate DB stats - for(const auto& [dbType, dbStat] : rhs.dbStats_) { + for (const auto& [dbType, dbStat] : rhs.dbStats_) { LOG_DEBUG_LIB(LibFdb5) << "dbtype " << dbType << std::endl; std::map::iterator j = dbStats_.find(dbType); - if(j != dbStats_.end()) { + if (j != dbStats_.end()) { j->second.add(dbStat); - } - else{ + } else { dbStats_[dbType] = dbStat; } } // collate Index stats - for(const auto& [dbType, indexStat] : rhs.indexStats_) { + for (const auto& [dbType, indexStat] : rhs.indexStats_) { LOG_DEBUG_LIB(LibFdb5) << "dbtype " << dbType << std::endl; std::map::iterator j = indexStats_.find(dbType); - if(j != indexStats_.end()) { + if (j != indexStats_.end()) { j->second.add(indexStat); - } - else{ + } else { indexStats_[dbType] = indexStat; } } // collate Data stats - for(const auto& [dbType, dataStat] : rhs.dataStats_) { + for (const auto& [dbType, dataStat] : rhs.dataStats_) { LOG_DEBUG_LIB(LibFdb5) << "dbtype " << dbType << std::endl; std::map::iterator j = dataStats_.find(dbType); - if(j != dataStats_.end()) { + if (j != dataStats_.end()) { j->second.add(dataStat); - } - else{ + } else { dataStats_[dbType] = dataStat; } } @@ -97,25 +88,24 @@ Report& Report::operator+=(const Report& rhs) { return *this; } -void Report::print(std::ostream& out) const -{ +void Report::print(std::ostream& out) const { const char* sep = ""; - for(std::set::const_iterator i = dbtypes_.begin(); i != dbtypes_.end(); ++i) { + for (std::set::const_iterator i = dbtypes_.begin(); i != dbtypes_.end(); ++i) { dbtype_t dbType = *i; out << sep << "Database Type \'" << dbType << "\'" << std::endl; std::map::const_iterator db = dbStats_.find(dbType); - if(db != dbStats_.end()) + if (db != dbStats_.end()) db->second.report(out); std::map::const_iterator idx = indexStats_.find(dbType); - if(idx != indexStats_.end()) + if (idx != indexStats_.end()) idx->second.report(out); std::map::const_iterator data = dataStats_.find(dbType); - if(data != dataStats_.end()) + if (data != dataStats_.end()) data->second.report(out); sep = "\n"; diff --git a/src/fdb5/database/Report.h b/src/fdb5/database/Report.h index 268f945fe..dacb85509 100644 --- a/src/fdb5/database/Report.h +++ b/src/fdb5/database/Report.h @@ -21,18 +21,16 @@ #include "eckit/memory/NonCopyable.h" -#include "fdb5/database/IndexStats.h" -#include "fdb5/database/DbStats.h" #include "fdb5/database/DataStats.h" +#include "fdb5/database/DbStats.h" +#include "fdb5/database/IndexStats.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- - class Report : private eckit::NonCopyable { public: - typedef std::string dbtype_t; ~Report(); @@ -41,27 +39,24 @@ class Report : private eckit::NonCopyable { void append(const dbtype_t& dbtype, fdb5::IndexStats stats); void append(const dbtype_t& dbtype, fdb5::DataStats stats); - Report& operator+= (const Report& rhs); + Report& operator+=(const Report& rhs); private: // methods - void print(std::ostream&) const; friend std::ostream& operator<<(std::ostream& s, const Report& o) { - o.print(s); return s; + o.print(s); + return s; } private: // members - std::set dbtypes_; - std::map dbStats_; + std::map dbStats_; std::map indexStats_; - std::map dataStats_; - + std::map dataStats_; }; - //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 diff --git a/src/fdb5/database/RetrieveVisitor.cc b/src/fdb5/database/RetrieveVisitor.cc index dfaea4244..fb2983084 100644 --- a/src/fdb5/database/RetrieveVisitor.cc +++ b/src/fdb5/database/RetrieveVisitor.cc @@ -19,25 +19,23 @@ #include "fdb5/types/Type.h" #include "fdb5/types/TypesRegistry.h" - - namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -RetrieveVisitor::RetrieveVisitor(const Notifier &wind, HandleGatherer &gatherer) : - store_(nullptr), wind_(wind), gatherer_(gatherer) { -} +RetrieveVisitor::RetrieveVisitor(const Notifier& wind, HandleGatherer& gatherer) + : store_(nullptr) + , wind_(wind) + , gatherer_(gatherer) {} -RetrieveVisitor::~RetrieveVisitor() { -} +RetrieveVisitor::~RetrieveVisitor() {} // From Visitor bool RetrieveVisitor::selectDatabase(const Key& dbKey, const TypedKey&) { - if(catalogue_) { - if(dbKey == catalogue_->key()) { + if (catalogue_) { + if (dbKey == catalogue_->key()) { return true; } } @@ -71,7 +69,7 @@ bool RetrieveVisitor::selectDatum(const TypedKey& datumKey, const TypedKey&) { ASSERT(catalogue_); Field field; - eckit::DataHandle *dh = nullptr; + eckit::DataHandle* dh = nullptr; if (catalogue_->retrieve(datumKey.canonical(), field)) { dh = store().retrieve(field); } @@ -83,10 +81,8 @@ bool RetrieveVisitor::selectDatum(const TypedKey& datumKey, const TypedKey&) { return (dh != 0); } -void RetrieveVisitor::values(const metkit::mars::MarsRequest &request, - const std::string &keyword, - const TypesRegistry ®istry, - eckit::StringList &values) { +void RetrieveVisitor::values(const metkit::mars::MarsRequest& request, const std::string& keyword, + const TypesRegistry& registry, eckit::StringList& values) { eckit::StringList list; registry.lookupType(keyword).getValues(request, keyword, list, wind_, catalogue_); @@ -96,7 +92,7 @@ void RetrieveVisitor::values(const metkit::mars::MarsRequest &request, toFilter = catalogue_->axis(keyword, filter); } - for(const auto& value: list) { + for (const auto& value : list) { std::string v = registry.lookupType(keyword).toKey(value); if (!toFilter || filter.find(v) != filter.end()) { values.push_back(value); @@ -113,7 +109,7 @@ Store& RetrieveVisitor::store() { return *store_; } -void RetrieveVisitor::print( std::ostream &out ) const { +void RetrieveVisitor::print(std::ostream& out) const { out << "RetrieveVisitor[]"; } diff --git a/src/fdb5/database/RetrieveVisitor.h b/src/fdb5/database/RetrieveVisitor.h index 80a469fbc..200388fdc 100644 --- a/src/fdb5/database/RetrieveVisitor.h +++ b/src/fdb5/database/RetrieveVisitor.h @@ -30,14 +30,11 @@ class Notifier; class RetrieveVisitor : public ReadVisitor { public: // methods - - RetrieveVisitor(const Notifier &wind, HandleGatherer &gatherer); + RetrieveVisitor(const Notifier& wind, HandleGatherer& gatherer); ~RetrieveVisitor(); - -private: // methods - +private: // methods // From Visitor bool selectDatabase(const Key& dbKey, const TypedKey& fullComputedKey) override; @@ -46,23 +43,20 @@ class RetrieveVisitor : public ReadVisitor { bool selectDatum(const TypedKey& datumKey, const TypedKey& fullComputedKey) override; - void values(const metkit::mars::MarsRequest& request, - const std::string& keyword, - const TypesRegistry& registry, - eckit::StringList& values) override; + void values(const metkit::mars::MarsRequest& request, const std::string& keyword, const TypesRegistry& registry, + eckit::StringList& values) override; - void print( std::ostream &out ) const override; + void print(std::ostream& out) const override; Store& store(); const Schema& databaseSchema() const override; private: - std::unique_ptr store_; - const Notifier &wind_; + const Notifier& wind_; - HandleGatherer &gatherer_; + HandleGatherer& gatherer_; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/StatsReportVisitor.h b/src/fdb5/database/StatsReportVisitor.h index 679543504..712a9763a 100644 --- a/src/fdb5/database/StatsReportVisitor.h +++ b/src/fdb5/database/StatsReportVisitor.h @@ -15,8 +15,8 @@ #define fdb5_StatsReportVisitor_H #include "fdb5/database/DbStats.h" -#include "fdb5/database/IndexStats.h" #include "fdb5/database/EntryVisitMechanism.h" +#include "fdb5/database/IndexStats.h" namespace fdb5 { @@ -25,11 +25,10 @@ namespace fdb5 { class StatsReportVisitor : public EntryVisitor { public: // methods - using EntryVisitor::EntryVisitor; virtual IndexStats indexStatistics() const = 0; - virtual DbStats dbStatistics() const = 0; + virtual DbStats dbStatistics() const = 0; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/Store.cc b/src/fdb5/database/Store.cc index 2c18203ef..92c6c487d 100644 --- a/src/fdb5/database/Store.cc +++ b/src/fdb5/database/Store.cc @@ -24,11 +24,12 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -void Store::archive(const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive) { +void Store::archive(const Key& key, const void* data, eckit::Length length, + std::function fieldLocation)> catalogue_archive) { catalogue_archive(archive(key, data, length)); } -std::unique_ptr Store::archive(const Key& key, const void *data, eckit::Length length) { +std::unique_ptr Store::archive(const Key& key, const void* data, eckit::Length length) { NOTIMP; } @@ -107,10 +108,11 @@ StoreBuilderBase::StoreBuilderBase(const std::string& name) : name_(name) { } StoreBuilderBase::~StoreBuilderBase() { - if(LibFdb5::instance().dontDeregisterFactories()) return; + if (LibFdb5::instance().dontDeregisterFactories()) + return; StoreFactory::instance().remove(name_); } //---------------------------------------------------------------------------------------------------------------------- -} // namespace eckit +} // namespace fdb5 diff --git a/src/fdb5/database/Store.h b/src/fdb5/database/Store.h index 2f7e7576f..e1af44d79 100644 --- a/src/fdb5/database/Store.h +++ b/src/fdb5/database/Store.h @@ -31,31 +31,36 @@ namespace fdb5 { class Store { public: - Store() {} virtual ~Store() = default; virtual eckit::DataHandle* retrieve(Field& field) const = 0; - virtual void archive(const Key& idxKey, const void* data, eckit::Length length, std::function fieldLocation)> catalogue_archive); + virtual void + archive(const Key& idxKey, const void* data, eckit::Length length, + std::function fieldLocation)> catalogue_archive); virtual std::unique_ptr archive(const Key& idxKey, const void* data, eckit::Length length); - virtual void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit = true) const = 0; + virtual void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, + bool doit = true) const = 0; - friend std::ostream &operator<<(std::ostream &s, const Store &x); - virtual void print( std::ostream &out ) const = 0; + friend std::ostream& operator<<(std::ostream& s, const Store& x); + virtual void print(std::ostream& out) const = 0; virtual std::string type() const = 0; virtual bool open() = 0; virtual size_t flush() = 0; virtual void close() = 0; -// virtual std::string owner() const = 0; + // virtual std::string owner() const = 0; virtual bool exists() const = 0; virtual void checkUID() const = 0; virtual bool canMoveTo(const Key& key, const Config& config, const eckit::URI& dest) const; - virtual void moveTo(const Key& key, const Config& config, const eckit::URI& dest, eckit::Queue& queue) const { NOTIMP; } + virtual void moveTo(const Key& key, const Config& config, const eckit::URI& dest, + eckit::Queue& queue) const { + NOTIMP; + } virtual void remove(const Key& key) const { NOTIMP; } virtual eckit::URI uri() const = 0; @@ -68,7 +73,6 @@ class Store { virtual bool auxiliaryURIExists(const eckit::URI&) const { NOTIMP; } }; - //---------------------------------------------------------------------------------------------------------------------- class StoreBuilderBase { @@ -82,7 +86,9 @@ class StoreBuilderBase { template class StoreBuilder : public StoreBuilderBase { - std::unique_ptr make(const Key& key, const Config& config) override { return std::unique_ptr(new T(key, config)); } + std::unique_ptr make(const Key& key, const Config& config) override { + return std::unique_ptr(new T(key, config)); + } public: StoreBuilder(const std::string& name) : StoreBuilderBase(name) {} @@ -111,4 +117,4 @@ class StoreFactory { eckit::Mutex mutex_; }; -} +} // namespace fdb5 diff --git a/src/fdb5/database/UriStore.cc b/src/fdb5/database/UriStore.cc index 7a50be58e..f1421319b 100644 --- a/src/fdb5/database/UriStore.cc +++ b/src/fdb5/database/UriStore.cc @@ -17,28 +17,20 @@ namespace fdb5 { - //---------------------------------------------------------------------------------------------------------------------- +UriStore::UriStore(const eckit::PathName& directory) : next_(0), readOnly_(false), directory_(directory) {} -UriStore::UriStore(const eckit::PathName &directory) : - next_(0), - readOnly_(false), - directory_(directory) { -} - -UriStore::~UriStore() { -} - +UriStore::~UriStore() {} -UriStore::UriStore(const eckit::PathName &directory, eckit::Stream &s): - next_(0), - readOnly_(true), - directory_(directory) { +UriStore::UriStore(const eckit::PathName& directory, eckit::Stream& s) + : next_(0) + , readOnly_(true) + , directory_(directory) { size_t n; s >> n; - for (size_t i = 0; i < n ; i++) { + for (size_t i = 0; i < n; i++) { UriStore::UriID id; std::string p; @@ -60,31 +52,27 @@ UriStore::UriStore(const eckit::PathName &directory, eckit::Stream &s): next_ = std::max(id + 1, next_); } - } - -void UriStore::encode(eckit::Stream &s) const { +void UriStore::encode(eckit::Stream& s) const { s << paths_.size(); - for ( URIStore::const_iterator i = paths_.begin(); i != paths_.end(); ++i ) { + for (URIStore::const_iterator i = paths_.begin(); i != paths_.end(); ++i) { s << i->first; - const eckit::URI &uri = i->second; + const eckit::URI& uri = i->second; if (uri.scheme() == "file") { - const eckit::PathName &path = uri.path(); - s << ( (path.dirName() == directory_) ? path.baseName() : path ); + const eckit::PathName& path = uri.path(); + s << ((path.dirName() == directory_) ? path.baseName() : path); } else { s << uri.asRawString(); } } } - -UriStore::UriID UriStore::insert( const eckit::URI &path ) { +UriStore::UriID UriStore::insert(const eckit::URI& path) { ASSERT(!readOnly_); - IdStore::iterator itr = ids_.find(path); - if ( itr != ids_.end() ) + if (itr != ids_.end()) return itr->second; UriStore::UriID current = next_; @@ -97,7 +85,7 @@ UriStore::UriID UriStore::insert( const eckit::URI &path ) { eckit::URI UriStore::get(const UriStore::UriID id) const { URIStore::const_iterator itr = paths_.find(id); - ASSERT( itr != paths_.end() ); + ASSERT(itr != paths_.end()); return itr->second; } @@ -113,20 +101,19 @@ std::vector UriStore::paths() const { return p; } -void UriStore::print( std::ostream &out ) const { - for ( URIStore::const_iterator itr = paths_.begin(); itr != paths_.end(); ++itr ) { +void UriStore::print(std::ostream& out) const { + for (URIStore::const_iterator itr = paths_.begin(); itr != paths_.end(); ++itr) { out << itr->first << " " << itr->second << std::endl; } } -void UriStore::dump(std::ostream &out, const char* indent) const { +void UriStore::dump(std::ostream& out, const char* indent) const { out << indent << "Files:" << std::endl; - for ( URIStore::const_iterator itr = paths_.begin(); itr != paths_.end(); ++itr ) { + for (URIStore::const_iterator itr = paths_.begin(); itr != paths_.end(); ++itr) { out << indent << indent << std::setw(3) << itr->first << " => " << itr->second << std::endl; } } //---------------------------------------------------------------------------------------------------------------------- - } // namespace fdb5 diff --git a/src/fdb5/database/UriStore.h b/src/fdb5/database/UriStore.h index 0a4978ab6..90761f9dd 100644 --- a/src/fdb5/database/UriStore.h +++ b/src/fdb5/database/UriStore.h @@ -15,7 +15,6 @@ #ifndef fdb5_FileStore_H #define fdb5_FileStore_H - #include "eckit/filesystem/PathName.h" #include "eckit/filesystem/URI.h" #include "eckit/io/DataHandle.h" @@ -29,66 +28,56 @@ class Stream; namespace fdb5 { - //---------------------------------------------------------------------------------------------------------------------- - class UriStore : private eckit::NonCopyable { public: // types - typedef size_t UriID; - typedef std::map< UriStore::UriID, eckit::URI > URIStore; - typedef std::map< eckit::URI, UriStore::UriID > IdStore; + typedef std::map URIStore; + typedef std::map IdStore; public: // methods - - UriStore(const eckit::PathName &directory); - UriStore(const eckit::PathName &directory, eckit::Stream & ); + UriStore(const eckit::PathName& directory); + UriStore(const eckit::PathName& directory, eckit::Stream&); ~UriStore(); /// Inserts the URI in the store /// @returns PathID associated to the provided eckit::URI - UriStore::UriID insert( const eckit::URI &path ); + UriStore::UriID insert(const eckit::URI& path); /// @returns a eckit::URI associated to the provided id /// @pre assumes that the eckit::filesystem/PathName.has already been inserted - eckit::URI get( const UriStore::UriID id ) const; + eckit::URI get(const UriStore::UriID id) const; /// Return all of the paths used std::vector paths() const; - void dump(std::ostream &out, const char* indent) const; + void dump(std::ostream& out, const char* indent) const; - void encode(eckit::Stream &s) const; + void encode(eckit::Stream& s) const; - friend std::ostream &operator<<(std::ostream &s, const UriStore &x) { + friend std::ostream& operator<<(std::ostream& s, const UriStore& x) { x.print(s); return s; } private: // members - - void print( std::ostream &out ) const; + void print(std::ostream& out) const; private: // members + UriStore::UriID next_; + bool readOnly_; - UriStore::UriID next_; - bool readOnly_; - - eckit::PathName directory_; - URIStore paths_; - IdStore ids_; - - + eckit::PathName directory_; + URIStore paths_; + IdStore ids_; }; - //---------------------------------------------------------------------------------------------------------------------- - } // namespace fdb5 #endif diff --git a/src/fdb5/database/WipeVisitor.cc b/src/fdb5/database/WipeVisitor.cc index a8eb606ad..c6c3f60f0 100644 --- a/src/fdb5/database/WipeVisitor.cc +++ b/src/fdb5/database/WipeVisitor.cc @@ -14,18 +14,14 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -WipeVisitor::WipeVisitor(const metkit::mars::MarsRequest& request, - std::ostream& out, - bool doit, - bool porcelain, - bool unsafeWipeAll) : - EntryVisitor(), - request_(request), - out_(out), - doit_(doit), - porcelain_(porcelain), - unsafeWipeAll_(unsafeWipeAll) {} - +WipeVisitor::WipeVisitor(const metkit::mars::MarsRequest& request, std::ostream& out, bool doit, bool porcelain, + bool unsafeWipeAll) + : EntryVisitor() + , request_(request) + , out_(out) + , doit_(doit) + , porcelain_(porcelain) + , unsafeWipeAll_(unsafeWipeAll) {} WipeVisitor::~WipeVisitor() {} diff --git a/src/fdb5/database/WipeVisitor.h b/src/fdb5/database/WipeVisitor.h index 272e15931..6f285c203 100644 --- a/src/fdb5/database/WipeVisitor.h +++ b/src/fdb5/database/WipeVisitor.h @@ -27,11 +27,7 @@ namespace fdb5 { class WipeVisitor : public EntryVisitor { public: // methods - - WipeVisitor(const metkit::mars::MarsRequest& request, - std::ostream& out, - bool doit, - bool porcelain, + WipeVisitor(const metkit::mars::MarsRequest& request, std::ostream& out, bool doit, bool porcelain, bool unsafeWipeAll); ~WipeVisitor() override; @@ -41,7 +37,6 @@ class WipeVisitor : public EntryVisitor { void visitDatum(const Field& /*field*/, const std::string& /*keyFingerprint*/) override { NOTIMP; } protected: // members - const metkit::mars::MarsRequest& request_; std::ostream& out_; diff --git a/src/fdb5/database/WriteVisitor.cc b/src/fdb5/database/WriteVisitor.cc index 02864b71c..648e833e1 100644 --- a/src/fdb5/database/WriteVisitor.cc +++ b/src/fdb5/database/WriteVisitor.cc @@ -14,14 +14,11 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -WriteVisitor::WriteVisitor(std::vector &prev) : - prev_(prev), - rule_(0) { +WriteVisitor::WriteVisitor(std::vector& prev) : prev_(prev), rule_(0) { prev.resize(3); } -WriteVisitor::~WriteVisitor() { -} +WriteVisitor::~WriteVisitor() {} //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/WriteVisitor.h b/src/fdb5/database/WriteVisitor.h index 6d09055b0..1ad48d334 100644 --- a/src/fdb5/database/WriteVisitor.h +++ b/src/fdb5/database/WriteVisitor.h @@ -24,7 +24,9 @@ #include "fdb5/database/Key.h" -namespace metkit { class MarsRequest; } +namespace metkit { +class MarsRequest; +} namespace fdb5 { @@ -36,8 +38,7 @@ class Schema; class WriteVisitor : public eckit::NonCopyable { public: // methods - - WriteVisitor(std::vector &); + WriteVisitor(std::vector&); virtual ~WriteVisitor(); @@ -48,30 +49,23 @@ class WriteVisitor : public eckit::NonCopyable { // Once we have selected a database, return its schema. Used for further iteration. virtual const Schema& databaseSchema() const = 0; - void rule(const Rule *r) { - rule_ = r; - } - const Rule *rule() const { - return rule_; - } + void rule(const Rule* r) { rule_ = r; } + const Rule* rule() const { return rule_; } protected: // methods - - virtual void print( std::ostream &out ) const = 0; + virtual void print(std::ostream& out) const = 0; private: // members - - friend std::ostream &operator<<(std::ostream &s, const WriteVisitor &x) { + friend std::ostream& operator<<(std::ostream& s, const WriteVisitor& x) { x.print(s); return s; } friend class Rule; - std::vector &prev_; - - const Rule *rule_; // Last rule used + std::vector& prev_; + const Rule* rule_; // Last rule used }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/io/FDBFileHandle.cc b/src/fdb5/io/FDBFileHandle.cc index 002ef89ee..9c2f9bbc5 100644 --- a/src/fdb5/io/FDBFileHandle.cc +++ b/src/fdb5/io/FDBFileHandle.cc @@ -8,8 +8,8 @@ * does it submit to any jurisdiction. */ -#include #include +#include #include "eckit/config/Resource.h" #include "eckit/eckit.h" @@ -29,11 +29,11 @@ void FDBFileHandle::print(std::ostream& s) const { s << "FDBFileHandle[file=" << path_ << ']'; } -FDBFileHandle::FDBFileHandle(const std::string& name, size_t buffer) : - path_(name), - file_(nullptr), - buffer_(buffer), - pos_(0) {} +FDBFileHandle::FDBFileHandle(const std::string& name, size_t buffer) + : path_(name) + , file_(nullptr) + , buffer_(buffer) + , pos_(0) {} FDBFileHandle::~FDBFileHandle() {} @@ -80,8 +80,7 @@ void FDBFileHandle::flush() { if (file_) { if (::fflush(file_)) - throw WriteError(std::string("FDBFileHandle::~FDBFileHandle(fflush(") + path_ + "))", - Here()); + throw WriteError(std::string("FDBFileHandle::~FDBFileHandle(fflush(") + path_ + "))", Here()); if (fdbDataSyncOnFlush) { int ret = eckit::fdatasync(::fileno(file_)); @@ -90,8 +89,7 @@ void FDBFileHandle::flush() { ret = eckit::fdatasync(::fileno(file_)); } if (ret < 0) { - Log::error() << "Cannot fdatasync(" << path_ << ") " << ::fileno(file_) - << Log::syserr << std::endl; + Log::error() << "Cannot fdatasync(" << path_ << ") " << ::fileno(file_) << Log::syserr << std::endl; throw eckit::WriteError(path_); } } @@ -122,7 +120,6 @@ std::string FDBFileHandle::title() const { return PathName::shorten(path_); } - //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/io/FDBFileHandle.h b/src/fdb5/io/FDBFileHandle.h index bd3cba62b..e5d8929e7 100644 --- a/src/fdb5/io/FDBFileHandle.h +++ b/src/fdb5/io/FDBFileHandle.h @@ -15,8 +15,8 @@ #ifndef fdb5_FDBFileHandle_h #define fdb5_FDBFileHandle_h -#include "eckit/io/DataHandle.h" #include "eckit/io/Buffer.h" +#include "eckit/io/DataHandle.h" namespace fdb5 { @@ -30,35 +30,31 @@ namespace fdb5 { /// * this is not thread-safe neither multi-process safe class FDBFileHandle : public eckit::DataHandle { -public: // methods - +public: // methods FDBFileHandle(const std::string&, size_t buffer); ~FDBFileHandle(); eckit::Length openForRead() override; - void openForWrite(const eckit::Length &) override; - void openForAppend(const eckit::Length &) override; - - long read(void *, long) override; - long write(const void *, long) override; - void close() override; - void flush() override; - void print(std::ostream &) const override; + void openForWrite(const eckit::Length&) override; + void openForAppend(const eckit::Length&) override; + + long read(void*, long) override; + long write(const void*, long) override; + void close() override; + void flush() override; + void print(std::ostream&) const override; eckit::Offset position() override; std::string title() const override; bool canSeek() const override { return false; } protected: // members - - std::string path_; + std::string path_; private: // members - - FILE *file_; - eckit::Buffer buffer_; + FILE* file_; + eckit::Buffer buffer_; off_t pos_; - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/io/HandleGatherer.cc b/src/fdb5/io/HandleGatherer.cc index 1853a216e..234f0a150 100644 --- a/src/fdb5/io/HandleGatherer.cc +++ b/src/fdb5/io/HandleGatherer.cc @@ -10,48 +10,45 @@ #include "fdb5/io/HandleGatherer.h" +#include "eckit/exception/Exceptions.h" #include "eckit/io/MultiHandle.h" #include "eckit/log/Plural.h" -#include "eckit/exception/Exceptions.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -HandleGatherer::HandleGatherer(bool sorted): - sorted_(sorted), - count_(0) { -} +HandleGatherer::HandleGatherer(bool sorted) : sorted_(sorted), count_(0) {} HandleGatherer::~HandleGatherer() { - for (std::vector::iterator j = handles_.begin(); j != handles_.end(); ++j) { + for (std::vector::iterator j = handles_.begin(); j != handles_.end(); ++j) { delete (*j); } } -eckit::DataHandle *HandleGatherer::dataHandle() { - for (std::vector::iterator j = handles_.begin(); j != handles_.end(); ++j) { +eckit::DataHandle* HandleGatherer::dataHandle() { + for (std::vector::iterator j = handles_.begin(); j != handles_.end(); ++j) { (*j)->compress(sorted_); } - eckit::DataHandle *h = new eckit::MultiHandle(handles_); + eckit::DataHandle* h = new eckit::MultiHandle(handles_); handles_.clear(); return h; } -void HandleGatherer::add(eckit::DataHandle *h) { +void HandleGatherer::add(eckit::DataHandle* h) { count_++; ASSERT(h); if (sorted_) { - for (std::vector::iterator j = handles_.begin(); j != handles_.end(); ++j) { - if ( (*j)->merge(h) ) { + for (std::vector::iterator j = handles_.begin(); j != handles_.end(); ++j) { + if ((*j)->merge(h)) { delete h; return; } } } else { if (handles_.size() > 0) { - if ( handles_.back()->merge(h) ) { + if (handles_.back()->merge(h)) { delete h; return; } @@ -64,7 +61,7 @@ size_t HandleGatherer::count() const { return count_; } -void HandleGatherer::print( std::ostream &out ) const { +void HandleGatherer::print(std::ostream& out) const { out << eckit::Plural(handles_.size(), "handle"); } diff --git a/src/fdb5/io/HandleGatherer.h b/src/fdb5/io/HandleGatherer.h index a84fa94a2..bc0d3a2dc 100644 --- a/src/fdb5/io/HandleGatherer.h +++ b/src/fdb5/io/HandleGatherer.h @@ -17,8 +17,8 @@ #define fdb5_HandleGatherer_H #include -#include #include +#include #include "eckit/memory/NonCopyable.h" @@ -26,40 +26,33 @@ namespace eckit { class DataHandle; } - namespace fdb5 { - //---------------------------------------------------------------------------------------------------------------------- class HandleGatherer : public eckit::NonCopyable { public: // methods - HandleGatherer(bool sorted); ~HandleGatherer(); - void add(eckit::DataHandle *); + void add(eckit::DataHandle*); - eckit::DataHandle *dataHandle(); + eckit::DataHandle* dataHandle(); size_t count() const; - private: // members - bool sorted_; - std::vector handles_; + std::vector handles_; size_t count_; - void print( std::ostream &out ) const; - friend std::ostream &operator<<(std::ostream &s, const HandleGatherer &x) { + void print(std::ostream& out) const; + friend std::ostream& operator<<(std::ostream& s, const HandleGatherer& x) { x.print(s); return s; } - - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/io/LustreFileHandle.h b/src/fdb5/io/LustreFileHandle.h index 7adcde756..77e854390 100644 --- a/src/fdb5/io/LustreFileHandle.h +++ b/src/fdb5/io/LustreFileHandle.h @@ -18,39 +18,30 @@ #include "fdb5/fdb5_config.h" -#include "eckit/io/Length.h" -#include "eckit/log/Bytes.h" -#include "eckit/filesystem/PathName.h" #include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/PathName.h" +#include "eckit/io/Length.h" +#include "eckit/log/Bytes.h" -#include "fdb5/io/LustreSettings.h" #include "fdb5/LibFdb5.h" +#include "fdb5/io/LustreSettings.h" namespace fdb5 { -template< class HANDLE > +template class LustreFileHandle : public HANDLE { public: // methods + LustreFileHandle(const std::string& path, LustreStripe stripe) : HANDLE(path), stripe_(stripe) {} - LustreFileHandle(const std::string& path, LustreStripe stripe) : - HANDLE(path), - stripe_(stripe) - { - } + LustreFileHandle(const std::string& path, size_t buffsize, LustreStripe stripe) + : HANDLE(path, buffsize) + , stripe_(stripe) {} - LustreFileHandle(const std::string& path, size_t buffsize, LustreStripe stripe) : - HANDLE(path, buffsize), - stripe_(stripe) - { - } - - LustreFileHandle(const std::string& path, size_t buffcount, size_t buffsize, LustreStripe stripe) : - HANDLE(path, buffcount, buffsize), - stripe_(stripe) - { - } + LustreFileHandle(const std::string& path, size_t buffcount, size_t buffsize, LustreStripe stripe) + : HANDLE(path, buffcount, buffsize) + , stripe_(stripe) {} ~LustreFileHandle() override {} @@ -59,28 +50,26 @@ class LustreFileHandle : public HANDLE { std::string pathStr = HANDLE::path_; eckit::PathName path{pathStr}; - if(path.exists()) return; //< Lustre API outputs ioctl error messages when called on files exist + if (path.exists()) + return; //< Lustre API outputs ioctl error messages when called on files exist /* From the docs: llapi_file_create closes the file descriptor. You must re-open the file afterwards */ - LOG_DEBUG_LIB(LibFdb5) << "Creating Lustre file " << pathStr - << " with " << stripe_.count_ << " stripes " - << "of " << eckit::Bytes(stripe_.size_) - << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "Creating Lustre file " << pathStr << " with " << stripe_.count_ << " stripes " + << "of " << eckit::Bytes(stripe_.size_) << std::endl; int err = fdb5LustreapiFileCreate(path, stripe_); - if(err == EINVAL) { + if (err == EINVAL) { std::ostringstream oss; oss << "Invalid stripe parameters for Lustre file system" - << " - stripe count " << stripe_.count_ - << " - stripe size " << stripe_.size_; + << " - stripe count " << stripe_.count_ << " - stripe size " << stripe_.size_; throw eckit::BadParameter(oss.str(), Here()); } - if(err && err != EEXIST && err != EALREADY) { + if (err && err != EEXIST && err != EALREADY) { throw eckit::FailedSystemCall("llapi_file_create", Here()); } @@ -88,9 +77,7 @@ class LustreFileHandle : public HANDLE { } private: // members - LustreStripe stripe_; - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/io/LustreSettings.cc b/src/fdb5/io/LustreSettings.cc index 5af6ced71..2ea8f0778 100644 --- a/src/fdb5/io/LustreSettings.cc +++ b/src/fdb5/io/LustreSettings.cc @@ -12,17 +12,17 @@ #include "eckit/config/Resource.h" -#include "fdb5/fdb5_config.h" #include "fdb5/LibFdb5.h" +#include "fdb5/fdb5_config.h" -#define LL_SUPER_MAGIC 0x0BD00BD0 +#define LL_SUPER_MAGIC 0x0BD00BD0 #if defined(fdb5_HAVE_LUSTRE) #include extern "C" { void fdb5_lustreapi_silence_msg(); -int fdb5_lustreapi_file_create(const char* path, size_t stripesize, size_t stripecount); +int fdb5_lustreapi_file_create(const char* path, size_t stripesize, size_t stripecount); } #endif @@ -49,7 +49,7 @@ int fdb5LustreapiFileCreate(const eckit::PathName& path, LustreStripe stripe) { static bool lustreapi_silence = false; - if(not lustreapi_silence) { + if (not lustreapi_silence) { fdb5_lustreapi_silence_msg(); lustreapi_silence = true; } @@ -73,17 +73,20 @@ bool stripeLustre() { LustreStripe stripeIndexLustreSettings() { - static unsigned int fdbIndexLustreStripeCount = eckit::Resource("fdbIndexLustreStripeCount;$FDB_INDEX_LUSTRE_STRIPE_COUNT", 1); - static size_t fdbIndexLustreStripeSize = eckit::Resource("fdbIndexLustreStripeSize;$FDB_INDEX_LUSTRE_STRIPE_SIZE", 8*1024*1024); + static unsigned int fdbIndexLustreStripeCount = + eckit::Resource("fdbIndexLustreStripeCount;$FDB_INDEX_LUSTRE_STRIPE_COUNT", 1); + static size_t fdbIndexLustreStripeSize = + eckit::Resource("fdbIndexLustreStripeSize;$FDB_INDEX_LUSTRE_STRIPE_SIZE", 8 * 1024 * 1024); return LustreStripe(fdbIndexLustreStripeCount, fdbIndexLustreStripeSize); } - LustreStripe stripeDataLustreSettings() { - static unsigned int fdbDataLustreStripeCount = eckit::Resource("fdbDataLustreStripeCount;$FDB_DATA_LUSTRE_STRIPE_COUNT", 8); - static size_t fdbDataLustreStripeSize = eckit::Resource("fdbDataLustreStripeSize;$FDB_DATA_LUSTRE_STRIPE_SIZE", 8*1024*1024); + static unsigned int fdbDataLustreStripeCount = + eckit::Resource("fdbDataLustreStripeCount;$FDB_DATA_LUSTRE_STRIPE_COUNT", 8); + static size_t fdbDataLustreStripeSize = + eckit::Resource("fdbDataLustreStripeSize;$FDB_DATA_LUSTRE_STRIPE_SIZE", 8 * 1024 * 1024); return LustreStripe(fdbDataLustreStripeCount, fdbDataLustreStripeSize); } diff --git a/src/fdb5/io/LustreSettings.h b/src/fdb5/io/LustreSettings.h index c3c12d47e..92dd48e66 100644 --- a/src/fdb5/io/LustreSettings.h +++ b/src/fdb5/io/LustreSettings.h @@ -23,11 +23,7 @@ namespace fdb5 { struct LustreStripe { - LustreStripe(unsigned int count, size_t size) : - count_(count), - size_(size) - { - } + LustreStripe(unsigned int count, size_t size) : count_(count), size_(size) {} unsigned int count_; size_t size_; diff --git a/src/fdb5/io/SingleGribMungePartFileHandle.cc b/src/fdb5/io/SingleGribMungePartFileHandle.cc index 591c4f7d2..63d453343 100644 --- a/src/fdb5/io/SingleGribMungePartFileHandle.cc +++ b/src/fdb5/io/SingleGribMungePartFileHandle.cc @@ -8,15 +8,14 @@ * does it submit to any jurisdiction. */ - #include "fdb5/io/SingleGribMungePartFileHandle.h" #include #include "eckit/log/Log.h" -#include #include "eccodes.h" +#include using namespace eckit; @@ -30,39 +29,31 @@ ::eckit::ClassSpec SingleGribMungePartFileHandle::classSpec_ = { }; ::eckit::Reanimator SingleGribMungePartFileHandle::reanimator_; -void SingleGribMungePartFileHandle::print(std::ostream& s) const -{ +void SingleGribMungePartFileHandle::print(std::ostream& s) const { if (format(s) == Log::compactFormat) s << "SingleGribMungePartFileHandle"; else - s << "SingleGribMungePartFileHandle[path=" << name_ - << ",offset=" << offset_ - << ",length=" << length_ << ']'; + s << "SingleGribMungePartFileHandle[path=" << name_ << ",offset=" << offset_ << ",length=" << length_ << ']'; } -SingleGribMungePartFileHandle::SingleGribMungePartFileHandle(const PathName& name, - const Offset& offset, - const Length& length, - const Key& substitute): - name_(name), - file_(nullptr), - pos_(0), - offset_(offset), - length_(length), - substitute_(substitute) {} - +SingleGribMungePartFileHandle::SingleGribMungePartFileHandle(const PathName& name, const Offset& offset, + const Length& length, const Key& substitute) + : name_(name) + , file_(nullptr) + , pos_(0) + , offset_(offset) + , length_(length) + , substitute_(substitute) {} DataHandle* SingleGribMungePartFileHandle::clone() const { return new SingleGribMungePartFileHandle(name_, offset_, length_, substitute_); } - bool SingleGribMungePartFileHandle::compress(bool) { return false; } -SingleGribMungePartFileHandle::~SingleGribMungePartFileHandle() -{ +SingleGribMungePartFileHandle::~SingleGribMungePartFileHandle() { if (file_) { Log::warning() << "Closing SingleGribMungePartFileHandle " << name_ << std::endl; ::fclose(file_); @@ -142,8 +133,7 @@ long SingleGribMungePartFileHandle::read(void* buffer, long length) { return readLength; } -void SingleGribMungePartFileHandle::close() -{ +void SingleGribMungePartFileHandle::close() { if (file_) { ::fclose(file_); file_ = 0; @@ -171,5 +161,4 @@ std::string SingleGribMungePartFileHandle::title() const { //-------------------------------------------------------------------------------------------------- -} // namespace eckit - +} // namespace fdb5 diff --git a/src/fdb5/io/SingleGribMungePartFileHandle.h b/src/fdb5/io/SingleGribMungePartFileHandle.h index dc760f3d6..eafa9abd5 100644 --- a/src/fdb5/io/SingleGribMungePartFileHandle.h +++ b/src/fdb5/io/SingleGribMungePartFileHandle.h @@ -24,7 +24,6 @@ #include "fdb5/database/Key.h" - namespace fdb5 { //----------------------------------------------------------------------------- @@ -34,24 +33,21 @@ namespace fdb5 { class SingleGribMungePartFileHandle : public eckit::DataHandle { public: + // -- Contructors -// -- Contructors - - SingleGribMungePartFileHandle(const eckit::PathName&, - const eckit::Offset&, - const eckit::Length&, + SingleGribMungePartFileHandle(const eckit::PathName&, const eckit::Offset&, const eckit::Length&, const Key& substitute); SingleGribMungePartFileHandle(eckit::Stream&) { NOTIMP; } ~SingleGribMungePartFileHandle() override; - // From DataHandle + // From DataHandle eckit::Length openForRead() override; void openForWrite(const eckit::Length&) override { NOTIMP; } void openForAppend(const eckit::Length&) override { NOTIMP; } - long read(void*,long) override; - long write(const void*,long) override { NOTIMP; } + long read(void*, long) override; + long write(const void*, long) override { NOTIMP; } void close() override; void rewind() override { NOTIMP; } @@ -71,30 +67,28 @@ class SingleGribMungePartFileHandle : public eckit::DataHandle { bool moveable() const override { return true; } eckit::DataHandle* clone() const override; - // From Streamable + // From Streamable void encode(eckit::Stream&) const override { NOTIMP; } const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } private: // members - eckit::PathName name_; - FILE* file_; - eckit::Offset pos_; - eckit::Offset offset_; - eckit::Length length_; - Key substitute_; - std::unique_ptr buffer_; + FILE* file_; + eckit::Offset pos_; + eckit::Offset offset_; + eckit::Length length_; + Key substitute_; + std::unique_ptr buffer_; // For Streamable static eckit::ClassSpec classSpec_; - static eckit:: Reanimator reanimator_; + static eckit::Reanimator reanimator_; }; - //----------------------------------------------------------------------------- -} // namespace eckit +} // namespace fdb5 #endif diff --git a/src/fdb5/message/MessageArchiver.cc b/src/fdb5/message/MessageArchiver.cc index 0cc059cbf..e055c0b9f 100644 --- a/src/fdb5/message/MessageArchiver.cc +++ b/src/fdb5/message/MessageArchiver.cc @@ -10,24 +10,24 @@ #include -#include "eckit/log/Timer.h" -#include "eckit/log/Plural.h" #include "eckit/log/Bytes.h" -#include "eckit/log/Seconds.h" +#include "eckit/log/Plural.h" #include "eckit/log/Progress.h" +#include "eckit/log/Seconds.h" +#include "eckit/log/Timer.h" #include "eckit/utils/Tokenizer.h" -#include "eckit/message/Reader.h" #include "eckit/message/Message.h" +#include "eckit/message/Reader.h" -#include "metkit/mars/MarsParser.h" #include "metkit/mars/MarsExpension.h" +#include "metkit/mars/MarsParser.h" #include "metkit/mars/MarsRequest.h" #include "fdb5/LibFdb5.h" -#include "fdb5/message/MessageArchiver.h" #include "fdb5/database/ArchiveVisitor.h" +#include "fdb5/message/MessageArchiver.h" // For HAVE_FAIL_ON_CCSDS #include "metkit/metkit_config.h" @@ -43,7 +43,7 @@ std::vector str_to_requests(const std::string& str) { // parse requests - std::string rs = std::string("retrieve,") + str; + std::string rs = std::string("retrieve,") + str; LOG_DEBUG_LIB(LibFdb5) << "Parsing request string : " << rs << std::endl; @@ -74,7 +74,8 @@ std::vector str_to_requests(const std::string& str) { std::vector make_filter_requests(const std::string& str) { - if(str.empty()) return {}; + if (str.empty()) + return {}; std::set keys = fdb5::Key::parseString(str).keys(); //< keys to filter from that request @@ -89,18 +90,16 @@ std::vector make_filter_requests(const std::string& s return r; } -} +} // namespace //---------------------------------------------------------------------------------------------------------------------- -MessageArchiver::MessageArchiver(const fdb5::Key& key, bool completeTransfers, bool verbose, const Config& config) : - MessageDecoder(), - fdb_(config), - key_(key), - completeTransfers_(completeTransfers), - verbose_(verbose) -{ -} +MessageArchiver::MessageArchiver(const fdb5::Key& key, bool completeTransfers, bool verbose, const Config& config) + : MessageDecoder() + , fdb_(config) + , key_(key) + , completeTransfers_(completeTransfers) + , verbose_(verbose) {} void MessageArchiver::filters(const std::string& include, const std::string& exclude) { include_ = make_filter_requests(include); @@ -116,9 +115,9 @@ void MessageArchiver::modifiers(const std::string& modify) { // Log::info() << "pairs : " << pairs << std::endl; - for(auto& pair: pairs) { + for (auto& pair : pairs) { std::vector kv = equal.tokenize(pair); - if(kv.size() != 2) + if (kv.size() != 2) throw eckit::BadValue("Invalid key-value pair " + pair); // Log::info() << "kv : " << kv[0] << " = " << kv[1] << std::endl; modifiers_[kv[0]] = kv[1]; @@ -132,7 +131,8 @@ eckit::message::Message MessageArchiver::transform(eckit::message::Message& msg) static bool matchAny(const metkit::mars::MarsRequest& f, const std::vector& v) { for (auto r = v.begin(); r != v.end(); ++r) { - if(f.matches(*r)) return true; + if (f.matches(*r)) + return true; } return false; } @@ -150,11 +150,13 @@ bool MessageArchiver::filterOut(const Key& k) const { // filter includes - if(include_.size() && not matchAny(field, include_)) return out; + if (include_.size() && not matchAny(field, include_)) + return out; // filter excludes - if(exclude_.size() && matchAny(field, exclude_)) return out; + if (exclude_.size() && matchAny(field, exclude_)) + return out; // datum wasn't filtered out @@ -176,11 +178,11 @@ eckit::Length MessageArchiver::archive(eckit::DataHandle& source) { eckit::message::Message msg; - while ( (msg = reader.next()) ) { + while ((msg = reader.next())) { #ifdef metkit_HAVE_FAIL_ON_CCSDS - if(msg.getString("packingType") == "grid_ccsds") { + if (msg.getString("packingType") == "grid_ccsds") { throw eckit::SeriousBug("grid_ccsds is disabled"); } @@ -191,8 +193,7 @@ eckit::Length MessageArchiver::archive(eckit::DataHandle& source) { messageToKey(msg, key); LOG_DEBUG_LIB(LibFdb5) << "Archiving message " - << " key: " << key << " length:" << msg.length() - << std::endl; + << " key: " << key << " length:" << msg.length() << std::endl; ASSERT(key.match(key_)); @@ -202,7 +203,7 @@ eckit::Length MessageArchiver::archive(eckit::DataHandle& source) { if (modifiers_.size()) { msg = transform(msg); key.clear(); - messageToKey(msg, key); // re-build the key, as it may have changed + messageToKey(msg, key); // re-build the key, as it may have changed } if (verbose_) { @@ -225,7 +226,8 @@ eckit::Length MessageArchiver::archive(eckit::DataHandle& source) { if (completeTransfers_) { eckit::Log::error() << "Exception received. Completing transfer." << std::endl; // Consume rest of datahandle otherwise client retries for ever - while ( reader.next() ) { /* empty */ } + while (reader.next()) { /* empty */ + } } throw; } diff --git a/src/fdb5/message/MessageArchiver.h b/src/fdb5/message/MessageArchiver.h index f274290ee..23b200cc1 100644 --- a/src/fdb5/message/MessageArchiver.h +++ b/src/fdb5/message/MessageArchiver.h @@ -21,11 +21,11 @@ #include "metkit/mars/MarsRequest.h" -#include "fdb5/database/Archiver.h" +#include "fdb5/api/FDB.h" #include "fdb5/config/Config.h" -#include "fdb5/message/MessageDecoder.h" +#include "fdb5/database/Archiver.h" #include "fdb5/database/Key.h" -#include "fdb5/api/FDB.h" +#include "fdb5/message/MessageDecoder.h" namespace eckit { class DataHandle; @@ -38,27 +38,22 @@ namespace fdb5 { class MessageArchiver : public MessageDecoder { public: // methods - - MessageArchiver(const fdb5::Key& key = Key(), - bool completeTransfers = false, - bool verbose = false, - const Config& config = Config().expandConfig()); + MessageArchiver(const fdb5::Key& key = Key(), bool completeTransfers = false, bool verbose = false, + const Config& config = Config().expandConfig()); void filters(const std::string& include, const std::string& exclude); void modifiers(const std::string& modify); - eckit::Length archive(eckit::DataHandle &source); + eckit::Length archive(eckit::DataHandle& source); void flush(); private: // protected - bool filterOut(const Key& k) const; eckit::message::Message transform(eckit::message::Message&); private: // members - FDB fdb_; fdb5::Key key_; diff --git a/src/fdb5/message/MessageDecoder.cc b/src/fdb5/message/MessageDecoder.cc old mode 100755 new mode 100644 index cd71d57d0..62526c5d3 --- a/src/fdb5/message/MessageDecoder.cc +++ b/src/fdb5/message/MessageDecoder.cc @@ -13,8 +13,8 @@ #include "fdb5/message/MessageDecoder.h" -#include "eckit/message/Reader.h" #include "eckit/message/Message.h" +#include "eckit/message/Reader.h" #include "metkit/mars/MarsExpandContext.h" #include "metkit/mars/MarsLanguage.h" @@ -22,12 +22,10 @@ namespace fdb5 { -namespace { +namespace { class KeySetter : public eckit::message::MetadataGatherer { - void setValue(const std::string& key, const std::string& value) override { - key_.set(key, value); - } + void setValue(const std::string& key, const std::string& value) override { key_.set(key, value); } void setValue(const std::string& key, long value) override { if (key_.find(key) == key_.end()) { @@ -45,19 +43,14 @@ class KeySetter : public eckit::message::MetadataGatherer { Key& key_; public: - - KeySetter(Key& key): key_(key) { - ASSERT(key_.empty()); - } + KeySetter(Key& key) : key_(key) { ASSERT(key_.empty()); } }; -} // namespace +} // namespace //---------------------------------------------------------------------------------------------------------------------- -MessageDecoder::MessageDecoder(bool checkDuplicates): - checkDuplicates_(checkDuplicates) { -} +MessageDecoder::MessageDecoder(bool checkDuplicates) : checkDuplicates_(checkDuplicates) {} MessageDecoder::~MessageDecoder() {} @@ -67,7 +60,6 @@ Key MessageDecoder::messageToKey(const eckit::message::Message& msg) { return key; } - void MessageDecoder::msgToKey(const eckit::message::Message& msg, Key& key) { KeySetter setter(key); @@ -82,28 +74,28 @@ void MessageDecoder::messageToKey(const eckit::message::Message& msg, Key& key) msgToKey(patched, key); - if ( checkDuplicates_ ) { - if ( seen_.find(key) != seen_.end() ) { + if (checkDuplicates_) { + if (seen_.find(key) != seen_.end()) { std::ostringstream oss; oss << "Message has duplicate parameters in the same request: " << key; - throw eckit::SeriousBug( oss.str() ); + throw eckit::SeriousBug(oss.str()); } seen_.insert(key); } } -metkit::mars::MarsRequest MessageDecoder::messageToRequest(const eckit::PathName &path, const char *verb) { +metkit::mars::MarsRequest MessageDecoder::messageToRequest(const eckit::PathName& path, const char* verb) { metkit::mars::MarsRequest request(verb); - for (auto& r: messageToRequests(path, verb)) + for (auto& r : messageToRequests(path, verb)) request.merge(r); return request; } - -std::vector MessageDecoder::messageToRequests(const eckit::PathName &path, const char *verb) { +std::vector MessageDecoder::messageToRequests(const eckit::PathName& path, + const char* verb) { std::vector requests; eckit::message::Reader reader(path); @@ -111,7 +103,7 @@ std::vector MessageDecoder::messageToRequests(const e std::map> s; - while ( (msg = reader.next()) ) { + while ((msg = reader.next())) { Key key; @@ -128,7 +120,6 @@ eckit::message::Message MessageDecoder::patch(const eckit::message::Message& msg return msg; } - //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 diff --git a/src/fdb5/message/MessageDecoder.h b/src/fdb5/message/MessageDecoder.h old mode 100755 new mode 100644 index 6738fad4d..8ae86fb30 --- a/src/fdb5/message/MessageDecoder.h +++ b/src/fdb5/message/MessageDecoder.h @@ -16,7 +16,6 @@ #ifndef fdb5_MessageDecoder_H #define fdb5_MessageDecoder_H - #include "eckit/io/Buffer.h" #include "metkit/mars/MarsRequest.h" #include @@ -31,8 +30,8 @@ namespace metkit { namespace data { class Reader; class Message; -} -} +} // namespace data +} // namespace metkit #include "fdb5/database/Key.h" @@ -40,22 +39,19 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- - class MessageDecoder { public: - MessageDecoder(bool checkDuplicates = false); virtual ~MessageDecoder(); static Key messageToKey(const eckit::message::Message& msg); void messageToKey(const eckit::message::Message& msg, Key& key); - metkit::mars::MarsRequest messageToRequest(const eckit::PathName &path, const char *verb = "retrieve"); - std::vector messageToRequests(const eckit::PathName &path, const char *verb = "retrieve"); - + metkit::mars::MarsRequest messageToRequest(const eckit::PathName& path, const char* verb = "retrieve"); + std::vector messageToRequests(const eckit::PathName& path, + const char* verb = "retrieve"); private: - virtual eckit::message::Message patch(const eckit::message::Message& msg); static void msgToKey(const eckit::message::Message& msg, Key& key); diff --git a/src/fdb5/message/MessageIndexer.cc b/src/fdb5/message/MessageIndexer.cc index 43e698886..a0452d53b 100644 --- a/src/fdb5/message/MessageIndexer.cc +++ b/src/fdb5/message/MessageIndexer.cc @@ -8,28 +8,25 @@ * does it submit to any jurisdiction. */ -#include "eckit/log/Timer.h" -#include "eckit/log/Plural.h" #include "eckit/log/Bytes.h" -#include "eckit/log/Seconds.h" +#include "eckit/log/Plural.h" #include "eckit/log/Progress.h" +#include "eckit/log/Seconds.h" +#include "eckit/log/Timer.h" -#include "eckit/message/Reader.h" #include "eckit/message/Message.h" +#include "eckit/message/Reader.h" #include "fdb5/message/MessageIndexer.h" #include "fdb5/toc/AdoptVisitor.h" - namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -MessageIndexer::MessageIndexer(bool checkDuplicates) : - MessageDecoder(checkDuplicates) { -} +MessageIndexer::MessageIndexer(bool checkDuplicates) : MessageDecoder(checkDuplicates) {} -void MessageIndexer::index(const eckit::PathName &path) { +void MessageIndexer::index(const eckit::PathName& path) { eckit::Timer timer("fdb::service::archive"); eckit::message::Reader reader(path); @@ -44,7 +41,7 @@ void MessageIndexer::index(const eckit::PathName &path) { eckit::PathName full(path.realName()); eckit::message::Message msg; - while ( (msg = reader.next()) ) { + while ((msg = reader.next())) { Key key; @@ -66,9 +63,8 @@ void MessageIndexer::index(const eckit::PathName &path) { eckit::Log::info() << "FDB indexer " << eckit::Plural(count, "message") << "," << " size " << eckit::Bytes(total_size) << "," - << " in " << eckit::Seconds(timer.elapsed()) - << " (" << eckit::Bytes(total_size, timer) << ")" << std::endl; - + << " in " << eckit::Seconds(timer.elapsed()) << " (" << eckit::Bytes(total_size, timer) << ")" + << std::endl; } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/message/MessageIndexer.h b/src/fdb5/message/MessageIndexer.h index 9961a041d..0c5ffe2e6 100644 --- a/src/fdb5/message/MessageIndexer.h +++ b/src/fdb5/message/MessageIndexer.h @@ -21,7 +21,9 @@ #include "fdb5/database/Archiver.h" #include "fdb5/message/MessageDecoder.h" -namespace eckit { class DataHandle; } +namespace eckit { +class DataHandle; +} namespace fdb5 { @@ -30,14 +32,11 @@ namespace fdb5 { class MessageIndexer : public Archiver, public MessageDecoder { public: // methods - MessageIndexer(bool checkDuplicates = false); void index(const eckit::PathName& path); private: // members - - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/rados/RadosFieldLocation.cc b/src/fdb5/rados/RadosFieldLocation.cc index cf37be670..4c1c773c5 100644 --- a/src/fdb5/rados/RadosFieldLocation.cc +++ b/src/fdb5/rados/RadosFieldLocation.cc @@ -8,34 +8,32 @@ * does it submit to any jurisdiction. */ -#include "eckit/io/rados/RadosReadHandle.h" #include "fdb5/rados/RadosFieldLocation.h" +#include "eckit/io/rados/RadosReadHandle.h" #include "fdb5/LibFdb5.h" #include "fdb5/io/SingleGribMungePartFileHandle.h" namespace fdb5 { -::eckit::ClassSpec RadosFieldLocation::classSpec_ = {&FieldLocation::classSpec(), "RadosFieldLocation",}; +::eckit::ClassSpec RadosFieldLocation::classSpec_ = { + &FieldLocation::classSpec(), + "RadosFieldLocation", +}; ::eckit::Reanimator RadosFieldLocation::reanimator_; //---------------------------------------------------------------------------------------------------------------------- -RadosFieldLocation::RadosFieldLocation(const eckit::PathName path, eckit::Offset offset, eckit::Length length ) : - FieldLocation(eckit::URI("rados", path), offset, length) {} +RadosFieldLocation::RadosFieldLocation(const eckit::PathName path, eckit::Offset offset, eckit::Length length) + : FieldLocation(eckit::URI("rados", path), offset, length) {} -RadosFieldLocation::RadosFieldLocation(const eckit::URI &uri) : - FieldLocation(uri) {} - -RadosFieldLocation::RadosFieldLocation(const eckit::URI &uri, eckit::Offset offset, eckit::Length length ) : - FieldLocation(uri, offset, length) { -} +RadosFieldLocation::RadosFieldLocation(const eckit::URI& uri) : FieldLocation(uri) {} -RadosFieldLocation::RadosFieldLocation(const RadosFieldLocation& rhs) : - FieldLocation(rhs.uri_) {} +RadosFieldLocation::RadosFieldLocation(const eckit::URI& uri, eckit::Offset offset, eckit::Length length) + : FieldLocation(uri, offset, length) {} -RadosFieldLocation::RadosFieldLocation(eckit::Stream& s) : - FieldLocation(s) {} +RadosFieldLocation::RadosFieldLocation(const RadosFieldLocation& rhs) : FieldLocation(rhs.uri_) {} +RadosFieldLocation::RadosFieldLocation(eckit::Stream& s) : FieldLocation(s) {} std::shared_ptr RadosFieldLocation::make_shared() const { return std::make_shared(std::move(*this)); @@ -47,11 +45,11 @@ eckit::DataHandle* RadosFieldLocation::dataHandle() const { return g; } -eckit::DataHandle *RadosFieldLocation::dataHandle(const Key& remapKey) const { +eckit::DataHandle* RadosFieldLocation::dataHandle(const Key& remapKey) const { return new SingleGribMungePartFileHandle(path(), offset(), length(), remapKey); } -void RadosFieldLocation::print(std::ostream &out) const { +void RadosFieldLocation::print(std::ostream& out) const { out << "RadosFieldLocation[uri=" << uri_ << "]"; } @@ -59,7 +57,7 @@ void RadosFieldLocation::visit(FieldLocationVisitor& visitor) const { visitor(*this); } -eckit::URI RadosFieldLocation::uri(const eckit::PathName &path) { +eckit::URI RadosFieldLocation::uri(const eckit::PathName& path) { return eckit::URI("rados", path); } diff --git a/src/fdb5/rados/RadosFieldLocation.h b/src/fdb5/rados/RadosFieldLocation.h index 6d04ced4d..e12b6de26 100644 --- a/src/fdb5/rados/RadosFieldLocation.h +++ b/src/fdb5/rados/RadosFieldLocation.h @@ -27,11 +27,10 @@ namespace fdb5 { class RadosFieldLocation : public FieldLocation { public: - RadosFieldLocation(const RadosFieldLocation& rhs); RadosFieldLocation(const eckit::PathName path, eckit::Offset offset, eckit::Length length); - RadosFieldLocation(const eckit::URI &uri); - RadosFieldLocation(const eckit::URI &uri, eckit::Offset offset, eckit::Length length); + RadosFieldLocation(const eckit::URI& uri); + RadosFieldLocation(const eckit::URI& uri, eckit::Offset offset, eckit::Length length); RadosFieldLocation(eckit::Stream&); eckit::DataHandle* dataHandle() const override; eckit::DataHandle* dataHandle(const Key& remapKey) const override; @@ -39,21 +38,18 @@ class RadosFieldLocation : public FieldLocation { void visit(FieldLocationVisitor& visitor) const override; public: // For Streamable - - static const eckit::ClassSpec& classSpec() { return classSpec_;} + static const eckit::ClassSpec& classSpec() { return classSpec_; } protected: // For Streamable - const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } - static eckit::ClassSpec classSpec_; + static eckit::ClassSpec classSpec_; static eckit::Reanimator reanimator_; private: // methods - void print(std::ostream &out) const override; - eckit::URI uri(const eckit::PathName &path); + void print(std::ostream& out) const override; + eckit::URI uri(const eckit::PathName& path); }; - //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index fe0ab6f2b..d18fea9de 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -8,19 +8,19 @@ * does it submit to any jurisdiction. */ -#include "eckit/log/Timer.h" #include "eckit/log/Bytes.h" +#include "eckit/log/Timer.h" #include "eckit/config/Resource.h" #include "eckit/io/EmptyHandle.h" #include "eckit/io/rados/RadosWriteHandle.h" #include "fdb5/LibFdb5.h" -#include "fdb5/rules/Rule.h" #include "fdb5/database/FieldLocation.h" +#include "fdb5/io/FDBFileHandle.h" #include "fdb5/rados/RadosFieldLocation.h" #include "fdb5/rados/RadosStore.h" -#include "fdb5/io/FDBFileHandle.h" +#include "fdb5/rules/Rule.h" using namespace eckit; @@ -28,11 +28,15 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -RadosStore::RadosStore(const Key& key, const Config& config) : - Store(), directory_("mars:"+key.valuesToString()), archivedFields_(0) {} +RadosStore::RadosStore(const Key& key, const Config& config) + : Store() + , directory_("mars:" + key.valuesToString()) + , archivedFields_(0) {} -RadosStore(const Key& key, const Config& config, const eckit::net::Endpoint& controlEndpoint) : - Store(), directory_("mars:"+key.valuesToString()), archivedFields_(0) { +RadosStore(const Key& key, const Config& config, const eckit::net::Endpoint& controlEndpoint) + : Store() + , directory_("mars:" + key.valuesToString()) + , archivedFields_(0) { NOTIMP; } @@ -45,22 +49,21 @@ bool RadosStore::exists() const { } eckit::DataHandle* RadosStore::retrieve(Field& field, Key& remapKey) const { - return remapKey.empty() ? - field.dataHandle() : - field.dataHandle(remapKey); + return remapKey.empty() ? field.dataHandle() : field.dataHandle(remapKey); } -std::unique_ptr RadosStore::archive(const uint32_t, const Key& key, const void *data, eckit::Length length) { +std::unique_ptr RadosStore::archive(const uint32_t, const Key& key, const void* data, + eckit::Length length) { archivedFields_++; eckit::PathName dataPath = getDataPath(key); eckit::URI dataUri("rados", dataPath); - eckit::DataHandle &dh = getDataHandle(dataPath); + eckit::DataHandle& dh = getDataHandle(dataPath); eckit::Offset position = dh.position(); - long len = dh.write( data, length ); + long len = dh.write(data, length); ASSERT(len == length); @@ -92,67 +95,70 @@ void RadosStore::remove(const eckit::URI& uri, std::ostream& logAlways, std::ost if (path.isDir()) { logVerbose << "rmdir: "; logAlways << path << std::endl; - if (doit) path.rmdir(false); + if (doit) + path.rmdir(false); } else { logVerbose << "Unlinking: "; logAlways << path << std::endl; - if (doit) path.unlink(false); + if (doit) + path.unlink(false); } } -eckit::DataHandle *RadosStore::getCachedHandle( const eckit::PathName &path ) const { - HandleStore::const_iterator j = handles_.find( path ); - if ( j != handles_.end() ) +eckit::DataHandle* RadosStore::getCachedHandle(const eckit::PathName& path) const { + HandleStore::const_iterator j = handles_.find(path); + if (j != handles_.end()) return j->second; else return nullptr; } void RadosStore::closeDataHandles() { - for ( HandleStore::iterator j = handles_.begin(); j != handles_.end(); ++j ) { - eckit::DataHandle *dh = j->second; + for (HandleStore::iterator j = handles_.begin(); j != handles_.end(); ++j) { + eckit::DataHandle* dh = j->second; dh->close(); delete dh; } handles_.clear(); } -eckit::DataHandle *RadosStore::createFileHandle(const eckit::PathName &path) { +eckit::DataHandle* RadosStore::createFileHandle(const eckit::PathName& path) { -// static size_t sizeBuffer = eckit::Resource("fdbBufferSize", 64 * 1024 * 1024); + // static size_t sizeBuffer = eckit::Resource("fdbBufferSize", 64 * 1024 * 1024); - LOG_DEBUG_LIB(LibFdb5) << "Creating RadosWriteHandle to " << path -// << " with buffer of " << eckit::Bytes(sizeBuffer) - << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "Creating RadosWriteHandle to " + << path + // << " with buffer of " << eckit::Bytes(sizeBuffer) + << std::endl; return new RadosWriteHandle(path, 0); } -eckit::DataHandle *RadosStore::createAsyncHandle(const eckit::PathName &path) { +eckit::DataHandle* RadosStore::createAsyncHandle(const eckit::PathName& path) { NOTIMP; -/* static size_t nbBuffers = eckit::Resource("fdbNbAsyncBuffers", 4); - static size_t sizeBuffer = eckit::Resource("fdbSizeAsyncBuffer", 64 * 1024 * 1024); + /* static size_t nbBuffers = eckit::Resource("fdbNbAsyncBuffers", 4); + static size_t sizeBuffer = eckit::Resource("fdbSizeAsyncBuffer", 64 * 1024 * 1024); - return new eckit::AIOHandle(path, nbBuffers, sizeBuffer);*/ + return new eckit::AIOHandle(path, nbBuffers, sizeBuffer);*/ } -eckit::DataHandle *RadosStore::createDataHandle(const eckit::PathName &path) { +eckit::DataHandle* RadosStore::createDataHandle(const eckit::PathName& path) { static bool fdbWriteToNull = eckit::Resource("fdbWriteToNull;$FDB_WRITE_TO_NULL", false); - if(fdbWriteToNull) + if (fdbWriteToNull) return new eckit::EmptyHandle(); static bool fdbAsyncWrite = eckit::Resource("fdbAsyncWrite;$FDB_ASYNC_WRITE", false); - if(fdbAsyncWrite) + if (fdbAsyncWrite) return createAsyncHandle(path); return createFileHandle(path); } -eckit::DataHandle& RadosStore::getDataHandle( const eckit::PathName &path ) { - eckit::DataHandle *dh = getCachedHandle(path); - if ( !dh ) { +eckit::DataHandle& RadosStore::getDataHandle(const eckit::PathName& path) { + eckit::DataHandle* dh = getCachedHandle(path); + if (!dh) { dh = createDataHandle(path); ASSERT(dh); handles_[path] = dh; @@ -163,20 +169,20 @@ eckit::DataHandle& RadosStore::getDataHandle( const eckit::PathName &path ) { eckit::PathName RadosStore::generateDataPath(const Key& key) const { - eckit::PathName dpath ( directory_ ); - dpath /= key.valuesToString(); + eckit::PathName dpath(directory_); + dpath /= key.valuesToString(); dpath = eckit::PathName::unique(dpath) + ".data"; return dpath; } eckit::PathName RadosStore::getDataPath(const Key& key) { PathStore::const_iterator j = dataPaths_.find(key); - if ( j != dataPaths_.end() ) + if (j != dataPaths_.end()) return j->second; eckit::PathName dataPath = generateDataPath(key); - dataPaths_[ key ] = dataPath; + dataPaths_[key] = dataPath; return dataPath; } @@ -184,12 +190,12 @@ eckit::PathName RadosStore::getDataPath(const Key& key) { void RadosStore::flushDataHandles() { for (HandleStore::iterator j = handles_.begin(); j != handles_.end(); ++j) { - eckit::DataHandle *dh = j->second; + eckit::DataHandle* dh = j->second; dh->flush(); } } -void RadosStore::print(std::ostream &out) const { +void RadosStore::print(std::ostream& out) const { out << "RadosStore(" << directory_ << ")"; } diff --git a/src/fdb5/rados/RadosStore.h b/src/fdb5/rados/RadosStore.h index c9d7db8b4..b24cca1b1 100644 --- a/src/fdb5/rados/RadosStore.h +++ b/src/fdb5/rados/RadosStore.h @@ -29,7 +29,6 @@ namespace fdb5 { class RadosStore : public Store { public: // methods - RadosStore(const Schema& schema, const Key& key, const Config& config); ~RadosStore() override {} @@ -43,40 +42,38 @@ class RadosStore : public Store { void checkUID() const override { /* nothing to do */ } protected: // methods - std::string type() const override { return "rados"; } bool exists() const override; eckit::DataHandle* retrieve(Field& field, Key& remapKey) const override; - std::unique_ptr archive(const uint32_t, const Key& key, const void *data, eckit::Length length) override; + std::unique_ptr archive(const uint32_t, const Key& key, const void* data, + eckit::Length length) override; void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override; - eckit::DataHandle *getCachedHandle( const eckit::PathName &path ) const; + eckit::DataHandle* getCachedHandle(const eckit::PathName& path) const; void closeDataHandles(); - eckit::DataHandle *createFileHandle(const eckit::PathName &path); - eckit::DataHandle *createAsyncHandle(const eckit::PathName &path); - eckit::DataHandle *createDataHandle(const eckit::PathName &path); - eckit::DataHandle& getDataHandle( const eckit::PathName &path ); + eckit::DataHandle* createFileHandle(const eckit::PathName& path); + eckit::DataHandle* createAsyncHandle(const eckit::PathName& path); + eckit::DataHandle* createDataHandle(const eckit::PathName& path); + eckit::DataHandle& getDataHandle(const eckit::PathName& path); eckit::PathName generateDataPath(const Key& key) const; eckit::PathName getDataPath(const Key& key); void flushDataHandles(); - void print( std::ostream &out ) const override; + void print(std::ostream& out) const override; private: // types + typedef std::map HandleStore; + typedef std::map PathStore; - typedef std::map< std::string, eckit::DataHandle * > HandleStore; - typedef std::map< Key, std::string > PathStore; - -private: // members +private: // members + HandleStore handles_; ///< stores the DataHandles being used by the Session - HandleStore handles_; ///< stores the DataHandles being used by the Session - - PathStore dataPaths_; + PathStore dataPaths_; eckit::PathName directory_; - + size_t archivedFields_; }; @@ -84,4 +81,4 @@ class RadosStore : public Store { } // namespace fdb5 -#endif //fdb5_RadosStore_H +#endif // fdb5_RadosStore_H diff --git a/src/fdb5/remote/Connection.cc b/src/fdb5/remote/Connection.cc index f4468f85f..7046b2390 100644 --- a/src/fdb5/remote/Connection.cc +++ b/src/fdb5/remote/Connection.cc @@ -18,15 +18,15 @@ void Connection::teardown() { try { // all done - disconnecting Connection::write(Message::Exit, false, 0, 0); - } catch(...) { - // if connection is already down, no need to escalate + } catch (...) { + // if connection is already down, no need to escalate } } try { // all done - disconnecting Connection::write(Message::Exit, true, 0, 0); - } catch(...) { - // if connection is already down, no need to escalate + } catch (...) { + // if connection is already down, no need to escalate } } @@ -90,34 +90,38 @@ eckit::Buffer Connection::read(bool control, MessageHeader& hdr) { if (hdr.message == Message::Error) { - char msg[hdr.payloadSize+1]; + char msg[hdr.payloadSize + 1]; if (hdr.payloadSize) { - char msg[hdr.payloadSize+1]; + char msg[hdr.payloadSize + 1]; } } return payload; } -void Connection::write(remote::Message msg, bool control, uint32_t clientID, uint32_t requestID, const void* data, uint32_t length) { +void Connection::write(remote::Message msg, bool control, uint32_t clientID, uint32_t requestID, const void* data, + uint32_t length) { write(msg, control, clientID, requestID, std::vector>{{data, length}}); } -void Connection::write(remote::Message msg, bool control, uint32_t clientID, uint32_t requestID, std::vector> data) { +void Connection::write(remote::Message msg, bool control, uint32_t clientID, uint32_t requestID, + std::vector> data) { uint32_t payloadLength = 0; - for (auto d: data) { + for (auto d : data) { ASSERT(d.first); payloadLength += d.second; } MessageHeader message{msg, control, clientID, requestID, payloadLength}; - LOG_DEBUG_LIB(LibFdb5) << "Connection::write [message=" << msg << ",clientID=" << message.clientID() << ",control=" << control << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "Connection::write [message=" << msg << ",clientID=" << message.clientID() + << ",control=" << control << ",requestID=" << requestID << ",data=" << data.size() + << ",payload=" << payloadLength << "]" << std::endl; std::lock_guard lock((control || single_) ? controlMutex_ : dataMutex_); writeUnsafe(control, &message, sizeof(message)); - for (auto d: data) { + for (auto d : data) { writeUnsafe(control, d.first, d.second); } writeUnsafe(control, &EndMarker, sizeof(EndMarker)); @@ -125,7 +129,8 @@ void Connection::write(remote::Message msg, bool control, uint32_t clientID, uin void Connection::error(const std::string& msg, uint32_t clientID, uint32_t requestID) { eckit::Log::error() << "[clientID=" << clientID << ",requestID=" << requestID << "] " << msg << std::endl; - write(Message::Error, false, clientID, requestID, std::vector>{{msg.c_str(), msg.length()}}); + write(Message::Error, false, clientID, requestID, + std::vector>{{msg.c_str(), msg.length()}}); } eckit::Buffer Connection::readControl(MessageHeader& hdr) { @@ -136,4 +141,4 @@ eckit::Buffer Connection::readData(MessageHeader& hdr) { return read(false, hdr); } -} // namespace fdb5::remote +} // namespace fdb5::remote diff --git a/src/fdb5/remote/Connection.h b/src/fdb5/remote/Connection.h index 560f8b01c..2e8a8e65b 100644 --- a/src/fdb5/remote/Connection.h +++ b/src/fdb5/remote/Connection.h @@ -28,8 +28,8 @@ namespace fdb5::remote { class TCPException : public eckit::Exception { public: - TCPException(const std::string& msg, const eckit::CodeLocation& here) : - eckit::Exception(std::string("TCPException: ") + msg, here) { + TCPException(const std::string& msg, const eckit::CodeLocation& here) + : eckit::Exception(std::string("TCPException: ") + msg, here) { std::cerr << "TCP Exception; backtrace(): " << std::endl; std::cerr << eckit::BackTrace::dump() << std::endl; @@ -45,7 +45,8 @@ class Connection : eckit::NonCopyable { virtual ~Connection(); void write(Message msg, bool control, uint32_t clientID, uint32_t requestID, const void* data, uint32_t length); - void write(Message msg, bool control, uint32_t clientID, uint32_t requestID, std::vector> data = {}); + void write(Message msg, bool control, uint32_t clientID, uint32_t requestID, + std::vector> data = {}); void error(const std::string& msg, uint32_t clientID, uint32_t requestID); @@ -55,7 +56,6 @@ class Connection : eckit::NonCopyable { void teardown(); private: // methods - eckit::Buffer read(bool control, MessageHeader& hdr); void writeUnsafe(bool control, const void* data, size_t length); @@ -65,18 +65,15 @@ class Connection : eckit::NonCopyable { virtual eckit::net::TCPSocket& dataSocket() = 0; protected: // members - bool single_; private: // members - std::mutex controlMutex_; std::mutex dataMutex_; std::mutex readControlMutex_; std::mutex readDataMutex_; - }; //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5::remote +} // namespace fdb5::remote diff --git a/src/fdb5/remote/FdbServer.cc b/src/fdb5/remote/FdbServer.cc index b4f7d60d3..13d6ffb3f 100644 --- a/src/fdb5/remote/FdbServer.cc +++ b/src/fdb5/remote/FdbServer.cc @@ -22,23 +22,21 @@ #include "fdb5/remote/FdbServer.h" +#include "eckit/config/Resource.h" #include "fdb5/remote/server/AvailablePortList.h" #include "fdb5/remote/server/CatalogueHandler.h" #include "fdb5/remote/server/StoreHandler.h" -#include "eckit/config/Resource.h" using namespace eckit; - namespace fdb5::remote { //---------------------------------------------------------------------------------------------------------------------- -FDBForker::FDBForker(net::TCPSocket &socket, const Config &config) : - ProcessControler(true), - socket_(socket), - config_(config) { - } +FDBForker::FDBForker(net::TCPSocket& socket, const Config& config) + : ProcessControler(true) + , socket_(socket) + , config_(config) {} FDBForker::~FDBForker() {} @@ -50,14 +48,17 @@ void FDBForker::run() { ::srand(::getpid() + ::time(nullptr)); ::srandom(::getpid() + ::time(nullptr)); - eckit::Log::info() << "FDB forked pid " << ::getpid() << " -- connection: " << socket_.localHost() << ":" << socket_.localPort() << "-->" << socket_.remoteHost() << ":" << socket_.remotePort() << std::endl; + eckit::Log::info() << "FDB forked pid " << ::getpid() << " -- connection: " << socket_.localHost() << ":" + << socket_.localPort() << "-->" << socket_.remoteHost() << ":" << socket_.remotePort() + << std::endl; - if (config_.getString("type", "local") == "catalogue" || (::getenv("FDB_IS_CAT") && ::getenv("FDB_IS_CAT")[0] == '1')) { + if (config_.getString("type", "local") == "catalogue" || + (::getenv("FDB_IS_CAT") && ::getenv("FDB_IS_CAT")[0] == '1')) { eckit::Log::info() << "FDB using Catalogue Handler" << std::endl; CatalogueHandler handler(socket_, config_); handler.handle(); - } - else if (config_.getString("type", "local") == "store" || (::getenv("FDB_IS_STORE") && ::getenv("FDB_IS_STORE")[0] == '1')) { + } else if (config_.getString("type", "local") == "store" || + (::getenv("FDB_IS_STORE") && ::getenv("FDB_IS_STORE")[0] == '1')) { eckit::Log::info() << "FDB using Store Handler" << std::endl; StoreHandler handler(socket_, config_); handler.handle(); @@ -69,35 +70,30 @@ void FDBForker::run() { class FDBServerThread : public eckit::Thread { public: // methods - FDBServerThread(eckit::net::TCPSocket& socket, const Config& config); private: // methods - virtual void run(); private: // members - eckit::net::TCPSocket socket_; eckit::LocalConfiguration config_; - }; //---------------------------------------------------------------------------------------------------------------------- -FDBServerThread::FDBServerThread(net::TCPSocket& socket, const Config& config) : - socket_(socket), - config_(config) {} +FDBServerThread::FDBServerThread(net::TCPSocket& socket, const Config& config) : socket_(socket), config_(config) {} void FDBServerThread::run() { eckit::Log::info() << "FDB started handler thread" << std::endl; - if (config_.getString("type", "local") == "catalogue" || (::getenv("FDB_IS_CAT") && ::getenv("FDB_IS_CAT")[0] == '1')) { + if (config_.getString("type", "local") == "catalogue" || + (::getenv("FDB_IS_CAT") && ::getenv("FDB_IS_CAT")[0] == '1')) { eckit::Log::info() << "FDB using Catalogue Handler" << std::endl; CatalogueHandler handler(socket_, config_); handler.handle(); - } - else if (config_.getString("type", "local") == "store" || (::getenv("FDB_IS_STORE") && ::getenv("FDB_IS_STORE")[0] == '1')) { + } else if (config_.getString("type", "local") == "store" || + (::getenv("FDB_IS_STORE") && ::getenv("FDB_IS_STORE")[0] == '1')) { eckit::Log::info() << "FDB using Store Handler" << std::endl; StoreHandler handler(socket_, config_); handler.handle(); @@ -134,20 +130,18 @@ void FdbServerBase::doRun() { if (threaded) { ThreadControler t(new FDBServerThread(server.accept(), config)); t.start(); - } - else { + } else { FDBForker f(server.accept(), config); f.start(); } - } - catch (std::exception& e) { + } catch (std::exception& e) { eckit::Log::error() << "** " << e.what() << " Caught in " << Here() << std::endl; eckit::Log::error() << "** Exception is ignored" << std::endl; } } } -void FdbServerBase::startPortReaperThread(const Config &config) { +void FdbServerBase::startPortReaperThread(const Config& config) { if (config.has("dataPortStart")) { ASSERT(config.has("dataPortCount")); @@ -155,14 +149,12 @@ void FdbServerBase::startPortReaperThread(const Config &config) { int startPort = config.getInt("dataPortStart"); size_t count = config.getLong("dataPortCount"); - eckit::Log::info() << "Using custom port list. startPort=" << startPort - << ", count=" << count << std::endl; + eckit::Log::info() << "Using custom port list. startPort=" << startPort << ", count=" << count << std::endl; AvailablePortList portList(startPort, count); portList.initialise(); reaperThread_ = std::thread([startPort, count]() { - AvailablePortList portList(startPort, count); while (true) { @@ -171,16 +163,11 @@ void FdbServerBase::startPortReaperThread(const Config &config) { } }); } - } //---------------------------------------------------------------------------------------------------------------------- -FdbServer::FdbServer(int argc, char **argv, const char *home) : - eckit::Application(argc, argv, home), - FdbServerBase() -{ -} +FdbServer::FdbServer(int argc, char** argv, const char* home) : eckit::Application(argc, argv, home), FdbServerBase() {} FdbServer::~FdbServer() {} diff --git a/src/fdb5/remote/FdbServer.h b/src/fdb5/remote/FdbServer.h index a3e08b6cf..11e059780 100644 --- a/src/fdb5/remote/FdbServer.h +++ b/src/fdb5/remote/FdbServer.h @@ -14,34 +14,31 @@ #pragma once -#include #include +#include #include "eckit/net/Port.h" #include "eckit/net/TCPServer.h" #include "eckit/net/TCPSocket.h" #include "eckit/runtime/Application.h" -#include "eckit/runtime/Monitorable.h" #include "eckit/runtime/Monitor.h" +#include "eckit/runtime/Monitorable.h" #include "eckit/runtime/ProcessControler.h" -#include "fdb5/config/Config.h" #include "fdb5/LibFdb5.h" +#include "fdb5/config/Config.h" namespace fdb5::remote { //---------------------------------------------------------------------------------------------------------------------- - class FDBForker : public eckit::ProcessControler { public: // methods - FDBForker(eckit::net::TCPSocket& socket, const Config& config); ~FDBForker() override; private: // methods - void run() override; eckit::net::TCPSocket socket_; @@ -52,7 +49,6 @@ class FDBForker : public eckit::ProcessControler { class FdbServerBase { public: - FdbServerBase(); virtual ~FdbServerBase(); @@ -60,7 +56,6 @@ class FdbServerBase { virtual void doRun(); private: - int port_; std::thread reaperThread_; @@ -83,7 +78,6 @@ class FdbServer : public eckit::Application, public FdbServerBase { void run() override; void hookUnique() override; // non-unique - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/remote/Messages.cc b/src/fdb5/remote/Messages.cc index 5e96fdab6..da91c0452 100644 --- a/src/fdb5/remote/Messages.cc +++ b/src/fdb5/remote/Messages.cc @@ -23,50 +23,102 @@ namespace fdb5::remote { std::ostream& operator<<(std::ostream& s, const Message& m) { switch (m) { - case Message::None: s << "None"; break; - case Message::Exit: s << "Exit"; break; - case Message::Startup: s << "Startup"; break; - case Message::Error: s << "Error"; break; - case Message::Stop: s << "Stop"; break; - case Message::Stores: s << "Stores"; break; - case Message::Schema: s << "Schema"; break; + case Message::None: + s << "None"; + break; + case Message::Exit: + s << "Exit"; + break; + case Message::Startup: + s << "Startup"; + break; + case Message::Error: + s << "Error"; + break; + case Message::Stop: + s << "Stop"; + break; + case Message::Stores: + s << "Stores"; + break; + case Message::Schema: + s << "Schema"; + break; - // API calls to forward - case Message::Flush: s << "Flush"; break; - case Message::Archive: s << "Archive"; break; - case Message::Retrieve: s << "Retrieve"; break; - case Message::List: s << "List"; break; - case Message::Dump: s << "Dump"; break; - case Message::Status: s << "Status"; break; - case Message::Wipe: s << "Wipe"; break; - case Message::Purge: s << "Purge"; break; - case Message::Stats: s << "Stats"; break; - case Message::Control: s << "Control"; break; - case Message::Inspect: s << "Inspect"; break; - case Message::Read: s << "Read"; break; - case Message::Move: s << "Move"; break; - case Message::Store: s << "Store"; break; - case Message::Axes: s << "Axes"; break; + // API calls to forward + case Message::Flush: + s << "Flush"; + break; + case Message::Archive: + s << "Archive"; + break; + case Message::Retrieve: + s << "Retrieve"; + break; + case Message::List: + s << "List"; + break; + case Message::Dump: + s << "Dump"; + break; + case Message::Status: + s << "Status"; + break; + case Message::Wipe: + s << "Wipe"; + break; + case Message::Purge: + s << "Purge"; + break; + case Message::Stats: + s << "Stats"; + break; + case Message::Control: + s << "Control"; + break; + case Message::Inspect: + s << "Inspect"; + break; + case Message::Read: + s << "Read"; + break; + case Message::Move: + s << "Move"; + break; + case Message::Store: + s << "Store"; + break; + case Message::Axes: + s << "Axes"; + break; - // Responses - case Message::Received: s << "Received"; break; - case Message::Complete: s << "Complete"; break; + // Responses + case Message::Received: + s << "Received"; + break; + case Message::Complete: + s << "Complete"; + break; - // Data communication - case Message::Blob: s << "Blob"; break; - case Message::MultiBlob: s << "MultiBlob"; break; + // Data communication + case Message::Blob: + s << "Blob"; + break; + case Message::MultiBlob: + s << "MultiBlob"; + break; } - s << "(" << ((int) m) << ")"; + s << "(" << ((int)m) << ")"; return s; } -MessageHeader::MessageHeader(Message message, bool control, uint32_t clientID, uint32_t requestID, uint32_t payloadSize) : - marker(StartMarker), - version(CurrentVersion), - message(message), - clientID_((clientID<<1) + (control ? 1 : 0)), - requestID(requestID), - payloadSize(payloadSize) {} +MessageHeader::MessageHeader(Message message, bool control, uint32_t clientID, uint32_t requestID, uint32_t payloadSize) + : marker(StartMarker) + , version(CurrentVersion) + , message(message) + , clientID_((clientID << 1) + (control ? 1 : 0)) + , requestID(requestID) + , payloadSize(payloadSize) {} //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/remote/Messages.h b/src/fdb5/remote/Messages.h index ce203b39b..a4e8827a4 100644 --- a/src/fdb5/remote/Messages.h +++ b/src/fdb5/remote/Messages.h @@ -18,21 +18,21 @@ #pragma once -#include #include +#include #include "eckit/types/FixedString.h" namespace eckit { - class Stream; +class Stream; } namespace fdb5::remote { //---------------------------------------------------------------------------------------------------------------------- -const static eckit::FixedString<4> StartMarker {"SFDB"}; -const static eckit::FixedString<4> EndMarker {"EFDB"}; +const static eckit::FixedString<4> StartMarker{"SFDB"}; +const static eckit::FixedString<4> EndMarker{"EFDB"}; constexpr uint16_t CurrentVersion = 12; @@ -80,40 +80,27 @@ std::ostream& operator<<(std::ostream& s, const Message& m); class MessageHeader { public: // methods - - MessageHeader() : - version(CurrentVersion), - message(Message::None), - clientID_(0), - requestID(0), - payloadSize(0) {} - + MessageHeader() : version(CurrentVersion), message(Message::None), clientID_(0), requestID(0), payloadSize(0) {} MessageHeader(Message message, bool control, uint32_t clientID, uint32_t requestID, uint32_t payloadSize); - - bool control() const { - return ((clientID_ & 0x00000001) == 1); - } - uint32_t clientID() const { - return (clientID_>>1); - } -public: + bool control() const { return ((clientID_ & 0x00000001) == 1); } + uint32_t clientID() const { return (clientID_ >> 1); } - eckit::FixedString<4> marker; // 4 bytes --> 4 +public: + eckit::FixedString<4> marker; // 4 bytes --> 4 - uint16_t version; // 2 bytes --> 6 + uint16_t version; // 2 bytes --> 6 - Message message; // 2 bytes --> 8 - - uint32_t clientID_; // 4 bytes --> 12 + Message message; // 2 bytes --> 8 - uint32_t requestID; // 4 bytes --> 16 + uint32_t clientID_; // 4 bytes --> 12 - uint32_t payloadSize; // 4 bytes --> 20 + uint32_t requestID; // 4 bytes --> 16 - eckit::FixedString<16> hash; // 16 bytes --> 36 + uint32_t payloadSize; // 4 bytes --> 20 + eckit::FixedString<16> hash; // 16 bytes --> 36 }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/remote/RemoteFieldLocation.cc b/src/fdb5/remote/RemoteFieldLocation.cc index 7615c3243..a169003f4 100644 --- a/src/fdb5/remote/RemoteFieldLocation.cc +++ b/src/fdb5/remote/RemoteFieldLocation.cc @@ -14,9 +14,9 @@ */ #include "fdb5/remote/RemoteFieldLocation.h" -#include "fdb5/remote/client/RemoteStore.h" -#include "fdb5/remote/client/ClientConnectionRouter.h" #include "fdb5/LibFdb5.h" +#include "fdb5/remote/client/ClientConnectionRouter.h" +#include "fdb5/remote/client/RemoteStore.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/URIManager.h" @@ -25,17 +25,17 @@ namespace fdb5 { namespace remote { -::eckit::ClassSpec RemoteFieldLocation::classSpec_ = {&FieldLocation::classSpec(), "RemoteFieldLocation",}; +::eckit::ClassSpec RemoteFieldLocation::classSpec_ = { + &FieldLocation::classSpec(), + "RemoteFieldLocation", +}; ::eckit::Reanimator RemoteFieldLocation::reanimator_; //---------------------------------------------------------------------------------------------------------------------- -RemoteFieldLocation::RemoteFieldLocation(const eckit::net::Endpoint& endpoint, const FieldLocation& remoteLocation) : - FieldLocation( - eckit::URI("fdb", remoteLocation.uri(), endpoint.hostname(), endpoint.port()), - remoteLocation.offset(), - remoteLocation.length(), - remoteLocation.remapKey()) { +RemoteFieldLocation::RemoteFieldLocation(const eckit::net::Endpoint& endpoint, const FieldLocation& remoteLocation) + : FieldLocation(eckit::URI("fdb", remoteLocation.uri(), endpoint.hostname(), endpoint.port()), + remoteLocation.offset(), remoteLocation.length(), remoteLocation.remapKey()) { ASSERT(remoteLocation.uri().scheme() != "fdb"); if (!remoteLocation.uri().scheme().empty()) { @@ -46,54 +46,48 @@ RemoteFieldLocation::RemoteFieldLocation(const eckit::net::Endpoint& endpoint, c } } -RemoteFieldLocation::RemoteFieldLocation(const eckit::net::Endpoint& endpoint, const RemoteFieldLocation& remoteLocation) : - FieldLocation( - eckit::URI("fdb", remoteLocation.uri(), endpoint.hostname(), endpoint.port()), - remoteLocation.offset(), - remoteLocation.length(), - remoteLocation.remapKey()) {} +RemoteFieldLocation::RemoteFieldLocation(const eckit::net::Endpoint& endpoint, + const RemoteFieldLocation& remoteLocation) + : FieldLocation(eckit::URI("fdb", remoteLocation.uri(), endpoint.hostname(), endpoint.port()), + remoteLocation.offset(), remoteLocation.length(), remoteLocation.remapKey()) {} -RemoteFieldLocation::RemoteFieldLocation(const eckit::URI& uri) : - FieldLocation(uri) { +RemoteFieldLocation::RemoteFieldLocation(const eckit::URI& uri) : FieldLocation(uri) { ASSERT(uri.scheme() == "fdb"); } -RemoteFieldLocation::RemoteFieldLocation(const eckit::URI& uri, const eckit::Offset& offset, const eckit::Length& length, const Key& remapKey) : - FieldLocation(uri, offset, length, remapKey) { +RemoteFieldLocation::RemoteFieldLocation(const eckit::URI& uri, const eckit::Offset& offset, + const eckit::Length& length, const Key& remapKey) + : FieldLocation(uri, offset, length, remapKey) { ASSERT(uri.scheme() == "fdb"); } -RemoteFieldLocation::RemoteFieldLocation(eckit::Stream& s) : - FieldLocation(s) { -} - -RemoteFieldLocation::RemoteFieldLocation(const RemoteFieldLocation& rhs) : - FieldLocation(rhs.uri_, rhs.offset_, rhs.length_, rhs.remapKey_) {} +RemoteFieldLocation::RemoteFieldLocation(eckit::Stream& s) : FieldLocation(s) {} +RemoteFieldLocation::RemoteFieldLocation(const RemoteFieldLocation& rhs) + : FieldLocation(rhs.uri_, rhs.offset_, rhs.length_, rhs.remapKey_) {} std::shared_ptr RemoteFieldLocation::make_shared() const { return std::make_shared(std::move(*this)); } eckit::DataHandle* RemoteFieldLocation::dataHandle() const { - + eckit::Log::debug() << "RemoteFieldLocation::dataHandle for location: "; dump(eckit::Log::debug()); eckit::Log::debug() << std::endl; RemoteStore& store = RemoteStore::get(uri_); - + const std::string scheme = uri_.query("internalScheme"); const std::string hostport = uri_.query("internalHost"); eckit::URI remote; if (hostport.empty()) { - remote = eckit::URI(scheme, uri_, "", -1); - } - else { + remote = eckit::URI(scheme, uri_, "", -1); + } else { eckit::net::Endpoint endpoint{hostport}; - remote = eckit::URI(scheme, uri_, endpoint.host(), endpoint.port()); + remote = eckit::URI(scheme, uri_, endpoint.host(), endpoint.port()); remote.query("internalHost", ""); } remote.query("internalScheme", ""); @@ -129,13 +123,12 @@ class FdbURIManager : public eckit::URIManager { eckit::DataHandle* newReadHandle(const eckit::URI& f) override { return f.path().fileHandle(); } - eckit::DataHandle* newReadHandle(const eckit::URI& f, const eckit::OffsetList& ol, const eckit::LengthList& ll) override { + eckit::DataHandle* newReadHandle(const eckit::URI& f, const eckit::OffsetList& ol, + const eckit::LengthList& ll) override { return f.path().partHandle(ol, ll); } - eckit::PathName path(const eckit::URI& u) const override { - return eckit::PathName{u.name()}; - } + eckit::PathName path(const eckit::URI& u) const override { return eckit::PathName{u.name()}; } std::string asString(const eckit::URI& uri) const override { std::string q = uri.query(); @@ -147,6 +140,7 @@ class FdbURIManager : public eckit::URIManager { return uri.name() + q + f; } + public: FdbURIManager(const std::string& name) : eckit::URIManager(name) {} }; diff --git a/src/fdb5/remote/RemoteFieldLocation.h b/src/fdb5/remote/RemoteFieldLocation.h index b1d0b8aab..66974add5 100644 --- a/src/fdb5/remote/RemoteFieldLocation.h +++ b/src/fdb5/remote/RemoteFieldLocation.h @@ -29,11 +29,11 @@ class RemoteStore; class RemoteFieldLocation : public FieldLocation { public: - RemoteFieldLocation(const eckit::net::Endpoint& endpoint, const FieldLocation& remoteLocation); RemoteFieldLocation(const eckit::net::Endpoint& endpoint, const RemoteFieldLocation& remoteLocation); - RemoteFieldLocation(const eckit::URI &uri); - RemoteFieldLocation(const eckit::URI &uri, const eckit::Offset &offset, const eckit::Length &length, const Key& remapKey); + RemoteFieldLocation(const eckit::URI& uri); + RemoteFieldLocation(const eckit::URI& uri, const eckit::Offset& offset, const eckit::Length& length, + const Key& remapKey); RemoteFieldLocation(eckit::Stream&); RemoteFieldLocation(const RemoteFieldLocation&); @@ -43,26 +43,21 @@ class RemoteFieldLocation : public FieldLocation { void visit(FieldLocationVisitor& visitor) const override; public: // For Streamable - - static const eckit::ClassSpec& classSpec() { return classSpec_;} + static const eckit::ClassSpec& classSpec() { return classSpec_; } protected: // For Streamable - void encode(eckit::Stream&) const override; const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } - static eckit::ClassSpec classSpec_; + static eckit::ClassSpec classSpec_; static eckit::Reanimator reanimator_; private: // methods - void print(std::ostream& out) const override; private: // members - }; - //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5::remote diff --git a/src/fdb5/remote/client/Client.cc b/src/fdb5/remote/client/Client.cc index d443e37ea..afc1d5709 100644 --- a/src/fdb5/remote/client/Client.cc +++ b/src/fdb5/remote/client/Client.cc @@ -8,7 +8,6 @@ * does it submit to any jurisdiction. */ - #include "fdb5/LibFdb5.h" #include "fdb5/remote/client/Client.h" @@ -26,15 +25,15 @@ void Client::setClientID() { id_ = ++clientId_; } -Client::Client(const eckit::net::Endpoint& endpoint, const std::string& defaultEndpoint) : - connection_(ClientConnectionRouter::instance().connection(endpoint, defaultEndpoint)) { +Client::Client(const eckit::net::Endpoint& endpoint, const std::string& defaultEndpoint) + : connection_(ClientConnectionRouter::instance().connection(endpoint, defaultEndpoint)) { setClientID(); connection_.add(*this); } -Client::Client(const std::vector>& endpoints) : - connection_(ClientConnectionRouter::instance().connection(endpoints)) { +Client::Client(const std::vector>& endpoints) + : connection_(ClientConnectionRouter::instance().connection(endpoints)) { setClientID(); connection_.add(*this); @@ -44,7 +43,8 @@ Client::~Client() { connection_.remove(id_); } -void Client::controlWriteCheckResponse(Message msg, uint32_t requestID, bool dataListener, const void* payload, uint32_t payloadLength) { +void Client::controlWriteCheckResponse(Message msg, uint32_t requestID, bool dataListener, const void* payload, + uint32_t payloadLength) { ASSERT(requestID); ASSERT(!(!payloadLength ^ !payload)); @@ -60,12 +60,13 @@ void Client::controlWriteCheckResponse(Message msg, uint32_t requestID, bool dat ASSERT(f.get().size() == 0); } -eckit::Buffer Client::controlWriteReadResponse(Message msg, uint32_t requestID, const void* payload, uint32_t payloadLength) { +eckit::Buffer Client::controlWriteReadResponse(Message msg, uint32_t requestID, const void* payload, + uint32_t payloadLength) { ASSERT(requestID); ASSERT(!(!payloadLength ^ !payload)); std::lock_guard lock(blockingRequestMutex_); - + std::vector> data{}; if (payloadLength) { data.push_back(std::make_pair(payload, payloadLength)); diff --git a/src/fdb5/remote/client/Client.h b/src/fdb5/remote/client/Client.h index 689e42607..0f78a996b 100644 --- a/src/fdb5/remote/client/Client.h +++ b/src/fdb5/remote/client/Client.h @@ -24,8 +24,8 @@ namespace fdb5::remote { class RemoteFDBException : public eckit::RemoteException { public: - RemoteFDBException(const std::string& msg, const eckit::net::Endpoint& endpoint): - eckit::RemoteException(msg, endpoint) {} + RemoteFDBException(const std::string& msg, const eckit::net::Endpoint& endpoint) + : eckit::RemoteException(msg, endpoint) {} }; //---------------------------------------------------------------------------------------------------------------------- @@ -44,17 +44,18 @@ class Client : eckit::NonCopyable { uint32_t generateRequestID() { return connection_.generateRequestID(); } // blocking requests - void controlWriteCheckResponse(Message msg, uint32_t requestID, bool dataListener, const void* payload=nullptr, uint32_t payloadLength=0); - eckit::Buffer controlWriteReadResponse (Message msg, uint32_t requestID, const void* payload=nullptr, uint32_t payloadLength=0); + void controlWriteCheckResponse(Message msg, uint32_t requestID, bool dataListener, const void* payload = nullptr, + uint32_t payloadLength = 0); + eckit::Buffer controlWriteReadResponse(Message msg, uint32_t requestID, const void* payload = nullptr, + uint32_t payloadLength = 0); + + void dataWrite(remote::Message msg, uint32_t requestID, std::vector> data = {}); - void dataWrite(remote::Message msg, uint32_t requestID, std::vector> data={}); - // handlers for incoming messages - to be defined in the client class virtual bool handle(Message message, uint32_t requestID) = 0; virtual bool handle(Message message, uint32_t requestID, eckit::Buffer&& payload) = 0; protected: - ClientConnection& connection_; private: @@ -66,4 +67,4 @@ class Client : eckit::NonCopyable { std::mutex blockingRequestMutex_; }; -} \ No newline at end of file +} // namespace fdb5::remote \ No newline at end of file diff --git a/src/fdb5/remote/client/ClientConnection.cc b/src/fdb5/remote/client/ClientConnection.cc index f85707156..22466e45e 100644 --- a/src/fdb5/remote/client/ClientConnection.cc +++ b/src/fdb5/remote/client/ClientConnection.cc @@ -26,12 +26,13 @@ namespace fdb5::remote { class DataWriteRequest { public: + DataWriteRequest() : client_(nullptr), msg_(Message::None), id_(0), data_(eckit::Buffer(0)) {} - DataWriteRequest() : - client_(nullptr), msg_(Message::None), id_(0), data_(eckit::Buffer(0)) {} - - DataWriteRequest(Client* client, Message msg, uint32_t id, eckit::Buffer&& data) : - client_(client), msg_(msg), id_(id), data_(std::move(data)) {} + DataWriteRequest(Client* client, Message msg, uint32_t id, eckit::Buffer&& data) + : client_(client) + , msg_(msg) + , id_(id) + , data_(std::move(data)) {} Client* client_; Message msg_; @@ -39,9 +40,14 @@ class DataWriteRequest { eckit::Buffer data_; }; - -ClientConnection::ClientConnection(const eckit::net::Endpoint& controlEndpoint, const std::string& defaultEndpoint): - controlEndpoint_(controlEndpoint), defaultEndpoint_(defaultEndpoint), id_(1), connected_(false), controlStopping_(false), dataStopping_(false), dataWriteQueue_(nullptr) { +ClientConnection::ClientConnection(const eckit::net::Endpoint& controlEndpoint, const std::string& defaultEndpoint) + : controlEndpoint_(controlEndpoint) + , defaultEndpoint_(defaultEndpoint) + , id_(1) + , connected_(false) + , controlStopping_(false) + , dataStopping_(false) + , dataWriteQueue_(nullptr) { LOG_DEBUG_LIB(LibFdb5) << "ClientConnection::ClientConnection() controlEndpoint: " << controlEndpoint << std::endl; } @@ -84,7 +90,7 @@ ClientConnection::~ClientConnection() { uint32_t ClientConnection::generateRequestID() { std::lock_guard lock(idMutex_); // we do not want to re-use previous request IDs - ASSERT(id_ < UINT32_MAX-2); + ASSERT(id_ < UINT32_MAX - 2); return ++id_; } @@ -118,7 +124,7 @@ bool ClientConnection::connect(bool singleAttempt) { listeningControlThread_ = std::thread([this] { listeningControlThreadLoop(); }); connected_ = true; - } catch(eckit::TooManyRetries& e) { + } catch (eckit::TooManyRetries& e) { if (controlClient_.isConnected()) { controlClient_.close(); } @@ -156,7 +162,7 @@ eckit::LocalConfiguration ClientConnection::availableFunctionality() const { eckit::LocalConfiguration conf; std::vector remoteFieldLocationVersions = {1}; conf.set("RemoteFieldLocation", remoteFieldLocationVersions); - std::vector numberOfConnections = {1,2}; + std::vector numberOfConnections = {1, 2}; conf.set("NumberOfConnections", numberOfConnections); conf.set("PreferSingleConnection", false); return conf; @@ -164,7 +170,9 @@ eckit::LocalConfiguration ClientConnection::availableFunctionality() const { // ----------------------------------------------------------------------------------------------------- -std::future ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, bool dataListener, std::vector> data) { +std::future ClientConnection::controlWrite(Client& client, Message msg, uint32_t requestID, + bool dataListener, + std::vector> data) { std::future f; { std::lock_guard lock(promisesMutex_); @@ -180,7 +188,8 @@ void ClientConnection::dataWrite(DataWriteRequest& r) { Connection::write(r.msg_, false, r.client_->clientId(), r.id_, r.data_.data(), r.data_.size()); } -void ClientConnection::dataWrite(Client& client, remote::Message msg, uint32_t requestID, std::vector> data) { +void ClientConnection::dataWrite(Client& client, remote::Message msg, uint32_t requestID, + std::vector> data) { static size_t maxQueueLength = eckit::Resource("fdbDataWriteQueueLength;$FDB_DATA_WRITE_QUEUE_LENGTH", 320); { @@ -200,14 +209,14 @@ void ClientConnection::dataWrite(Client& client, remote::Message msg, uint32_t r } } uint32_t payloadLength = 0; - for (auto d: data) { + for (auto d : data) { ASSERT(d.first); payloadLength += d.second; } eckit::Buffer buffer{payloadLength}; uint32_t offset = 0; - for (auto d: data) { + for (auto d : data) { buffer.copy(d.first, d.second, offset); offset += d.second; } @@ -215,7 +224,6 @@ void ClientConnection::dataWrite(Client& client, remote::Message msg, uint32_t r dataWriteQueue_->emplace(&client, msg, requestID, std::move(buffer)); } - void ClientConnection::dataWriteThreadLoop() { eckit::Timer timer; @@ -253,7 +261,8 @@ void ClientConnection::writeControlStartupMessage() { // essentially JSON) over the wire for flexibility. s << availableFunctionality().get(); - LOG_DEBUG_LIB(LibFdb5) << "writeControlStartupMessage - Sending session " << sessionID_ << " to control " << controlEndpoint_ << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "writeControlStartupMessage - Sending session " << sessionID_ << " to control " + << controlEndpoint_ << std::endl; Connection::write(Message::Startup, true, 0, 0, payload, s.position()); } @@ -265,7 +274,8 @@ void ClientConnection::writeDataStartupMessage(const eckit::SessionID& serverSes s << sessionID_; s << serverSession; - LOG_DEBUG_LIB(LibFdb5) << "writeDataStartupMessage - Sending session " << sessionID_ << " to data " << dataEndpoint_ << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "writeDataStartupMessage - Sending session " << sessionID_ << " to data " << dataEndpoint_ + << std::endl; Connection::write(Message::Startup, false, 0, 0, payload, s.position()); } @@ -284,27 +294,25 @@ eckit::SessionID ClientConnection::verifyServerStartupResponse() { dataEndpoint_ = dataEndpoint; - LOG_DEBUG_LIB(LibFdb5) << "verifyServerStartupResponse - Received from server " << clientSession << " " << serverSession << " " << dataEndpoint << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "verifyServerStartupResponse - Received from server " << clientSession << " " + << serverSession << " " << dataEndpoint << std::endl; if (dataEndpoint_.hostname() != controlEndpoint_.hostname()) { - eckit::Log::warning() << "Data and control interface hostnames do not match. " - << dataEndpoint_.hostname() << " /= " - << controlEndpoint_.hostname() << std::endl; + eckit::Log::warning() << "Data and control interface hostnames do not match. " << dataEndpoint_.hostname() + << " /= " << controlEndpoint_.hostname() << std::endl; } if (clientSession != sessionID_) { std::stringstream ss; - ss << "Session ID does not match session received from server: " - << sessionID_ << " != " << clientSession; + ss << "Session ID does not match session received from server: " << sessionID_ << " != " << clientSession; throw eckit::BadValue(ss.str(), Here()); } - if (serverFunctionality.has("NumberOfConnections") && serverFunctionality.getInt("NumberOfConnections")==1) { + if (serverFunctionality.has("NumberOfConnections") && serverFunctionality.getInt("NumberOfConnections") == 1) { single_ = true; } if (single_ && !(dataEndpoint_ == controlEndpoint_)) { - eckit::Log::warning() << "Returned control interface does not match. " - << dataEndpoint_ << " /= " - << controlEndpoint_ << std::endl; + eckit::Log::warning() << "Returned control interface does not match. " << dataEndpoint_ + << " /= " << controlEndpoint_ << std::endl; } return serverSession; @@ -320,7 +328,10 @@ void ClientConnection::listeningControlThreadLoop() { eckit::Buffer payload = Connection::readControl(hdr); - LOG_DEBUG_LIB(LibFdb5) << "ClientConnection::listeningControlThreadLoop - got [message=" << hdr.message << ",clientID=" << hdr.clientID() << ",control=" << hdr.control() << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "ClientConnection::listeningControlThreadLoop - got [message=" << hdr.message + << ",clientID=" << hdr.clientID() << ",control=" << hdr.control() + << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" + << std::endl; if (hdr.message == Message::Exit) { controlStopping_ = true; @@ -350,7 +361,9 @@ void ClientConnection::listeningControlThreadLoop() { auto it = clients_.find(hdr.clientID()); if (it == clients_.end()) { std::stringstream ss; - ss << "ERROR: connection=" << controlEndpoint_ << " received [clientID="<< hdr.clientID() << ",requestID="<< hdr.requestID << ",message=" << hdr.message << ",payload=" << hdr.payloadSize << "]" << std::endl; + ss << "ERROR: connection=" << controlEndpoint_ + << " received [clientID=" << hdr.clientID() << ",requestID=" << hdr.requestID + << ",message=" << hdr.message << ",payload=" << hdr.payloadSize << "]" << std::endl; ss << "Unexpected answer for clientID recieved (" << hdr.clientID() << "). ABORTING"; eckit::Log::status() << ss.str() << std::endl; eckit::Log::error() << "Retrieving... " << ss.str() << std::endl; @@ -361,15 +374,16 @@ void ClientConnection::listeningControlThreadLoop() { if (hdr.payloadSize == 0) { handled = client->handle(hdr.message, hdr.requestID); - } - else { + } else { handled = client->handle(hdr.message, hdr.requestID, std::move(payload)); } } if (!handled) { std::stringstream ss; - ss << "ERROR: connection=" << controlEndpoint_ << "Unexpected message recieved [message=" << hdr.message << ",clientID=" << hdr.clientID() << ",requestID=" << hdr.requestID << "]. ABORTING"; + ss << "ERROR: connection=" << controlEndpoint_ + << "Unexpected message recieved [message=" << hdr.message << ",clientID=" << hdr.clientID() + << ",requestID=" << hdr.requestID << "]. ABORTING"; eckit::Log::status() << ss.str() << std::endl; eckit::Log::error() << "Client Retrieving... " << ss.str() << std::endl; throw eckit::SeriousBug(ss.str(), Here()); @@ -378,12 +392,10 @@ void ClientConnection::listeningControlThreadLoop() { } } - // We don't want to let exceptions escape inside a worker thread. + // We don't want to let exceptions escape inside a worker thread. } catch (const std::exception& e) { - ClientConnectionRouter::instance().teardown(std::make_exception_ptr(e)); - } catch (...) { - ClientConnectionRouter::instance().teardown(std::current_exception()); - } + ClientConnectionRouter::instance().teardown(std::make_exception_ptr(e)); + } catch (...) { ClientConnectionRouter::instance().teardown(std::current_exception()); } } void ClientConnection::listeningDataThreadLoop() { @@ -398,7 +410,9 @@ void ClientConnection::listeningDataThreadLoop() { eckit::Buffer payload = Connection::readData(hdr); - LOG_DEBUG_LIB(LibFdb5) << "ClientConnection::listeningDataThreadLoop - got [message=" << hdr.message << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "ClientConnection::listeningDataThreadLoop - got [message=" << hdr.message + << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" + << std::endl; if (hdr.message == Message::Exit) { dataStopping_ = true; @@ -413,7 +427,8 @@ void ClientConnection::listeningDataThreadLoop() { auto it = clients_.find(hdr.clientID()); if (it == clients_.end()) { std::stringstream ss; - ss << "ERROR: Received [clientID="<< hdr.clientID() << ",requestID="<< hdr.requestID << ",message=" << hdr.message << ",payload=" << hdr.payloadSize << "]" << std::endl; + ss << "ERROR: Received [clientID=" << hdr.clientID() << ",requestID=" << hdr.requestID + << ",message=" << hdr.message << ",payload=" << hdr.payloadSize << "]" << std::endl; ss << "Unexpected answer for clientID recieved (" << hdr.clientID() << "). ABORTING"; eckit::Log::status() << ss.str() << std::endl; eckit::Log::error() << "Retrieving... " << ss.str() << std::endl; @@ -426,14 +441,14 @@ void ClientConnection::listeningDataThreadLoop() { ASSERT(!hdr.control()); if (hdr.payloadSize == 0) { handled = client->handle(hdr.message, hdr.requestID); - } - else { + } else { handled = client->handle(hdr.message, hdr.requestID, std::move(payload)); } if (!handled) { std::stringstream ss; - ss << "ERROR: DATA connection=" << controlEndpoint_ << " Unexpected message recieved (" << hdr.message << "). ABORTING"; + ss << "ERROR: DATA connection=" << controlEndpoint_ << " Unexpected message recieved (" + << hdr.message << "). ABORTING"; eckit::Log::status() << ss.str() << std::endl; eckit::Log::error() << "Client Retrieving... " << ss.str() << std::endl; throw eckit::SeriousBug(ss.str(), Here()); @@ -442,12 +457,10 @@ void ClientConnection::listeningDataThreadLoop() { } } - // We don't want to let exceptions escape inside a worker thread. + // We don't want to let exceptions escape inside a worker thread. } catch (const std::exception& e) { - ClientConnectionRouter::instance().teardown(std::make_exception_ptr(e)); - } catch (...) { - ClientConnectionRouter::instance().teardown(std::current_exception()); - } + ClientConnectionRouter::instance().teardown(std::make_exception_ptr(e)); + } catch (...) { ClientConnectionRouter::instance().teardown(std::current_exception()); } } -} // namespace fdb5::remote +} // namespace fdb5::remote diff --git a/src/fdb5/remote/client/ClientConnection.h b/src/fdb5/remote/client/ClientConnection.h index f24cb8112..3af0d8e9a 100644 --- a/src/fdb5/remote/client/ClientConnection.h +++ b/src/fdb5/remote/client/ClientConnection.h @@ -10,8 +10,8 @@ #pragma once -#include #include +#include #include "eckit/config/LocalConfiguration.h" #include "eckit/container/Queue.h" @@ -21,8 +21,8 @@ #include "eckit/net/TCPStream.h" #include "eckit/runtime/SessionID.h" -#include "fdb5/remote/Messages.h" #include "fdb5/remote/Connection.h" +#include "fdb5/remote/Messages.h" namespace fdb5::remote { @@ -35,11 +35,12 @@ class DataWriteRequest; class ClientConnection : protected Connection { public: // methods - virtual ~ClientConnection(); - std::future controlWrite(Client& client, Message msg, uint32_t requestID, bool startDataListener, std::vector> data={}); - void dataWrite(Client& client, Message msg, uint32_t requestID, std::vector> data={}); + std::future controlWrite(Client& client, Message msg, uint32_t requestID, bool startDataListener, + std::vector> data = {}); + void dataWrite(Client& client, Message msg, uint32_t requestID, + std::vector> data = {}); void add(Client& client); bool remove(uint32_t clientID); @@ -52,7 +53,6 @@ class ClientConnection : protected Connection { const std::string& defaultEndpoint() const { return defaultEndpoint_; } private: // methods - friend class ClientConnectionRouter; ClientConnection(const eckit::net::Endpoint& controlEndpoint, const std::string& defaultEndpoint); @@ -77,7 +77,6 @@ class ClientConnection : protected Connection { eckit::net::TCPSocket& dataSocket() override { return dataClient_; } private: // members - eckit::SessionID sessionID_; eckit::net::Endpoint controlEndpoint_; @@ -114,4 +113,4 @@ class ClientConnection : protected Connection { //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5::remote +} // namespace fdb5::remote diff --git a/src/fdb5/remote/client/ClientConnectionRouter.cc b/src/fdb5/remote/client/ClientConnectionRouter.cc index 68d8c5a45..f817b5ba0 100644 --- a/src/fdb5/remote/client/ClientConnectionRouter.cc +++ b/src/fdb5/remote/client/ClientConnectionRouter.cc @@ -1,6 +1,6 @@ #include "fdb5/remote/client/ClientConnectionRouter.h" -namespace{ +namespace { class ConnectionError : public eckit::Exception { public: @@ -23,12 +23,13 @@ ConnectionError::ConnectionError(const eckit::net::Endpoint& endpoint) { reason(s.str()); eckit::Log::status() << what() << std::endl; } -} +} // namespace namespace fdb5::remote { //---------------------------------------------------------------------------------------------------------------------- -ClientConnection& ClientConnectionRouter::connection(const eckit::net::Endpoint& endpoint, const std::string& defaultEndpoint) { +ClientConnection& ClientConnectionRouter::connection(const eckit::net::Endpoint& endpoint, + const std::string& defaultEndpoint) { std::lock_guard lock(connectionMutex_); @@ -47,12 +48,13 @@ ClientConnection& ClientConnectionRouter::connection(const eckit::net::Endpoint& } } -ClientConnection& ClientConnectionRouter::connection(const std::vector>& endpoints) { +ClientConnection& +ClientConnectionRouter::connection(const std::vector>& endpoints) { std::vector> fullEndpoints{endpoints}; std::lock_guard lock(connectionMutex_); - while (fullEndpoints.size()>0) { + while (fullEndpoints.size() > 0) { // select a random endpoint size_t idx = std::rand() % fullEndpoints.size(); @@ -62,9 +64,9 @@ ClientConnection& ClientConnectionRouter::connection(const std::vectorsecond); - } - else { // not yet there, trying to connect - std::unique_ptr clientConnection = std::unique_ptr(new ClientConnection{endpoint, fullEndpoints.at(idx).second}); + } else { // not yet there, trying to connect + std::unique_ptr clientConnection = + std::unique_ptr(new ClientConnection{endpoint, fullEndpoints.at(idx).second}); if (clientConnection->connect(true)) { auto it = (connections_.emplace(endpoint, std::move(clientConnection))).first; return *(it->second); @@ -72,7 +74,7 @@ ClientConnection& ClientConnectionRouter::connection(const std::vector lock(connectionMutex_); - for (const auto& [endp,conn] : connections_) { + for (const auto& [endp, conn] : connections_) { if (conn) { eckit::Log::warning() << "closing connection " << endp << std::endl; conn->teardown(); @@ -117,4 +115,4 @@ void ClientConnectionRouter::teardown(std::exception_ptr e) { } } -} // namespace fdb5::remote +} // namespace fdb5::remote diff --git a/src/fdb5/remote/client/ClientConnectionRouter.h b/src/fdb5/remote/client/ClientConnectionRouter.h index 026511506..a719a62e1 100644 --- a/src/fdb5/remote/client/ClientConnectionRouter.h +++ b/src/fdb5/remote/client/ClientConnectionRouter.h @@ -21,7 +21,6 @@ namespace fdb5::remote { class ClientConnectionRouter : eckit::NonCopyable { public: - static ClientConnectionRouter& instance(); ClientConnection& connection(const eckit::net::Endpoint& endpoint, const std::string& defaultEndpoint); @@ -32,7 +31,6 @@ class ClientConnectionRouter : eckit::NonCopyable { void deregister(ClientConnection& connection); private: - ClientConnectionRouter() {} ///< private constructor only used by singleton std::mutex connectionMutex_; @@ -40,4 +38,4 @@ class ClientConnectionRouter : eckit::NonCopyable { std::unordered_map> connections_; }; -} \ No newline at end of file +} // namespace fdb5::remote \ No newline at end of file diff --git a/src/fdb5/remote/client/RemoteCatalogue.cc b/src/fdb5/remote/client/RemoteCatalogue.cc index e0a93d854..928419637 100644 --- a/src/fdb5/remote/client/RemoteCatalogue.cc +++ b/src/fdb5/remote/client/RemoteCatalogue.cc @@ -20,22 +20,29 @@ using namespace eckit; namespace fdb5::remote { -RemoteCatalogue::RemoteCatalogue(const Key& key, const Config& config): - CatalogueImpl(key, ControlIdentifiers(), config), // xxx what are control identifiers? Setting empty here... - Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port")), ""), - config_(config), schema_(nullptr), numLocations_(0) { +RemoteCatalogue::RemoteCatalogue(const Key& key, const Config& config) + : CatalogueImpl(key, ControlIdentifiers(), config) + , // xxx what are control identifiers? Setting empty here... + Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port")), "") + , config_(config) + , schema_(nullptr) + , numLocations_(0) { loadSchema(); } -// Catalogue(URI, Config) is only used by the Visitors to traverse the catalogue. In the remote, we use the RemoteFDB for catalogue traversal -// this ctor is here only to comply with the factory -RemoteCatalogue::RemoteCatalogue(const eckit::URI& uri, const Config& config): - Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port")), ""), config_(config), schema_(nullptr), numLocations_(0) { +// Catalogue(URI, Config) is only used by the Visitors to traverse the catalogue. In the remote, we use the RemoteFDB +// for catalogue traversal this ctor is here only to comply with the factory +RemoteCatalogue::RemoteCatalogue(const eckit::URI& uri, const Config& config) + : Client(eckit::net::Endpoint(config.getString("host"), config.getInt("port")), "") + , config_(config) + , schema_(nullptr) + , numLocations_(0) { NOTIMP; } -void RemoteCatalogue::archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) { +void RemoteCatalogue::archive(const Key& idxKey, const Key& datumKey, + std::shared_ptr fieldLocation) { ASSERT(!datumKey.empty()); ASSERT(fieldLocation); @@ -68,7 +75,7 @@ bool RemoteCatalogue::selectIndex(const Key& idxKey) { return true; // xxx whats the return used for? TOC always returns true } -const Index& RemoteCatalogue::currentIndex(){ +const Index& RemoteCatalogue::currentIndex() { NOTIMP; } const Key RemoteCatalogue::currentIndexKey() { @@ -104,11 +111,17 @@ void RemoteCatalogue::flush(size_t archivedFields) { } } -void RemoteCatalogue::clean() {NOTIMP;} +void RemoteCatalogue::clean() { + NOTIMP; +} -void RemoteCatalogue::close() {NOTIMP;} +void RemoteCatalogue::close() { + NOTIMP; +} -bool RemoteCatalogue::exists() const {NOTIMP;} +bool RemoteCatalogue::exists() const { + NOTIMP; +} void RemoteCatalogue::checkUID() const {} @@ -128,7 +141,8 @@ void RemoteCatalogue::loadSchema() { eckit::MemoryStream keyStream(keyBuffer); keyStream << dbKey_; - eckit::Buffer buf = controlWriteReadResponse(Message::Schema, generateRequestID(), keyBuffer, keyStream.position()); + eckit::Buffer buf = + controlWriteReadResponse(Message::Schema, generateRequestID(), keyBuffer, keyStream.position()); eckit::MemoryStream s(buf); schema_.reset(eckit::Reanimator::reanimate(s)); @@ -136,33 +150,66 @@ void RemoteCatalogue::loadSchema() { } bool RemoteCatalogue::handle(Message message, uint32_t requestID) { - Log::warning() << *this << " - Received [message=" << ((uint) message) << ",requestID=" << requestID << "]" << std::endl; + Log::warning() << *this << " - Received [message=" << ((uint)message) << ",requestID=" << requestID << "]" + << std::endl; return false; } bool RemoteCatalogue::handle(Message message, uint32_t requestID, eckit::Buffer&& payload) { - LOG_DEBUG_LIB(LibFdb5) << *this << " - Received [message=" << ((uint) message) << ",requestID=" << requestID << ",payloadSize=" << payload.size() << "]" << std::endl; + LOG_DEBUG_LIB(LibFdb5) << *this << " - Received [message=" << ((uint)message) << ",requestID=" << requestID + << ",payloadSize=" << payload.size() << "]" << std::endl; return false; } -void RemoteCatalogue::overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) {NOTIMP;} -void RemoteCatalogue::index(const Key& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) {NOTIMP;} -void RemoteCatalogue::reconsolidate(){NOTIMP;} -std::vector RemoteCatalogue::metadataPaths() const {NOTIMP;} -void RemoteCatalogue::visitEntries(EntryVisitor& visitor, bool sorted) {NOTIMP;} -void RemoteCatalogue::dump(std::ostream& out, bool simple, const eckit::Configuration& conf) const {NOTIMP;} -StatsReportVisitor* RemoteCatalogue::statsReportVisitor() const {NOTIMP;} -PurgeVisitor* RemoteCatalogue::purgeVisitor(const Store& store) const {NOTIMP;} -WipeVisitor* RemoteCatalogue::wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, bool doit, bool porcelain, bool unsafeWipeAll) const {NOTIMP;} -MoveVisitor* RemoteCatalogue::moveVisitor(const Store& store, const metkit::mars::MarsRequest& request, const eckit::URI& dest, eckit::Queue& queue) const {NOTIMP;} -void RemoteCatalogue::control(const ControlAction& action, const ControlIdentifiers& identifiers) const {NOTIMP;} -std::vector RemoteCatalogue::indexes(bool sorted) const {NOTIMP;} -void RemoteCatalogue::maskIndexEntry(const Index& index) const {NOTIMP;} -void RemoteCatalogue::allMasked(std::set>& metadata, std::set& data) const {NOTIMP;} -void RemoteCatalogue::print( std::ostream &out ) const { +void RemoteCatalogue::overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, + bool unmount) { + NOTIMP; +} +void RemoteCatalogue::index(const Key& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) { + NOTIMP; +} +void RemoteCatalogue::reconsolidate() { + NOTIMP; +} +std::vector RemoteCatalogue::metadataPaths() const { + NOTIMP; +} +void RemoteCatalogue::visitEntries(EntryVisitor& visitor, bool sorted) { + NOTIMP; +} +void RemoteCatalogue::dump(std::ostream& out, bool simple, const eckit::Configuration& conf) const { + NOTIMP; +} +StatsReportVisitor* RemoteCatalogue::statsReportVisitor() const { + NOTIMP; +} +PurgeVisitor* RemoteCatalogue::purgeVisitor(const Store& store) const { + NOTIMP; +} +WipeVisitor* RemoteCatalogue::wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, + std::ostream& out, bool doit, bool porcelain, bool unsafeWipeAll) const { + NOTIMP; +} +MoveVisitor* RemoteCatalogue::moveVisitor(const Store& store, const metkit::mars::MarsRequest& request, + const eckit::URI& dest, eckit::Queue& queue) const { + NOTIMP; +} +void RemoteCatalogue::control(const ControlAction& action, const ControlIdentifiers& identifiers) const { + NOTIMP; +} +std::vector RemoteCatalogue::indexes(bool sorted) const { + NOTIMP; +} +void RemoteCatalogue::maskIndexEntry(const Index& index) const { + NOTIMP; +} +void RemoteCatalogue::allMasked(std::set>& metadata, + std::set& data) const { + NOTIMP; +} +void RemoteCatalogue::print(std::ostream& out) const { out << "RemoteCatalogue(endpoint=" << controlEndpoint() << ",clientID=" << clientId() << ")"; } - std::string RemoteCatalogue::type() const { return "remote"; } diff --git a/src/fdb5/remote/client/RemoteCatalogue.h b/src/fdb5/remote/client/RemoteCatalogue.h index 1d10b9716..0ac75c71e 100644 --- a/src/fdb5/remote/client/RemoteCatalogue.h +++ b/src/fdb5/remote/client/RemoteCatalogue.h @@ -15,7 +15,6 @@ namespace fdb5::remote { class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public CatalogueImpl, public Client { public: - RemoteCatalogue(const Key& key, const Config& config); RemoteCatalogue(const eckit::URI& uri, const Config& config); @@ -28,7 +27,7 @@ class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public C void index(const Key& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) override; void reconsolidate() override; - //From CatalogueReader + // From CatalogueReader DbStats stats() const override { return DbStats(); } bool retrieve(const Key& key, Field& field) const override { return false; } @@ -40,16 +39,19 @@ class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public C std::vector metadataPaths() const override; void visitEntries(EntryVisitor& visitor, bool sorted = false) override; - void dump(std::ostream& out, bool simple=false, const eckit::Configuration& conf = eckit::LocalConfiguration()) const override; + void dump(std::ostream& out, bool simple = false, + const eckit::Configuration& conf = eckit::LocalConfiguration()) const override; StatsReportVisitor* statsReportVisitor() const override; PurgeVisitor* purgeVisitor(const Store& store) const override; - WipeVisitor* wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, bool doit, bool porcelain, bool unsafeWipeAll) const override; - MoveVisitor* moveVisitor(const Store& store, const metkit::mars::MarsRequest& request, const eckit::URI& dest, eckit::Queue& queue) const override; + WipeVisitor* wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, bool doit, + bool porcelain, bool unsafeWipeAll) const override; + MoveVisitor* moveVisitor(const Store& store, const metkit::mars::MarsRequest& request, const eckit::URI& dest, + eckit::Queue& queue) const override; void control(const ControlAction& action, const ControlIdentifiers& identifiers) const override; - std::vector indexes(bool sorted=false) const override; + std::vector indexes(bool sorted = false) const override; void maskIndexEntry(const Index& index) const override; void allMasked(std::set>& metadata, std::set& data) const override; - void print( std::ostream &out ) const override; + void print(std::ostream& out) const override; std::string type() const override; bool open() override; void flush(size_t archivedFields) override; @@ -60,7 +62,6 @@ class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public C eckit::URI uri() const override; protected: - void loadSchema() override; private: @@ -70,12 +71,10 @@ class RemoteCatalogue : public CatalogueReader, public CatalogueWriter, public C bool handle(Message message, uint32_t requestID, eckit::Buffer&& payload) override; protected: - Config config_; ControlIdentifiers controlIdentifiers_; private: - Key currentIndexKey_; std::unique_ptr schema_; diff --git a/src/fdb5/remote/client/RemoteStore.cc b/src/fdb5/remote/client/RemoteStore.cc index d1951f77b..35ab0f61d 100644 --- a/src/fdb5/remote/client/RemoteStore.cc +++ b/src/fdb5/remote/client/RemoteStore.cc @@ -20,11 +20,11 @@ #include "eckit/serialisation/MemoryStream.h" #include "fdb5/LibFdb5.h" -#include "fdb5/rules/Rule.h" #include "fdb5/database/FieldLocation.h" -#include "fdb5/remote/client/RemoteStore.h" -#include "fdb5/remote/RemoteFieldLocation.h" #include "fdb5/io/FDBFileHandle.h" +#include "fdb5/remote/RemoteFieldLocation.h" +#include "fdb5/remote/client/RemoteStore.h" +#include "fdb5/rules/Rule.h" #include @@ -48,33 +48,25 @@ namespace fdb5::remote { /// /// --> Retrieve is a _streaming_ service. - class FDBRemoteDataHandle : public DataHandle { public: // methods - - FDBRemoteDataHandle(uint32_t requestID, Length estimate, - std::shared_ptr queue, - const net::Endpoint& remoteEndpoint) : - requestID_(requestID), - estimate_(estimate), - queue_(queue), - remoteEndpoint_(remoteEndpoint), - pos_(0), - overallPosition_(0), - currentBuffer_(0), - complete_(false) {} + FDBRemoteDataHandle(uint32_t requestID, Length estimate, std::shared_ptr queue, + const net::Endpoint& remoteEndpoint) + : requestID_(requestID) + , estimate_(estimate) + , queue_(queue) + , remoteEndpoint_(remoteEndpoint) + , pos_(0) + , overallPosition_(0) + , currentBuffer_(0) + , complete_(false) {} virtual bool canSeek() const override { return false; } private: // methods + void print(std::ostream& s) const override { s << "FDBRemoteDataHandle(id=" << requestID_ << ")"; } - void print(std::ostream& s) const override { - s << "FDBRemoteDataHandle(id=" << requestID_ << ")"; - } - - Length openForRead() override { - return estimate(); - } + Length openForRead() override { return estimate(); } void openForWrite(const Length&) override { NOTIMP; } void openForAppend(const Length&) override { NOTIMP; } long write(const void*, long) override { NOTIMP; } @@ -82,16 +74,17 @@ class FDBRemoteDataHandle : public DataHandle { long read(void* pos, long sz) override { - if (complete_) return 0; + if (complete_) + return 0; long total = 0; long n; - char* p = static_cast(pos); + char* p = static_cast(pos); if (currentBuffer_.size() != 0) { n = bufferRead(pos, sz); sz -= n; total += n; - p+=n; + p += n; } while (sz > 0 && !complete_) { @@ -124,8 +117,7 @@ class FDBRemoteDataHandle : public DataHandle { n = bufferRead(p, sz); sz -= n; total += n; - p+=n; - + p += n; } return total; } @@ -135,7 +127,6 @@ class FDBRemoteDataHandle : public DataHandle { long bufferRead(void* pos, long sz) { - ASSERT(currentBuffer_.size() != 0); ASSERT(pos_ < currentBuffer_.size()); @@ -157,16 +148,11 @@ class FDBRemoteDataHandle : public DataHandle { return read; } - Length estimate() override { - return estimate_; - } + Length estimate() override { return estimate_; } - Offset position() override { - return overallPosition_; - } + Offset position() override { return overallPosition_; } private: // members - uint32_t requestID_; Length estimate_; std::shared_ptr queue_; @@ -186,7 +172,7 @@ std::vector> storeEndpoints(const C ASSERT(stores.size() == fieldLocationEndpoints.size()); std::vector> out; - for (size_t i=0; i> storeEndpoints(const C //---------------------------------------------------------------------------------------------------------------------- -RemoteStore::RemoteStore(const Key& dbKey, const Config& config) : - Client(storeEndpoints(config)), - dbKey_(dbKey), config_(config) - {} +RemoteStore::RemoteStore(const Key& dbKey, const Config& config) + : Client(storeEndpoints(config)) + , dbKey_(dbKey) + , config_(config) {} // this is used only in retrieval, with an URI already referring to an accessible Store -RemoteStore::RemoteStore(const eckit::URI& uri, const Config& config) : - Client(eckit::net::Endpoint(uri.hostport()), uri.hostport()), - dbKey_(Key()), config_(config) - { +RemoteStore::RemoteStore(const eckit::URI& uri, const Config& config) + : Client(eckit::net::Endpoint(uri.hostport()), uri.hostport()) + , dbKey_(Key()) + , config_(config) { // no need to set the local_ flag on the read path ASSERT(uri.scheme() == "fdb"); } @@ -230,14 +216,16 @@ eckit::DataHandle* RemoteStore::retrieve(Field& field) const { return field.dataHandle(); } -void RemoteStore::archive(const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive) { +void RemoteStore::archive( + const Key& key, const void* data, eckit::Length length, + std::function fieldLocation)> catalogue_archive) { ASSERT(!key.empty()); ASSERT(data); ASSERT(length != 0); uint32_t id = generateRequestID(); - { // send the archival request + { // send the archival request std::lock_guard lock(locations_.mutex()); if (locations_.archived() == 0) { // if this is the first archival request, notify the server controlWriteCheckResponse(Message::Store, id, true); @@ -290,8 +278,7 @@ size_t RemoteStore::flush() { return locations; } -void RemoteStore::close() { -} +void RemoteStore::close() {} void RemoteStore::remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const { NOTIMP; @@ -301,7 +288,7 @@ void RemoteStore::remove(const Key& key) const { NOTIMP; } -void RemoteStore::print(std::ostream &out) const { +void RemoteStore::print(std::ostream& out) const { out << "RemoteStore(host=" << controlEndpoint() << ")"; } @@ -320,7 +307,7 @@ bool RemoteStore::handle(Message message, uint32_t requestID) { } else { std::lock_guard lock(retrieveMessageMutex_); auto id = retrieveMessageQueues_.find(requestID); - ASSERT (id != retrieveMessageQueues_.end()); + ASSERT(id != retrieveMessageQueues_.end()); id->second->emplace(std::make_pair(message, Buffer(0))); @@ -338,7 +325,6 @@ bool RemoteStore::handle(Message message, uint32_t requestID) { // goes out of scope in the worker thread). messageQueues_.erase(it); return true; - } return false; } @@ -350,13 +336,15 @@ bool RemoteStore::handle(Message message, uint32_t requestID, eckit::Buffer&& pa switch (message) { - case Message::Store: { // received a Field location from the remote store, can forward to the archiver for the indexing + case Message::Store: { // received a Field location from the remote store, can forward to the archiver for the + // indexing MemoryStream s(payload); std::unique_ptr location(eckit::Reanimator::reanimate(s)); if (defaultEndpoint().empty()) { return locations_.location(requestID, std::move(location)); } else { - std::unique_ptr remoteLocation = std::unique_ptr(new RemoteFieldLocation(eckit::net::Endpoint{defaultEndpoint()}, *location)); + std::unique_ptr remoteLocation = std::unique_ptr( + new RemoteFieldLocation(eckit::net::Endpoint{defaultEndpoint()}, *location)); return locations_.location(requestID, std::move(remoteLocation)); } } @@ -367,7 +355,7 @@ bool RemoteStore::handle(Message message, uint32_t requestID, eckit::Buffer&& pa } else { std::lock_guard lock(retrieveMessageMutex_); auto id = retrieveMessageQueues_.find(requestID); - ASSERT (id != retrieveMessageQueues_.end()); + ASSERT(id != retrieveMessageQueues_.end()); id->second->emplace(std::make_pair(message, std::move(payload))); } return true; @@ -384,7 +372,6 @@ bool RemoteStore::handle(Message message, uint32_t requestID, eckit::Buffer&& pa // Remove entry (shared_ptr --> message queue will be destroyed when it // goes out of scope in the worker thread). messageQueues_.erase(it); - } return true; } @@ -437,18 +424,18 @@ RemoteStore& RemoteStore::get(const eckit::URI& uri) { return *(readStores_[endpoint] = std::unique_ptr(new RemoteStore(uri, Config()))); } - bool RemoteStore::uriBelongs(const eckit::URI&) const { - NOTIMP; - } - bool RemoteStore::uriExists(const eckit::URI&) const { - NOTIMP; - } - std::vector RemoteStore::collocatedDataURIs() const { - NOTIMP; - } - std::set RemoteStore::asCollocatedDataURIs(const std::vector&) const { - NOTIMP; - } +bool RemoteStore::uriBelongs(const eckit::URI&) const { + NOTIMP; +} +bool RemoteStore::uriExists(const eckit::URI&) const { + NOTIMP; +} +std::vector RemoteStore::collocatedDataURIs() const { + NOTIMP; +} +std::set RemoteStore::asCollocatedDataURIs(const std::vector&) const { + NOTIMP; +} static StoreBuilder builder("remote"); diff --git a/src/fdb5/remote/client/RemoteStore.h b/src/fdb5/remote/client/RemoteStore.h index 3446d318e..52436aec2 100644 --- a/src/fdb5/remote/client/RemoteStore.h +++ b/src/fdb5/remote/client/RemoteStore.h @@ -27,17 +27,16 @@ class Locations { public: Locations() : fieldsArchived_(0), locationsReceived_(0) {} - std::mutex& mutex() { - return locationMutex_; - } + std::mutex& mutex() { return locationMutex_; } size_t archived() { return fieldsArchived_; } - void archive(uint32_t requestID, std::function fieldLocation)> catalogue_archive) { + void archive(uint32_t requestID, + std::function fieldLocation)> catalogue_archive) { std::lock_guard lock(locationMutex_); locations_[requestID] = catalogue_archive; fieldsArchived_++; - ASSERT(fieldsArchived_-locationsReceived_ == locations_.size()); + ASSERT(fieldsArchived_ - locationsReceived_ == locations_.size()); } bool location(uint32_t requestID, std::unique_ptr location) { @@ -52,7 +51,7 @@ class Locations { locations_.erase(it); locationsReceived_++; - ASSERT(fieldsArchived_-locationsReceived_ == locations_.size()); + ASSERT(fieldsArchived_ - locationsReceived_ == locations_.size()); if (archivalCompleted_.valid() && (fieldsArchived_ == locationsReceived_)) { promiseArchivalCompleted_.set_value(locationsReceived_); } @@ -62,7 +61,7 @@ class Locations { bool complete() { std::lock_guard lock(locationMutex_); - ASSERT(fieldsArchived_-locationsReceived_ == locations_.size()); + ASSERT(fieldsArchived_ - locationsReceived_ == locations_.size()); if (fieldsArchived_ == locationsReceived_) return true; @@ -78,20 +77,18 @@ class Locations { void reset() { std::lock_guard lock(locationMutex_); - ASSERT(fieldsArchived_-locationsReceived_ == locations_.size()); + ASSERT(fieldsArchived_ - locationsReceived_ == locations_.size()); fieldsArchived_ = locations_.size(); locationsReceived_ = 0; } private: - std::mutex locationMutex_; std::map fieldLocation)>> locations_; size_t fieldsArchived_; size_t locationsReceived_; std::promise promiseArchivalCompleted_; std::future archivalCompleted_; - }; //---------------------------------------------------------------------------------------------------------------------- @@ -100,12 +97,10 @@ class Locations { class RemoteStore : public Store, public Client { public: // types - - using StoredMessage = std::pair; + using StoredMessage = std::pair; using MessageQueue = eckit::Queue; public: // methods - RemoteStore(const Key& key, const Config& config); RemoteStore(const eckit::URI& uri, const Config& config); @@ -119,43 +114,44 @@ class RemoteStore : public Store, public Client { size_t flush() override; void close() override; - void checkUID() const override { } + void checkUID() const override {} eckit::DataHandle* dataHandle(const FieldLocation& fieldLocation); eckit::DataHandle* dataHandle(const FieldLocation& fieldLocation, const Key& remapKey); bool canMoveTo(const Key& key, const Config& config, const eckit::URI& dest) const override { return false; } - void moveTo(const Key& key, const Config& config, const eckit::URI& dest, eckit::Queue& queue) const override { NOTIMP; } + void moveTo(const Key& key, const Config& config, const eckit::URI& dest, + eckit::Queue& queue) const override { + NOTIMP; + } void remove(const Key& key) const override; bool uriBelongs(const eckit::URI&) const override; bool uriExists(const eckit::URI&) const override; std::vector collocatedDataURIs() const override; std::set asCollocatedDataURIs(const std::vector&) const override; - const Config& config() { return config_; } - + const Config& config() { return config_; } protected: // methods - std::string type() const override { return "remote"; } bool exists() const override; eckit::DataHandle* retrieve(Field& field) const override; - void archive(const Key& key, const void *data, eckit::Length length, std::function fieldLocation)> catalogue_archive) override; + void + archive(const Key& key, const void* data, eckit::Length length, + std::function fieldLocation)> catalogue_archive) override; void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override; - void print( std::ostream &out ) const override; + void print(std::ostream& out) const override; private: // methods - // handlers for incoming messages - to be defined in the client class bool handle(Message message, uint32_t requestID) override; bool handle(Message message, uint32_t requestID, eckit::Buffer&& payload) override; private: // members - Key dbKey_; const Config& config_; diff --git a/src/fdb5/remote/fdb-monitor.cc b/src/fdb5/remote/fdb-monitor.cc index 8feb655f8..80a6fb2b0 100644 --- a/src/fdb5/remote/fdb-monitor.cc +++ b/src/fdb5/remote/fdb-monitor.cc @@ -16,25 +16,20 @@ #include "eckit/cmd/CmdApplication.h" #include "eckit/runtime/Application.h" - //---------------------------------------------------------------------------------------------------------------------- class FdbMonitor : public eckit::Application, public eckit::CmdApplication { public: - FdbMonitor(int argc, char** argv) : - Application(argc, argv, "FDB_HOME") {} + FdbMonitor(int argc, char** argv) : Application(argc, argv, "FDB_HOME") {} - virtual void run() { - eckit::CmdApplication::execute(); - } + virtual void run() { eckit::CmdApplication::execute(); } }; - //---------------------------------------------------------------------------------------------------------------------- -int main(int argc,char **argv) { - FdbMonitor app(argc,argv); +int main(int argc, char** argv) { + FdbMonitor app(argc, argv); app.start(); return 0; } diff --git a/src/fdb5/remote/server/AvailablePortList.cc b/src/fdb5/remote/server/AvailablePortList.cc index 112c66f55..0fdce919e 100644 --- a/src/fdb5/remote/server/AvailablePortList.cc +++ b/src/fdb5/remote/server/AvailablePortList.cc @@ -17,8 +17,8 @@ #include #include -#include #include +#include #include "eckit/exception/Exceptions.h" #include "eckit/runtime/ProcessControler.h" @@ -34,7 +34,6 @@ namespace remote { //---------------------------------------------------------------------------------------------------------------------- - /// Read a list of ports to skip from /etc/services std::set readServices() { @@ -42,7 +41,8 @@ std::set readServices() { std::set portsToSkip; PathName servicesFile = "/etc/services"; - if (!servicesFile.exists()) return portsToSkip; + if (!servicesFile.exists()) + return portsToSkip; std::ifstream in(servicesFile.localPath()); @@ -55,7 +55,7 @@ std::set readServices() { eckit::Tokenizer parse("\t /"); - eckit::Translator toInt; + eckit::Translator toInt; char line[1024]; while (in.getline(line, sizeof(line))) { @@ -65,7 +65,7 @@ std::set readServices() { size_t i = 0; while (i < s.size()) { - if(s[i][0] == '#') { + if (s[i][0] == '#') { s.erase(s.begin() + i, s.end()); } else if (s[i].length() == 0) { s.erase(s.begin() + i); @@ -80,31 +80,30 @@ std::set readServices() { if (s.size() >= 3) { - const std::string& sname = s[0]; - const std::string& port = s[1]; - const std::string& protocol = s[2]; + const std::string& sname = s[0]; + const std::string& port = s[1]; + const std::string& protocol = s[2]; - LOG_DEBUG_LIB(LibFdb5) << "Skipping port " << port << " service " << sname << " protocol " << protocol << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "Skipping port " << port << " service " << sname << " protocol " << protocol + << std::endl; - int p = toInt(port); - portsToSkip.insert(p); - } - else { - eckit::Log::warning() << "Unrecognized format in " << servicesFile << " line: [" << line << "]" << std::endl; + int p = toInt(port); + portsToSkip.insert(p); + } else { + eckit::Log::warning() << "Unrecognized format in " << servicesFile << " line: [" << line << "]" + << std::endl; } } return portsToSkip; } - - //---------------------------------------------------------------------------------------------------------------------- -AvailablePortList::AvailablePortList(int startPort, size_t count) : - shared_("~fdb/etc/fdb/serverPortLock", "/fdb-server-data-ports", count), - startPort_(startPort), - count_(count) {} +AvailablePortList::AvailablePortList(int startPort, size_t count) + : shared_("~fdb/etc/fdb/serverPortLock", "/fdb-server-data-ports", count) + , startPort_(startPort) + , count_(count) {} void AvailablePortList::initialise() { @@ -119,7 +118,8 @@ void AvailablePortList::initialise() { break; } } - if (initialised) return; + if (initialised) + return; // Get a list of everything that needs to be skipped @@ -132,9 +132,10 @@ void AvailablePortList::initialise() { while (foundCount < count_) { bool found = true; - if (portsToSkip.find(port) != portsToSkip.end()) found = false; + if (portsToSkip.find(port) != portsToSkip.end()) + found = false; -// if (found && getservbyport(port, 0) != 0) found = false; + // if (found && getservbyport(port, 0) != 0) found = false; if (found) { shared_[foundCount].port = port; @@ -149,7 +150,6 @@ void AvailablePortList::initialise() { shared_.sync(); } - int AvailablePortList::acquire() { std::lock_guard lock(shared_); @@ -167,8 +167,7 @@ int AvailablePortList::acquire() { eckit::Log::info() << "Acquiring port: " << it->port << std::endl; return it->port; } else { - eckit::Log::info() << "Port " << it->port - << " already belongs to a service." + eckit::Log::info() << "Port " << it->port << " already belongs to a service." << " Marking as in use, and trying another." << std::endl; } } diff --git a/src/fdb5/remote/server/AvailablePortList.h b/src/fdb5/remote/server/AvailablePortList.h index 64aa805af..d5a00ae0b 100644 --- a/src/fdb5/remote/server/AvailablePortList.h +++ b/src/fdb5/remote/server/AvailablePortList.h @@ -19,8 +19,8 @@ #ifndef fdb5_remote_AvailablePortList_H #define fdb5_remote_AvailablePortList_H -#include #include +#include #include #include "eckit/container/SharedMemArray.h" @@ -33,7 +33,6 @@ namespace remote { class AvailablePortList { private: // types - // TODO: Cleanup with reaper struct Entry { @@ -43,7 +42,6 @@ class AvailablePortList { }; public: // methods - /// Create and populate the port list if it does not exist AvailablePortList(int startPort, size_t count); @@ -55,12 +53,11 @@ class AvailablePortList { /// Clean up any processes that have been been dead for deadTime seconds. /// Any newly dead processes should be marked with the current time. - void reap(int deadTime=60); + void reap(int deadTime = 60); void initialise(); private: // members - eckit::SharedMemArray shared_; int startPort_; diff --git a/src/fdb5/remote/server/CatalogueHandler.cc b/src/fdb5/remote/server/CatalogueHandler.cc index 22cccf76b..29b814456 100644 --- a/src/fdb5/remote/server/CatalogueHandler.cc +++ b/src/fdb5/remote/server/CatalogueHandler.cc @@ -29,8 +29,10 @@ namespace fdb5::remote { // // *************************************************************************************** -CatalogueHandler::CatalogueHandler(eckit::net::TCPSocket& socket, const Config& config): - ServerConnection(socket, config), fdbControlConnection_(false), fdbDataConnection_(false) {} +CatalogueHandler::CatalogueHandler(eckit::net::TCPSocket& socket, const Config& config) + : ServerConnection(socket, config) + , fdbControlConnection_(false) + , fdbDataConnection_(false) {} CatalogueHandler::~CatalogueHandler() {} @@ -39,18 +41,18 @@ Handled CatalogueHandler::handleControl(Message message, uint32_t clientID, uint try { switch (message) { case Message::Schema: // request top-level schema - { - std::lock_guard lock(handlerMutex_); - auto it = fdbs_.find(clientID); - if (it == fdbs_.end()) { - fdbs_[clientID]; - fdbControlConnection_ = true; - fdbDataConnection_ = !single_; - numControlConnection_++; - if (fdbDataConnection_) - numDataConnection_++; - } + { + std::lock_guard lock(handlerMutex_); + auto it = fdbs_.find(clientID); + if (it == fdbs_.end()) { + fdbs_[clientID]; + fdbControlConnection_ = true; + fdbDataConnection_ = !single_; + numControlConnection_++; + if (fdbDataConnection_) + numDataConnection_++; } + } schema(clientID, requestID, eckit::Buffer(0)); return Handled::Replied; @@ -58,7 +60,6 @@ Handled CatalogueHandler::handleControl(Message message, uint32_t clientID, uint stores(clientID, requestID); return Handled::Replied; - case Message::Archive: // notification that the client is starting to send data locations for archival archiver(); return Handled::YesAddArchiveListener; @@ -71,18 +72,15 @@ Handled CatalogueHandler::handleControl(Message message, uint32_t clientID, uint throw SeriousBug(ss.str(), Here()); } } - } - catch (std::exception& e) { + } catch (std::exception& e) { // n.b. more general than eckit::Exception error(e.what(), clientID, requestID); - } - catch (...) { - error("Caught unexpected and unknown error", clientID, requestID); - } + } catch (...) { error("Caught unexpected and unknown error", clientID, requestID); } return Handled::No; } -Handled CatalogueHandler::handleControl(Message message, uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) { +Handled CatalogueHandler::handleControl(Message message, uint32_t clientID, uint32_t requestID, + eckit::Buffer&& payload) { try { switch (message) { @@ -119,18 +117,13 @@ Handled CatalogueHandler::handleControl(Message message, uint32_t clientID, uint throw SeriousBug(ss.str(), Here()); } } - } - catch (std::exception& e) { + } catch (std::exception& e) { // n.b. more general than eckit::Exception error(e.what(), clientID, requestID); - } - catch (...) { - error("Caught unexpected and unknown error", clientID, requestID); - } + } catch (...) { error("Caught unexpected and unknown error", clientID, requestID); } return Handled::No; } - // API forwarding logic, adapted from original remoteHandler // Used for Inspect and List // *************************************************************************************** @@ -160,34 +153,25 @@ struct BaseHelper { }; struct ListHelper : public BaseHelper { - ListIterator apiCall(FDB& fdb, const FDBToolRequest& request) const { - return fdb.list(request); - } + ListIterator apiCall(FDB& fdb, const FDBToolRequest& request) const { return fdb.list(request); } }; struct AxesHelper : public BaseHelper { virtual size_t encodeBufferSize(const AxesElement& el) const { return el.encodeSize(); } - void extraDecode(eckit::Stream& s) { - s >> level_; - } - AxesIterator apiCall(FDB& fdb, const FDBToolRequest& request) const { - return fdb.axesIterator(request, level_); - } + void extraDecode(eckit::Stream& s) { s >> level_; } + AxesIterator apiCall(FDB& fdb, const FDBToolRequest& request) const { return fdb.axesIterator(request, level_); } + private: int level_; }; struct InspectHelper : public BaseHelper { - ListIterator apiCall(FDB& fdb, const FDBToolRequest& request) const { - return fdb.inspect(request.request()); - } + ListIterator apiCall(FDB& fdb, const FDBToolRequest& request) const { return fdb.inspect(request.request()); } }; struct StatsHelper : public BaseHelper { - StatsIterator apiCall(FDB& fdb, const FDBToolRequest& request) const { - return fdb.stats(request); - } + StatsIterator apiCall(FDB& fdb, const FDBToolRequest& request) const { return fdb.stats(request); } }; template @@ -212,7 +196,6 @@ void CatalogueHandler::forwardApiCall(uint32_t clientID, uint32_t requestID, eck workerThreads_.emplace( requestID, std::async(std::launch::async, [request, clientID, requestID, helper, this]() { - try { FDB* fdb = nullptr; { @@ -226,15 +209,14 @@ void CatalogueHandler::forwardApiCall(uint32_t clientID, uint32_t requestID, eck typename decltype(iterator)::value_type elem; while (iterator.next(elem)) { auto encoded(helper.encode(elem, *this)); - write(Message::Blob, false, clientID, requestID, std::vector>{{encoded.buf, encoded.position}}); + write(Message::Blob, false, clientID, requestID, + std::vector>{{encoded.buf, encoded.position}}); } write(Message::Complete, false, clientID, requestID); - } - catch (std::exception& e) { + } catch (std::exception& e) { // n.b. more general than eckit::Exception error(e.what(), clientID, requestID); - } - catch (...) { + } catch (...) { // We really don't want to std::terminate the thread error("Caught unexpected, unknown exception in worker", clientID, requestID); } @@ -259,7 +241,7 @@ void CatalogueHandler::stats(uint32_t clientID, uint32_t requestID, eckit::Buffe void CatalogueHandler::schema(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) { - eckit::Buffer schemaBuffer(256*1024); + eckit::Buffer schemaBuffer(256 * 1024); eckit::MemoryStream stream(schemaBuffer); if (payload.size() == 0) { // client requesting the top-level schema @@ -284,7 +266,7 @@ void CatalogueHandler::stores(uint32_t clientID, uint32_t requestID) { std::string clientNetwork = ""; if (config_.has("networks")) { - for (const auto& net: config_.getSubConfigurations("networks")) { + for (const auto& net : config_.getSubConfigurations("networks")) { if (net.has("name") && net.has("netmask")) { eckit::net::NetMask netmask{net.getString("netmask")}; if (netmask.contains(clientIPaddress)) { @@ -299,7 +281,7 @@ void CatalogueHandler::stores(uint32_t clientID, uint32_t requestID) { ASSERT(config_.has("stores")); std::map> stores; - for (const auto& configStore: config_.getSubConfigurations("stores")) { + for (const auto& configStore : config_.getSubConfigurations("stores")) { ASSERT(configStore.has("default")); eckit::net::Endpoint fieldLocationEndpoint{configStore.getString("default")}; eckit::net::Endpoint storeEndpoint{fieldLocationEndpoint}; @@ -327,13 +309,13 @@ void CatalogueHandler::stores(uint32_t clientID, uint32_t requestID) { } { - Buffer startupBuffer(16*1024); + Buffer startupBuffer(16 * 1024); MemoryStream s(startupBuffer); s << stores.size(); for (const auto& store : stores) { s << store.first << store.second.size(); - for (const auto& ee: store.second) { + for (const auto& ee : store.second) { s << ee; } } @@ -341,12 +323,6 @@ void CatalogueHandler::stores(uint32_t clientID, uint32_t requestID) { } } - - - - - - void CatalogueHandler::flush(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) { ASSERT(payload.size() > 0); @@ -364,7 +340,8 @@ void CatalogueHandler::flush(uint32_t clientID, uint32_t requestID, eckit::Buffe { std::lock_guard lock(fieldLocationsMutex_); - it->second.locationsExpected = numArchived; // setting locationsExpected also means that a flush has been requested + it->second.locationsExpected = + numArchived; // setting locationsExpected also means that a flush has been requested it->second.archivalCompleted = it->second.fieldLocationsReceived.get_future(); if (it->second.locationsArchived == numArchived) { it->second.fieldLocationsReceived.set_value(numArchived); @@ -395,7 +372,8 @@ void CatalogueHandler::archiveBlob(const uint32_t clientID, const uint32_t reque std::unique_ptr location(eckit::Reanimator::reanimate(s)); - LOG_DEBUG_LIB(LibFdb5) << "CatalogueHandler::archiveBlob key: " << idxKey << datumKey << " location: " << location->uri() << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "CatalogueHandler::archiveBlob key: " << idxKey << datumKey + << " location: " << location->uri() << std::endl; std::map::iterator it; { @@ -413,7 +391,8 @@ void CatalogueHandler::archiveBlob(const uint32_t clientID, const uint32_t reque { std::lock_guard lock(fieldLocationsMutex_); it->second.locationsArchived++; - if (it->second.locationsExpected != 0 && it->second.archivalCompleted.valid() && it->second.locationsExpected == it->second.locationsArchived) { + if (it->second.locationsExpected != 0 && it->second.archivalCompleted.valid() && + it->second.locationsExpected == it->second.locationsArchived) { it->second.fieldLocationsReceived.set_value(it->second.locationsExpected); } } @@ -461,4 +440,4 @@ CatalogueWriter& CatalogueHandler::catalogue(uint32_t id, const Key& dbKey) { return *((catalogues_.emplace(id, CatalogueArchiver(!single_, dbKey, config_)).first)->second.catalogue); } -} // namespace fdb5::remote +} // namespace fdb5::remote diff --git a/src/fdb5/remote/server/CatalogueHandler.h b/src/fdb5/remote/server/CatalogueHandler.h index f63055b3a..aa835e5b1 100644 --- a/src/fdb5/remote/server/CatalogueHandler.h +++ b/src/fdb5/remote/server/CatalogueHandler.h @@ -10,21 +10,24 @@ #pragma once -#include "fdb5/remote/server/ServerConnection.h" #include "fdb5/api/FDB.h" +#include "fdb5/remote/server/ServerConnection.h" namespace fdb5::remote { //---------------------------------------------------------------------------------------------------------------------- struct CatalogueArchiver { - CatalogueArchiver(bool dataConnection, const Key& dbKey, const Config& config) : - controlConnection(true), dataConnection(dataConnection), - catalogue(CatalogueWriterFactory::instance().build(dbKey, config)), locationsExpected(0), locationsArchived(0) {} + CatalogueArchiver(bool dataConnection, const Key& dbKey, const Config& config) + : controlConnection(true) + , dataConnection(dataConnection) + , catalogue(CatalogueWriterFactory::instance().build(dbKey, config)) + , locationsExpected(0) + , locationsArchived(0) {} bool controlConnection; bool dataConnection; - + std::unique_ptr catalogue; size_t locationsExpected; size_t locationsArchived; @@ -35,13 +38,11 @@ struct CatalogueArchiver { //---------------------------------------------------------------------------------------------------------------------- class CatalogueHandler : public ServerConnection { -public: // methods - +public: // methods CatalogueHandler(eckit::net::TCPSocket& socket, const Config& config); ~CatalogueHandler() override; -private: // methods - +private: // methods Handled handleControl(Message message, uint32_t clientID, uint32_t requestID) override; Handled handleControl(Message message, uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) override; @@ -63,8 +64,7 @@ class CatalogueHandler : public ServerConnection { CatalogueWriter& catalogue(uint32_t catalogueID, const Key& dbKey); -private: // member - +private: // member // clientID --> std::map catalogues_; std::map fdbs_; @@ -78,4 +78,4 @@ class CatalogueHandler : public ServerConnection { //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5::remote +} // namespace fdb5::remote diff --git a/src/fdb5/remote/server/ServerConnection.cc b/src/fdb5/remote/server/ServerConnection.cc index 8eaf0c5fc..13635a281 100644 --- a/src/fdb5/remote/server/ServerConnection.cc +++ b/src/fdb5/remote/server/ServerConnection.cc @@ -17,21 +17,21 @@ #include #include "eckit/config/Resource.h" +#include "eckit/log/Log.h" #include "eckit/maths/Functions.h" #include "eckit/net/Endpoint.h" #include "eckit/runtime/Main.h" #include "eckit/runtime/SessionID.h" #include "eckit/serialisation/MemoryStream.h" -#include "eckit/log/Log.h" #include "fdb5/LibFdb5.h" -#include "fdb5/fdb5_version.h" +#include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" #include "fdb5/database/Key.h" -#include "fdb5/remote/server/AvailablePortList.h" +#include "fdb5/fdb5_version.h" #include "fdb5/remote/Messages.h" #include "fdb5/remote/RemoteFieldLocation.h" -#include "fdb5/api/FDB.h" +#include "fdb5/remote/server/AvailablePortList.h" #include "fdb5/remote/server/ServerConnection.h" @@ -40,7 +40,8 @@ namespace fdb5::remote { // helpers namespace { -std::vector intersection(const eckit::LocalConfiguration& c1, const eckit::LocalConfiguration& c2, const std::string& field){ +std::vector intersection(const eckit::LocalConfiguration& c1, const eckit::LocalConfiguration& c2, + const std::string& field) { std::vector v1 = c1.getIntVector(field); std::vector v2 = c2.getIntVector(field); @@ -49,21 +50,23 @@ std::vector intersection(const eckit::LocalConfiguration& c1, const eckit:: std::sort(v1.begin(), v1.end()); std::sort(v2.begin(), v2.end()); - std::set_intersection(v1.begin(),v1.end(), - v2.begin(),v2.end(), - back_inserter(v3)); + std::set_intersection(v1.begin(), v1.end(), v2.begin(), v2.end(), back_inserter(v3)); return v3; } } // namespace -ServerConnection::ServerConnection(eckit::net::TCPSocket& socket, const Config& config) : - Connection(), config_(config), - dataListenHostname_(config.getString("dataListenHostname", "")), - readLocationQueue_(eckit::Resource("fdbRetrieveQueueSize", 10000)), - archiveQueue_(eckit::Resource("fdbServerMaxQueueSize", 320)), - controlSocket_(socket), numControlConnection_(0), numDataConnection_(0), - dataSocket_(nullptr), dataListener_(0) { +ServerConnection::ServerConnection(eckit::net::TCPSocket& socket, const Config& config) + : Connection() + , config_(config) + , dataListenHostname_(config.getString("dataListenHostname", "")) + , readLocationQueue_(eckit::Resource("fdbRetrieveQueueSize", 10000)) + , archiveQueue_(eckit::Resource("fdbServerMaxQueueSize", 320)) + , controlSocket_(socket) + , numControlConnection_(0) + , numDataConnection_(0) + , dataSocket_(nullptr) + , dataListener_(0) { LOG_DEBUG_LIB(LibFdb5) << "ServerConnection::ServerConnection initialized" << std::endl; } @@ -75,15 +78,14 @@ ServerConnection::~ServerConnection() { if (archiveFuture_.valid()) { archiveFuture_.wait(); } - + eckit::Log::info() << "Done" << std::endl; } - Handled ServerConnection::handleData(Message message, uint32_t clientID, uint32_t requestID) { try { switch (message) { - case Message::Flush: // notification that the client has sent all data for archival + case Message::Flush: // notification that the client has sent all data for archival return Handled::YesRemoveArchiveListener; // ???? default: { @@ -94,14 +96,10 @@ Handled ServerConnection::handleData(Message message, uint32_t clientID, uint32_ throw eckit::SeriousBug(ss.str(), Here()); } } - } - catch (std::exception& e) { + } catch (std::exception& e) { // n.b. more general than eckit::Exception error(e.what(), clientID, requestID); - } - catch (...) { - error("Caught unexpected and unknown error", clientID, requestID); - } + } catch (...) { error("Caught unexpected and unknown error", clientID, requestID); } return Handled::No; } @@ -121,22 +119,19 @@ Handled ServerConnection::handleData(Message message, uint32_t clientID, uint32_ throw eckit::SeriousBug(ss.str(), Here()); } } - } - catch (std::exception& e) { + } catch (std::exception& e) { // n.b. more general than eckit::Exception error(e.what(), clientID, requestID); - } - catch (...) { - error("Caught unexpected and unknown error", clientID, requestID); - } + } catch (...) { error("Caught unexpected and unknown error", clientID, requestID); } return Handled::No; } eckit::LocalConfiguration ServerConnection::availableFunctionality() const { eckit::LocalConfiguration conf; -// Add to the configuration all the components that require to be versioned, as in the following example, with a vector of supported version numbers + // Add to the configuration all the components that require to be versioned, as in the following example, with a + // vector of supported version numbers static std::vector remoteFieldLocationVersions = {1}; - static std::vector numberOfConnections = config_.getIntVector("supportedConnections", {1,2}); + static std::vector numberOfConnections = config_.getIntVector("supportedConnections", {1, 2}); ASSERT(0 < numberOfConnections.size()); ASSERT(numberOfConnections[0] == 1 || numberOfConnections[0] == 2); @@ -169,14 +164,13 @@ void ServerConnection::initialiseConnections() { try { s1 >> remoteProtocolVersion; - } catch (...) { - errorMsg = "Error retrieving client protocol version"; - } + } catch (...) { errorMsg = "Error retrieving client protocol version"; } if (errorMsg.empty() && !LibFdb5::instance().remoteProtocolVersion().check(remoteProtocolVersion, false)) { std::stringstream ss; ss << "FDB server version " << fdb5_version_str() << " - remote protocol version not supported:" << std::endl; - ss << " versions supported by server: " << LibFdb5::instance().remoteProtocolVersion().supportedStr() << std::endl; + ss << " versions supported by server: " << LibFdb5::instance().remoteProtocolVersion().supportedStr() + << std::endl; ss << " version requested by client: " << remoteProtocolVersion << std::endl; errorMsg = ss.str(); } @@ -187,10 +181,10 @@ void ServerConnection::initialiseConnections() { agreedConf_ = eckit::LocalConfiguration(); bool compatibleProtocol = true; - std::vector rflCommon = intersection(clientAvailableFunctionality, serverConf, "RemoteFieldLocation"); if (rflCommon.size() > 0) { - LOG_DEBUG_LIB(LibFdb5) << "Protocol negotiation - RemoteFieldLocation version " << rflCommon.back() << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "Protocol negotiation - RemoteFieldLocation version " << rflCommon.back() + << std::endl; agreedConf_.set("RemoteFieldLocation", rflCommon.back()); } else { compatibleProtocol = false; @@ -210,7 +204,9 @@ void ServerConnection::initialiseConnections() { } else { ncSelected = ncCommon.back(); if (clientAvailableFunctionality.has("PreferSingleConnection")) { - if ( std::find(ncCommon.begin(), ncCommon.end(), (clientAvailableFunctionality.getBool("PreferSingleConnection") ? 1 : 2)) != ncCommon.end() ) { + if (std::find(ncCommon.begin(), ncCommon.end(), + (clientAvailableFunctionality.getBool("PreferSingleConnection") ? 1 : 2)) != + ncCommon.end()) { ncSelected = (clientAvailableFunctionality.getBool("PreferSingleConnection") ? 1 : 2); } } @@ -219,10 +215,10 @@ void ServerConnection::initialiseConnections() { LOG_DEBUG_LIB(LibFdb5) << "Protocol negotiation - NumberOfConnections " << ncSelected << std::endl; agreedConf_.set("NumberOfConnections", ncSelected); single_ = (ncSelected == 1); - } - else { + } else { std::stringstream ss; - ss << "FDB server version " << fdb5_version_str() << " - failed protocol negotiation with FDB client" << std::endl; + ss << "FDB server version " << fdb5_version_str() << " - failed protocol negotiation with FDB client" + << std::endl; ss << " server functionality: " << serverConf << std::endl; ss << " client functionality: " << clientAvailableFunctionality << std::endl; errorMsg = ss.str(); @@ -246,7 +242,10 @@ void ServerConnection::initialiseConnections() { dataEndpoint = eckit::net::Endpoint{endpointFromClient.hostname(), dataSocket_->localPort()}; - dataSocketFuture = std::async(std::launch::async, [this] { dataSocket_->accept(); return true; }); + dataSocketFuture = std::async(std::launch::async, [this] { + dataSocket_->accept(); + return true; + }); } eckit::Buffer startupBuffer(1024); @@ -257,9 +256,10 @@ void ServerConnection::initialiseConnections() { s << dataEndpoint; s << agreedConf_.get(); - LOG_DEBUG_LIB(LibFdb5) << "Protocol negotiation - configuration: " << agreedConf_ <>{{startupBuffer.data(), s.position()}}); + write(Message::Startup, true, 0, 0, + std::vector>{{startupBuffer.data(), s.position()}}); if (!single_) { ASSERT(dataSocketFuture.valid()); @@ -291,7 +291,7 @@ void ServerConnection::initialiseConnections() { std::stringstream ss; ss << "Session IDs do not match: " << serverSession << " != " << sessionID_; throw eckit::BadValue(ss.str(), Here()); - } + } } if (!errorMsg.empty()) { @@ -305,8 +305,7 @@ int ServerConnection::selectDataPort() { eckit::Log::info() << config_ << std::endl; if (config_.has("dataPortStart")) { ASSERT(config_.has("dataPortCount")); - return AvailablePortList(config_.getInt("dataPortStart"), config_.getLong("dataPortCount")) - .acquire(); + return AvailablePortList(config_.getInt("dataPortStart"), config_.getLong("dataPortCount")).acquire(); } // Use a system assigned port @@ -324,7 +323,7 @@ size_t ServerConnection::archiveThreadLoop() { if (elem.multiblob_) { // Handle MultiBlob - const char* firstData = static_cast(elem.payload_.data()); // For pointer arithmetic + const char* firstData = static_cast(elem.payload_.data()); // For pointer arithmetic const char* charData = firstData; while (size_t(charData - firstData) < elem.payload_.size()) { const MessageHeader* hdr = static_cast(static_cast(charData)); @@ -336,22 +335,21 @@ size_t ServerConnection::archiveThreadLoop() { const void* payloadData = charData; charData += hdr->payloadSize; - const decltype(EndMarker)* e = static_cast(static_cast(charData)); + const decltype(EndMarker)* e = + static_cast(static_cast(charData)); ASSERT(*e == EndMarker); charData += sizeof(EndMarker); archiveBlob(elem.clientID_, elem.requestID_, payloadData, hdr->payloadSize); totalArchived += 1; } - } - else { + } else { // Handle single blob archiveBlob(elem.clientID_, elem.requestID_, elem.payload_.data(), elem.payload_.size()); totalArchived += 1; } } - } - catch (...) { + } catch (...) { // Ensure exception propagates across the queue back to the parent thread. archiveQueue_.interrupt(std::current_exception()); throw; @@ -360,8 +358,6 @@ size_t ServerConnection::archiveThreadLoop() { return totalArchived; } - - void ServerConnection::listeningThreadLoopData() { MessageHeader hdr; @@ -382,7 +378,7 @@ void ServerConnection::listeningThreadLoopData() { break; } else { - + Handled handled; if (payload.size() == 0) { handled = handleData(hdr.message, hdr.clientID(), hdr.requestID); @@ -390,18 +386,16 @@ void ServerConnection::listeningThreadLoopData() { handled = handleData(hdr.message, hdr.clientID(), hdr.requestID, std::move(payload)); } - switch (handled) - { + switch (handled) { case Handled::YesRemoveArchiveListener: - case Handled::YesRemoveReadListener: - { - std::lock_guard lock(handlerMutex_); - dataListener_--; - if (dataListener_ == 0) { - return; - } - break; + case Handled::YesRemoveReadListener: { + std::lock_guard lock(handlerMutex_); + dataListener_--; + if (dataListener_ == 0) { + return; } + break; + } case Handled::Replied: // nothing to do case Handled::Yes: break; @@ -414,13 +408,11 @@ void ServerConnection::listeningThreadLoopData() { } } - } - catch (std::exception& e) { + } catch (std::exception& e) { // n.b. more general than eckit::Exception error(e.what(), hdr.clientID(), hdr.requestID); throw; - } - catch (...) { + } catch (...) { error("Caught unexpected, unknown exception in retrieve worker", hdr.clientID(), hdr.requestID); throw; } @@ -428,7 +420,7 @@ void ServerConnection::listeningThreadLoopData() { void ServerConnection::handle() { initialiseConnections(); - + std::thread listeningThreadData; MessageHeader hdr; @@ -436,7 +428,9 @@ void ServerConnection::handle() { // listen loop while (true) { eckit::Buffer payload = readControl(hdr); // READ CONTROL - LOG_DEBUG_LIB(LibFdb5) << "ServerConnection::handle - got [message=" << hdr.message << ",clientID="<< hdr.clientID() << ",requestID=" << hdr.requestID << ",payload=" << hdr.payloadSize << "]" << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "ServerConnection::handle - got [message=" << hdr.message + << ",clientID=" << hdr.clientID() << ",requestID=" << hdr.requestID + << ",payload=" << hdr.payloadSize << "]" << std::endl; if (hdr.message == Message::Stop) { ASSERT(hdr.clientID()); @@ -450,8 +444,7 @@ void ServerConnection::handle() { eckit::Log::info() << "Terminating CONTROL listener" << std::endl; break; - } - else { + } else { Handled handled = Handled::No; ASSERT(single_ || hdr.control()); @@ -469,21 +462,18 @@ void ServerConnection::handle() { handled = handleData(hdr.message, hdr.clientID(), hdr.requestID); } } - - switch (handled) - { + switch (handled) { case Handled::Replied: // nothing to do break; case Handled::YesAddArchiveListener: - case Handled::YesAddReadListener: - { - std::lock_guard lock(handlerMutex_); - dataListener_++; - if (dataListener_ == 1 && !single_) { - listeningThreadData = std::thread([this] { listeningThreadLoopData(); }); - } + case Handled::YesAddReadListener: { + std::lock_guard lock(handlerMutex_); + dataListener_++; + if (dataListener_ == 1 && !single_) { + listeningThreadData = std::thread([this] { listeningThreadLoopData(); }); } + } case Handled::Yes: write(Message::Received, true, hdr.clientID(), hdr.requestID); break; @@ -507,21 +497,15 @@ void ServerConnection::handle() { } void ServerConnection::handleException(std::exception_ptr e) { - try - { + try { if (e) std::rethrow_exception(e); - } - catch(const std::exception& e) - { - error(e.what(), 0, 0); - } + } catch (const std::exception& e) { error(e.what(), 0, 0); } } void ServerConnection::queue(Message message, uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) { - archiveQueue_.emplace( - ArchiveElem{clientID, requestID, std::move(payload), message == Message::MultiBlob}); + archiveQueue_.emplace(ArchiveElem{clientID, requestID, std::move(payload), message == Message::MultiBlob}); } void ServerConnection::tidyWorkers() { @@ -533,14 +517,12 @@ void ServerConnection::tidyWorkers() { if (stat == std::future_status::ready) { eckit::Log::info() << "Tidying up worker for request ID: " << it->first << std::endl; workerThreads_.erase(it++); - } - else { + } else { ++it; } } } - void ServerConnection::archiver() { // Ensure that we aren't already running a catalogue/store @@ -568,4 +550,4 @@ void ServerConnection::waitForWorkers() { } } -} // namespace fdb5::remote +} // namespace fdb5::remote diff --git a/src/fdb5/remote/server/ServerConnection.h b/src/fdb5/remote/server/ServerConnection.h index 66ef81f56..93230cd8a 100644 --- a/src/fdb5/remote/server/ServerConnection.h +++ b/src/fdb5/remote/server/ServerConnection.h @@ -28,8 +28,8 @@ #include "eckit/runtime/SessionID.h" #include "fdb5/config/Config.h" -#include "fdb5/remote/Messages.h" #include "fdb5/remote/Connection.h" +#include "fdb5/remote/Messages.h" namespace fdb5::remote { @@ -48,7 +48,6 @@ enum class Handled { class Handler { public: - virtual Handled handleControl(Message message, uint32_t clientID, uint32_t requestID) = 0; virtual Handled handleControl(Message message, uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) = 0; virtual Handled handleData(Message message, uint32_t clientID, uint32_t requestID) = 0; @@ -66,9 +65,11 @@ struct readLocationElem { std::unique_ptr readLocation; readLocationElem() : clientID(0), requestID(0), readLocation(nullptr) {} - - readLocationElem(uint32_t clientID, uint32_t requestID, std::unique_ptr readLocation) : - clientID(clientID), requestID(requestID), readLocation(std::move(readLocation)) {} + + readLocationElem(uint32_t clientID, uint32_t requestID, std::unique_ptr readLocation) + : clientID(clientID) + , requestID(requestID) + , readLocation(std::move(readLocation)) {} }; struct ArchiveElem { @@ -80,12 +81,15 @@ struct ArchiveElem { ArchiveElem() : clientID_(0), requestID_(0), payload_(0), multiblob_(false) {} - ArchiveElem(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload, bool multiblob) : - clientID_(clientID), requestID_(requestID), payload_(std::move(payload)), multiblob_(multiblob) {} + ArchiveElem(uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload, bool multiblob) + : clientID_(clientID) + , requestID_(requestID) + , payload_(std::move(payload)) + , multiblob_(multiblob) {} }; class ServerConnection : public Connection, public Handler { -public: // methods +public: // methods ServerConnection(eckit::net::TCPSocket& socket, const Config& config); ~ServerConnection() override; @@ -101,11 +105,10 @@ class ServerConnection : public Connection, public Handler { Handled handleData(Message message, uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) override; protected: - // socket methods int selectDataPort(); eckit::LocalConfiguration availableFunctionality() const; - + // Worker functionality void tidyWorkers(); void waitForWorkers(); @@ -121,17 +124,15 @@ class ServerConnection : public Connection, public Handler { void handleException(std::exception_ptr e) override; private: - void listeningThreadLoopData(); eckit::net::TCPSocket& controlSocket() override { return controlSocket_; } - eckit::net::TCPSocket& dataSocket() override { + eckit::net::TCPSocket& dataSocket() override { ASSERT(dataSocket_); return *dataSocket_; } protected: - virtual bool remove(bool control, uint32_t clientID) = 0; Config config_; @@ -143,7 +144,7 @@ class ServerConnection : public Connection, public Handler { eckit::LocalConfiguration agreedConf_; std::mutex readLocationMutex_; std::thread readLocationWorker_; - + std::map> workerThreads_; eckit::Queue archiveQueue_; std::future archiveFuture_; @@ -155,7 +156,6 @@ class ServerConnection : public Connection, public Handler { size_t numDataConnection_; private: - std::mutex dataPortMutex_; // data connection @@ -165,4 +165,4 @@ class ServerConnection : public Connection, public Handler { //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5::remote +} // namespace fdb5::remote diff --git a/src/fdb5/remote/server/StoreHandler.cc b/src/fdb5/remote/server/StoreHandler.cc index 39a9cbbec..db8a7d8e3 100644 --- a/src/fdb5/remote/server/StoreHandler.cc +++ b/src/fdb5/remote/server/StoreHandler.cc @@ -20,10 +20,9 @@ using metkit::mars::MarsRequest; namespace fdb5::remote { -StoreHandler::StoreHandler(eckit::net::TCPSocket& socket, const Config& config): - ServerConnection(socket, config) { - LibFdb5::instance().constructorCallback()(*this); - } +StoreHandler::StoreHandler(eckit::net::TCPSocket& socket, const Config& config) : ServerConnection(socket, config) { + LibFdb5::instance().constructorCallback()(*this); +} StoreHandler::~StoreHandler() {} @@ -34,7 +33,7 @@ Handled StoreHandler::handleControl(Message message, uint32_t clientID, uint32_t case Message::Store: // notification that the client is starting to send data for archival archiver(); return Handled::YesAddArchiveListener; - + default: { std::stringstream ss; ss << "ERROR: Unexpected message recieved (" << message << "). ABORTING"; @@ -43,14 +42,10 @@ Handled StoreHandler::handleControl(Message message, uint32_t clientID, uint32_t throw SeriousBug(ss.str(), Here()); } } - } - catch (std::exception& e) { + } catch (std::exception& e) { // n.b. more general than eckit::Exception error(e.what(), clientID, requestID); - } - catch (...) { - error("Caught unexpected and unknown error", clientID, requestID); - } + } catch (...) { error("Caught unexpected and unknown error", clientID, requestID); } return Handled::No; } @@ -75,14 +70,10 @@ Handled StoreHandler::handleControl(Message message, uint32_t clientID, uint32_t throw SeriousBug(ss.str(), Here()); } } - } - catch (std::exception& e) { + } catch (std::exception& e) { // n.b. more general than eckit::Exception error(e.what(), clientID, requestID); - } - catch (...) { - error("Caught unexpected and unknown error", clientID, requestID); - } + } catch (...) { error("Caught unexpected and unknown error", clientID, requestID); } return Handled::No; } @@ -119,45 +110,43 @@ void StoreHandler::readLocationThreadLoop() { } } -void StoreHandler::writeToParent(const uint32_t clientID, const uint32_t requestID, std::unique_ptr dh) { - try { +void StoreHandler::writeToParent(const uint32_t clientID, const uint32_t requestID, + std::unique_ptr dh) { + try { Log::status() << "Reading: " << requestID << std::endl; // Write the data to the parent, in chunks if necessary. - Buffer writeBuffer(4 * 1024 * 1024 - 2048); // slightly smaller than 4MiB to nicely fit in a TCP window with scale factor 6 + Buffer writeBuffer(4 * 1024 * 1024 - + 2048); // slightly smaller than 4MiB to nicely fit in a TCP window with scale factor 6 long dataRead; dh->openForRead(); - LOG_DEBUG_LIB(LibFdb5) << "Reading: " << requestID << " dh size: " << dh->size() - << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "Reading: " << requestID << " dh size: " << dh->size() << std::endl; while ((dataRead = dh->read(writeBuffer, writeBuffer.size())) != 0) { - write(Message::Blob, false, clientID, requestID, std::vector>{{writeBuffer, dataRead}}); + write(Message::Blob, false, clientID, requestID, + std::vector>{{writeBuffer, dataRead}}); } // And when we are done, add a complete message. - LOG_DEBUG_LIB(LibFdb5) << "Writing retrieve complete message: " << requestID - << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "Writing retrieve complete message: " << requestID << std::endl; write(Message::Complete, false, clientID, requestID); Log::status() << "Done retrieve: " << requestID << std::endl; LOG_DEBUG_LIB(LibFdb5) << "Done retrieve: " << requestID << std::endl; - } - catch (std::exception& e) { + } catch (std::exception& e) { // n.b. more general than eckit::Exception error(e.what(), clientID, requestID); - } - catch (...) { + } catch (...) { // We really don't want to std::terminate the thread error("Caught unexpected , unknown exception in retrieve worker", clientID, requestID); } } - void StoreHandler::archiveBlob(const uint32_t clientID, const uint32_t requestID, const void* data, size_t length) { - + MemoryStream s(data, length); fdb5::Key dbKey(s); @@ -166,9 +155,9 @@ void StoreHandler::archiveBlob(const uint32_t clientID, const uint32_t requestID std::stringstream ss_key; ss_key << dbKey << idxKey; - const char* charData = static_cast(data); // To allow pointer arithmetic + const char* charData = static_cast(data); // To allow pointer arithmetic Log::status() << "Archiving data: " << ss_key.str() << std::endl; - + Store& ss = store(clientID, dbKey); std::shared_ptr location = ss.archive(idxKey, charData + s.position(), length - s.position()); @@ -194,7 +183,6 @@ void StoreHandler::flush(uint32_t clientID, uint32_t requestID, const eckit::Buf size_t numArchived = 0; - if (requestID != 0) { ASSERT(payload.size() > 0); MemoryStream stream(payload); @@ -217,7 +205,7 @@ void StoreHandler::flush(uint32_t clientID, uint32_t requestID, const eckit::Buf } bool StoreHandler::remove(bool control, uint32_t clientID) { - + std::lock_guard lock(handlerMutex_); auto it = stores_.find(clientID); if (it != stores_.end()) { @@ -264,4 +252,4 @@ Store& StoreHandler::store(uint32_t clientID, const Key& dbKey) { return *((stores_.emplace(clientID, StoreHelper(!single_, dbKey, config_)).first)->second.store); } -} // namespace fdb5::remote +} // namespace fdb5::remote diff --git a/src/fdb5/remote/server/StoreHandler.h b/src/fdb5/remote/server/StoreHandler.h index 40362b09a..a3f559cad 100644 --- a/src/fdb5/remote/server/StoreHandler.h +++ b/src/fdb5/remote/server/StoreHandler.h @@ -15,29 +15,28 @@ #include "fdb5/remote/server/ServerConnection.h" namespace fdb5::remote { - + //---------------------------------------------------------------------------------------------------------------------- struct StoreHelper { - StoreHelper(bool dataConnection, const Key& dbKey, const Config& config) : - controlConnection(true), dataConnection(dataConnection), - store(StoreFactory::instance().build(dbKey, config)) {} + StoreHelper(bool dataConnection, const Key& dbKey, const Config& config) + : controlConnection(true) + , dataConnection(dataConnection) + , store(StoreFactory::instance().build(dbKey, config)) {} bool controlConnection; bool dataConnection; - + std::unique_ptr store; }; //---------------------------------------------------------------------------------------------------------------------- class StoreHandler : public ServerConnection, public CallbackRegistry { -public: // methods - +public: // methods StoreHandler(eckit::net::TCPSocket& socket, const Config& config); ~StoreHandler() override; -private: // methods - +private: // methods Handled handleControl(Message message, uint32_t clientID, uint32_t requestID) override; Handled handleControl(Message message, uint32_t clientID, uint32_t requestID, eckit::Buffer&& payload) override; @@ -54,12 +53,11 @@ class StoreHandler : public ServerConnection, public CallbackRegistry { Store& store(uint32_t clientID); Store& store(uint32_t clientID, const Key& dbKey); -private: // members - +private: // members // clientID --> Store std::map stores_; }; //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5::remote +} // namespace fdb5::remote diff --git a/src/fdb5/rules/MatchAlways.cc b/src/fdb5/rules/MatchAlways.cc index af0554a49..74feeb054 100644 --- a/src/fdb5/rules/MatchAlways.cc +++ b/src/fdb5/rules/MatchAlways.cc @@ -17,34 +17,30 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -eckit::ClassSpec MatchAlways::classSpec_ = { &Matcher::classSpec(), "MatchAlways", }; +eckit::ClassSpec MatchAlways::classSpec_ = { + &Matcher::classSpec(), + "MatchAlways", +}; eckit::Reanimator MatchAlways::reanimator_; +MatchAlways::MatchAlways() : Matcher() {} -MatchAlways::MatchAlways() : - Matcher() { -} +MatchAlways::MatchAlways(eckit::Stream&) : Matcher() {} -MatchAlways::MatchAlways(eckit::Stream&) : - Matcher() { -} +void MatchAlways::encode(eckit::Stream& s) const {} -void MatchAlways::encode(eckit::Stream& s) const { -} - -MatchAlways::~MatchAlways() { -} +MatchAlways::~MatchAlways() {} bool MatchAlways::match(const std::string&, const Key&) const { return true; } -void MatchAlways::dump(std::ostream &s, const std::string &keyword, const TypesRegistry ®istry) const { +void MatchAlways::dump(std::ostream& s, const std::string& keyword, const TypesRegistry& registry) const { registry.dump(s, keyword); } -void MatchAlways::print(std::ostream &out) const { +void MatchAlways::print(std::ostream& out) const { out << "MatchAlways[]"; } diff --git a/src/fdb5/rules/MatchAlways.h b/src/fdb5/rules/MatchAlways.h index 9d699c7b6..1a6fb3e50 100644 --- a/src/fdb5/rules/MatchAlways.h +++ b/src/fdb5/rules/MatchAlways.h @@ -26,7 +26,6 @@ namespace fdb5 { class MatchAlways : public Matcher { public: // methods - MatchAlways(); MatchAlways(eckit::Stream& s); @@ -37,19 +36,16 @@ class MatchAlways : public Matcher { void dump(std::ostream& s, const std::string& keyword, const TypesRegistry& registry) const override; const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } - static const eckit::ClassSpec& classSpec() { return classSpec_; } + static const eckit::ClassSpec& classSpec() { return classSpec_; } private: // methods - void encode(eckit::Stream&) const override; - void print( std::ostream& out ) const override; + void print(std::ostream& out) const override; private: // members - static eckit::ClassSpec classSpec_; static eckit::Reanimator reanimator_; - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/rules/MatchAny.cc b/src/fdb5/rules/MatchAny.cc index 2ecee2510..d510a03ff 100644 --- a/src/fdb5/rules/MatchAny.cc +++ b/src/fdb5/rules/MatchAny.cc @@ -8,32 +8,30 @@ * does it submit to any jurisdiction. */ -#include "fdb5/database/Key.h" #include "fdb5/rules/MatchAny.h" +#include "fdb5/database/Key.h" #include "fdb5/types/TypesRegistry.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -eckit::ClassSpec MatchAny::classSpec_ = { &Matcher::classSpec(), "MatchAny", }; +eckit::ClassSpec MatchAny::classSpec_ = { + &Matcher::classSpec(), + "MatchAny", +}; eckit::Reanimator MatchAny::reanimator_; +MatchAny::MatchAny(const std::set& values) : Matcher(), values_(values) {} -MatchAny::MatchAny(const std::set &values) : - Matcher(), - values_(values) { -} - -MatchAny::MatchAny(eckit::Stream& s) : - Matcher() { +MatchAny::MatchAny(eckit::Stream& s) : Matcher() { size_t numValues; std::string value; s >> numValues; - for (size_t i=0; i < numValues; i++) { + for (size_t i = 0; i < numValues; i++) { s >> value; values_.insert(value); } @@ -46,10 +44,9 @@ void MatchAny::encode(eckit::Stream& s) const { } } -MatchAny::~MatchAny() { -} +MatchAny::~MatchAny() {} -bool MatchAny::match(const std::string &keyword, const Key& key) const { +bool MatchAny::match(const std::string& keyword, const Key& key) const { auto i = key.find(keyword); @@ -60,20 +57,20 @@ bool MatchAny::match(const std::string &keyword, const Key& key) const { return (values_.find(i->second) != values_.end()); } -void MatchAny::dump(std::ostream &s, const std::string &keyword, const TypesRegistry ®istry) const { - const char *sep = ""; +void MatchAny::dump(std::ostream& s, const std::string& keyword, const TypesRegistry& registry) const { + const char* sep = ""; registry.dump(s, keyword); s << "="; - for ( std::set::const_iterator i = values_.begin(); i != values_.end(); ++i) { + for (std::set::const_iterator i = values_.begin(); i != values_.end(); ++i) { s << sep << *i; sep = "/"; } } -void MatchAny::print(std::ostream &out) const { +void MatchAny::print(std::ostream& out) const { out << "MatchAny[values="; - const char *sep = ""; - for ( std::set::const_iterator i = values_.begin(); i != values_.end(); ++i) { + const char* sep = ""; + for (std::set::const_iterator i = values_.begin(); i != values_.end(); ++i) { out << sep << *i; sep = ","; } diff --git a/src/fdb5/rules/MatchAny.h b/src/fdb5/rules/MatchAny.h index 37294c9cb..828d5f1c6 100644 --- a/src/fdb5/rules/MatchAny.h +++ b/src/fdb5/rules/MatchAny.h @@ -24,11 +24,10 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -class MatchAny : public Matcher{ +class MatchAny : public Matcher { public: // methods - - MatchAny(const std::set &values); + MatchAny(const std::set& values); MatchAny(eckit::Stream& s); ~MatchAny() override; @@ -38,21 +37,18 @@ class MatchAny : public Matcher{ void dump(std::ostream& s, const std::string& keyword, const TypesRegistry& registry) const override; const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } - static const eckit::ClassSpec& classSpec() { return classSpec_; } + static const eckit::ClassSpec& classSpec() { return classSpec_; } private: // methods - void encode(eckit::Stream&) const override; - void print( std::ostream& out ) const override; + void print(std::ostream& out) const override; private: // members - static eckit::ClassSpec classSpec_; static eckit::Reanimator reanimator_; std::set values_; - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/rules/MatchHidden.cc b/src/fdb5/rules/MatchHidden.cc index 8dc5fb8d6..f8d6959ed 100644 --- a/src/fdb5/rules/MatchHidden.cc +++ b/src/fdb5/rules/MatchHidden.cc @@ -10,9 +10,9 @@ #include "eckit/log/Log.h" -#include "fdb5/rules/MatchHidden.h" -#include "fdb5/database/Key.h" #include "eckit/types/Types.h" +#include "fdb5/database/Key.h" +#include "fdb5/rules/MatchHidden.h" #include "fdb5/types/TypesRegistry.h" namespace fdb5 { @@ -21,24 +21,24 @@ static std::string empty; //---------------------------------------------------------------------------------------------------------------------- -eckit::ClassSpec MatchHidden::classSpec_ = { &Matcher::classSpec(), "MatchHidden", }; +eckit::ClassSpec MatchHidden::classSpec_ = { + &Matcher::classSpec(), + "MatchHidden", +}; eckit::Reanimator MatchHidden::reanimator_; - -MatchHidden::MatchHidden(const std::string &def) : - Matcher() { +MatchHidden::MatchHidden(const std::string& def) : Matcher() { default_.push_back(def); } -MatchHidden::MatchHidden(eckit::Stream& s) : - Matcher() { - +MatchHidden::MatchHidden(eckit::Stream& s) : Matcher() { + size_t numValues; std::string value; s >> numValues; - for (size_t i=0; i < numValues; i++) { + for (size_t i = 0; i < numValues; i++) { s >> value; default_.push_back(value); } @@ -51,8 +51,7 @@ void MatchHidden::encode(eckit::Stream& s) const { } } -MatchHidden::~MatchHidden() { -} +MatchHidden::~MatchHidden() {} bool MatchHidden::match(const std::string&, const Key&) const { return true; @@ -62,24 +61,25 @@ bool MatchHidden::optional() const { return true; } -const std::string &MatchHidden::value(const Key&, const std::string&) const { +const std::string& MatchHidden::value(const Key&, const std::string&) const { return default_[0]; } -const std::vector& MatchHidden::values(const metkit::mars::MarsRequest &rq, const std::string &keyword) const { +const std::vector& MatchHidden::values(const metkit::mars::MarsRequest& rq, + const std::string& keyword) const { return default_; } -const std::string &MatchHidden::defaultValue() const { +const std::string& MatchHidden::defaultValue() const { return default_[0]; } -void MatchHidden::dump(std::ostream &s, const std::string &keyword, const TypesRegistry ®istry) const { +void MatchHidden::dump(std::ostream& s, const std::string& keyword, const TypesRegistry& registry) const { registry.dump(s, keyword); s << '-' << default_[0]; } -void MatchHidden::print(std::ostream &out) const { +void MatchHidden::print(std::ostream& out) const { out << "MatchHidden[default" << default_[0] << "]"; } diff --git a/src/fdb5/rules/MatchHidden.h b/src/fdb5/rules/MatchHidden.h index 115dc916c..bab6c6802 100644 --- a/src/fdb5/rules/MatchHidden.h +++ b/src/fdb5/rules/MatchHidden.h @@ -25,11 +25,10 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -class MatchHidden : public Matcher{ +class MatchHidden : public Matcher { public: // methods - - MatchHidden(const std::string &def); + MatchHidden(const std::string& def); MatchHidden(eckit::Stream& s); ~MatchHidden() override; @@ -39,25 +38,23 @@ class MatchHidden : public Matcher{ void dump(std::ostream& s, const std::string& keyword, const TypesRegistry& registry) const override; const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } - static const eckit::ClassSpec& classSpec() { return classSpec_; } + static const eckit::ClassSpec& classSpec() { return classSpec_; } private: // methods - void encode(eckit::Stream&) const override; bool optional() const override; - const std::string &value(const Key&, const std::string& keyword) const override; - const std::vector& values(const metkit::mars::MarsRequest& rq, const std::string& keyword) const override; - void print( std::ostream& out ) const override; + const std::string& value(const Key&, const std::string& keyword) const override; + const std::vector& values(const metkit::mars::MarsRequest& rq, + const std::string& keyword) const override; + void print(std::ostream& out) const override; const std::string& defaultValue() const override; private: // members - static eckit::ClassSpec classSpec_; static eckit::Reanimator reanimator_; std::vector default_; - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/rules/MatchOptional.cc b/src/fdb5/rules/MatchOptional.cc index 8f90170b2..f20f7a386 100644 --- a/src/fdb5/rules/MatchOptional.cc +++ b/src/fdb5/rules/MatchOptional.cc @@ -24,24 +24,24 @@ static std::string empty; //---------------------------------------------------------------------------------------------------------------------- -eckit::ClassSpec MatchOptional::classSpec_ = { &Matcher::classSpec(), "MatchOptional", }; +eckit::ClassSpec MatchOptional::classSpec_ = { + &Matcher::classSpec(), + "MatchOptional", +}; eckit::Reanimator MatchOptional::reanimator_; - -MatchOptional::MatchOptional(const std::string &def) : - Matcher() { +MatchOptional::MatchOptional(const std::string& def) : Matcher() { default_.push_back(def); } -MatchOptional::MatchOptional(eckit::Stream& s) : - Matcher() { - +MatchOptional::MatchOptional(eckit::Stream& s) : Matcher() { + size_t numValues; std::string value; s >> numValues; - for (size_t i=0; i < numValues; i++) { + for (size_t i = 0; i < numValues; i++) { s >> value; default_.push_back(value); } @@ -54,8 +54,7 @@ void MatchOptional::encode(eckit::Stream& s) const { } } -MatchOptional::~MatchOptional() { -} +MatchOptional::~MatchOptional() {} bool MatchOptional::match(const std::string&, const Key&) const { return true; @@ -65,13 +64,13 @@ bool MatchOptional::optional() const { return true; } -void MatchOptional::fill(BaseKey& key, const std::string &keyword, const std::string& value) const { +void MatchOptional::fill(BaseKey& key, const std::string& keyword, const std::string& value) const { if (!value.empty()) { key.push(keyword, value); } } -const std::string &MatchOptional::value(const Key& key, const std::string &keyword) const { +const std::string& MatchOptional::value(const Key& key, const std::string& keyword) const { Key::const_iterator i = key.find(keyword); if (i == key.end()) { @@ -81,7 +80,8 @@ const std::string &MatchOptional::value(const Key& key, const std::string &keywo return key.get(keyword); } -const std::vector& MatchOptional::values(const metkit::mars::MarsRequest& rq, const std::string& keyword) const { +const std::vector& MatchOptional::values(const metkit::mars::MarsRequest& rq, + const std::string& keyword) const { if (rq.has(keyword)) { return rq.values(keyword); @@ -89,16 +89,16 @@ const std::vector& MatchOptional::values(const metkit::mars::MarsRe return default_; } -const std::string &MatchOptional::defaultValue() const { +const std::string& MatchOptional::defaultValue() const { return default_[0]; } -void MatchOptional::dump(std::ostream &s, const std::string &keyword, const TypesRegistry ®istry) const { +void MatchOptional::dump(std::ostream& s, const std::string& keyword, const TypesRegistry& registry) const { registry.dump(s, keyword); s << '?' << default_[0]; } -void MatchOptional::print(std::ostream &out) const { +void MatchOptional::print(std::ostream& out) const { out << "MatchOptional[default=" << default_[0] << "]"; } diff --git a/src/fdb5/rules/MatchOptional.h b/src/fdb5/rules/MatchOptional.h index 837d88e00..f3a36ff6a 100644 --- a/src/fdb5/rules/MatchOptional.h +++ b/src/fdb5/rules/MatchOptional.h @@ -25,11 +25,10 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -class MatchOptional : public Matcher{ +class MatchOptional : public Matcher { public: // methods - - MatchOptional(const std::string &def); + MatchOptional(const std::string& def); MatchOptional(eckit::Stream& s); ~MatchOptional() override; @@ -39,26 +38,24 @@ class MatchOptional : public Matcher{ void dump(std::ostream& s, const std::string& keyword, const TypesRegistry& registry) const override; const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } - static const eckit::ClassSpec& classSpec() { return classSpec_; } + static const eckit::ClassSpec& classSpec() { return classSpec_; } private: // methods - void encode(eckit::Stream&) const override; bool optional() const override; const std::string& value(const Key&, const std::string& keyword) const override; - const std::vector& values(const metkit::mars::MarsRequest& rq, const std::string& keyword) const override; - void print( std::ostream& out ) const override; + const std::vector& values(const metkit::mars::MarsRequest& rq, + const std::string& keyword) const override; + void print(std::ostream& out) const override; const std::string& defaultValue() const override; void fill(BaseKey& key, const std::string& keyword, const std::string& value) const override; private: // members - static eckit::ClassSpec classSpec_; static eckit::Reanimator reanimator_; std::vector default_; - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/rules/MatchValue.cc b/src/fdb5/rules/MatchValue.cc index 63a2b46bc..95ab7f751 100644 --- a/src/fdb5/rules/MatchValue.cc +++ b/src/fdb5/rules/MatchValue.cc @@ -8,52 +8,49 @@ * does it submit to any jurisdiction. */ -#include "fdb5/database/Key.h" #include "fdb5/rules/MatchValue.h" +#include "fdb5/database/Key.h" #include "fdb5/types/TypesRegistry.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -eckit::ClassSpec MatchValue::classSpec_ = { &Matcher::classSpec(), "MatchValue", }; +eckit::ClassSpec MatchValue::classSpec_ = { + &Matcher::classSpec(), + "MatchValue", +}; eckit::Reanimator MatchValue::reanimator_; +MatchValue::MatchValue(const std::string& value) : Matcher(), value_(value) {} -MatchValue::MatchValue(const std::string &value) : - Matcher(), - value_(value) { -} +MatchValue::MatchValue(eckit::Stream& s) : Matcher() { -MatchValue::MatchValue(eckit::Stream& s) : - Matcher() { - s >> value_; } void MatchValue::encode(eckit::Stream& s) const { s << value_; } -MatchValue::~MatchValue() { -} +MatchValue::~MatchValue() {} -bool MatchValue::match(const std::string &keyword, const Key& key) const { +bool MatchValue::match(const std::string& keyword, const Key& key) const { auto i = key.find(keyword); if (i == key.end()) { return false; } - return ( i->second == value_ ); + return (i->second == value_); } -void MatchValue::dump(std::ostream &s, const std::string &keyword, const TypesRegistry ®istry) const { +void MatchValue::dump(std::ostream& s, const std::string& keyword, const TypesRegistry& registry) const { registry.dump(s, keyword); s << "=" << value_; } -void MatchValue::print(std::ostream &out) const { +void MatchValue::print(std::ostream& out) const { out << "MatchValue[value=" << value_ << "]"; } diff --git a/src/fdb5/rules/MatchValue.h b/src/fdb5/rules/MatchValue.h index b8663f918..1cbe4840a 100644 --- a/src/fdb5/rules/MatchValue.h +++ b/src/fdb5/rules/MatchValue.h @@ -25,35 +25,31 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -class MatchValue : public Matcher{ +class MatchValue : public Matcher { public: // methods - - MatchValue(const std::string &value); + MatchValue(const std::string& value); MatchValue(eckit::Stream& s); ~MatchValue() override; - bool match(const std::string &keyword, const Key& key) const override; + bool match(const std::string& keyword, const Key& key) const override; - void dump(std::ostream &s, const std::string &keyword, const TypesRegistry ®istry) const override; + void dump(std::ostream& s, const std::string& keyword, const TypesRegistry& registry) const override; const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } - static const eckit::ClassSpec& classSpec() { return classSpec_; } + static const eckit::ClassSpec& classSpec() { return classSpec_; } private: // methods - void encode(eckit::Stream&) const override; - void print( std::ostream &out ) const override; + void print(std::ostream& out) const override; private: // members - static eckit::ClassSpec classSpec_; static eckit::Reanimator reanimator_; std::string value_; - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/rules/Matcher.cc b/src/fdb5/rules/Matcher.cc index 15a56ef76..0c43db753 100644 --- a/src/fdb5/rules/Matcher.cc +++ b/src/fdb5/rules/Matcher.cc @@ -12,49 +12,47 @@ #include "metkit/mars/MarsRequest.h" -#include "fdb5/rules/Matcher.h" #include "fdb5/database/Key.h" +#include "fdb5/rules/Matcher.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -eckit::ClassSpec Matcher::classSpec_ = { &eckit::Streamable::classSpec(), "Matcher", }; +eckit::ClassSpec Matcher::classSpec_ = { + &eckit::Streamable::classSpec(), + "Matcher", +}; -Matcher::Matcher() { -} +Matcher::Matcher() {} -Matcher::Matcher(eckit::Stream&) { -} +Matcher::Matcher(eckit::Stream&) {} -void Matcher::encode(eckit::Stream& s) const { -} +void Matcher::encode(eckit::Stream& s) const {} -Matcher::~Matcher() { -} +Matcher::~Matcher() {} bool Matcher::optional() const { return false; } -const std::string &Matcher::value(const Key& key, const std::string &keyword) const { +const std::string& Matcher::value(const Key& key, const std::string& keyword) const { return key.get(keyword); } -const std::vector &Matcher::values(const metkit::mars::MarsRequest& rq, const std::string &keyword) const { +const std::vector& Matcher::values(const metkit::mars::MarsRequest& rq, const std::string& keyword) const { return rq.values(keyword); } -void Matcher::fill(BaseKey& key, const std::string &keyword, const std::string& value) const { +void Matcher::fill(BaseKey& key, const std::string& keyword, const std::string& value) const { key.push(keyword, value); } - -const std::string &Matcher::defaultValue() const { +const std::string& Matcher::defaultValue() const { NOTIMP; } -std::ostream &operator<<(std::ostream &s, const Matcher &x) { +std::ostream& operator<<(std::ostream& s, const Matcher& x) { x.print(s); return s; } diff --git a/src/fdb5/rules/Matcher.h b/src/fdb5/rules/Matcher.h index 14b6f9627..d3ef74170 100644 --- a/src/fdb5/rules/Matcher.h +++ b/src/fdb5/rules/Matcher.h @@ -24,9 +24,9 @@ class MarsTask; namespace metkit { namespace mars { - class MarsRequest; -} +class MarsRequest; } +} // namespace metkit namespace fdb5 { @@ -39,7 +39,6 @@ class TypesRegistry; class Matcher : public eckit::Streamable { public: // methods - Matcher(); Matcher(eckit::Stream& s); @@ -47,31 +46,28 @@ class Matcher : public eckit::Streamable { virtual bool optional() const; - virtual const std::string &value(const Key& , const std::string &keyword) const; - virtual const std::vector& values(const metkit::mars::MarsRequest& rq, const std::string& keyword) const; - virtual const std::string &defaultValue() const; - - virtual bool match(const std::string &keyword, const Key& key) const = 0; - virtual void fill(BaseKey& key, const std::string &keyword, const std::string& value) const; + virtual const std::string& value(const Key&, const std::string& keyword) const; + virtual const std::vector& values(const metkit::mars::MarsRequest& rq, + const std::string& keyword) const; + virtual const std::string& defaultValue() const; + virtual bool match(const std::string& keyword, const Key& key) const = 0; + virtual void fill(BaseKey& key, const std::string& keyword, const std::string& value) const; - virtual void dump(std::ostream &s, const std::string &keyword, const TypesRegistry ®istry) const = 0; + virtual void dump(std::ostream& s, const std::string& keyword, const TypesRegistry& registry) const = 0; - friend std::ostream &operator<<(std::ostream &s, const Matcher &x); + friend std::ostream& operator<<(std::ostream& s, const Matcher& x); - static const eckit::ClassSpec& classSpec() { return classSpec_; } + static const eckit::ClassSpec& classSpec() { return classSpec_; } private: // methods - void encode(eckit::Stream&) const override; - virtual void print( std::ostream &out ) const = 0; + virtual void print(std::ostream& out) const = 0; private: // members - static eckit::ClassSpec classSpec_; static eckit::Reanimator reanimator_; - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/rules/Predicate.cc b/src/fdb5/rules/Predicate.cc index 6d7b08ad6..b7614f4cf 100644 --- a/src/fdb5/rules/Predicate.cc +++ b/src/fdb5/rules/Predicate.cc @@ -11,21 +11,21 @@ #include "metkit/mars/MarsRequest.h" #include "fdb5/database/Key.h" -#include "fdb5/rules/Predicate.h" #include "fdb5/rules/Matcher.h" +#include "fdb5/rules/Predicate.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -eckit::ClassSpec Predicate::classSpec_ = { &eckit::Streamable::classSpec(), "Predicate", }; +eckit::ClassSpec Predicate::classSpec_ = { + &eckit::Streamable::classSpec(), + "Predicate", +}; eckit::Reanimator Predicate::reanimator_; -Predicate::Predicate(const std::string &keyword, Matcher *matcher) : - matcher_(matcher), - keyword_(keyword) { -} +Predicate::Predicate(const std::string& keyword, Matcher* matcher) : matcher_(matcher), keyword_(keyword) {} Predicate::Predicate(eckit::Stream& s) { s >> keyword_; @@ -37,18 +37,17 @@ void Predicate::encode(eckit::Stream& s) const { s << *matcher_; } -Predicate::~Predicate() { -} +Predicate::~Predicate() {} bool Predicate::match(const Key& key) const { return matcher_->match(keyword_, key); } -void Predicate::dump(std::ostream &s, const TypesRegistry ®istry) const { +void Predicate::dump(std::ostream& s, const TypesRegistry& registry) const { matcher_->dump(s, keyword_, registry); } -void Predicate::print(std::ostream &out) const { +void Predicate::print(std::ostream& out) const { out << "Predicate[keyword=" << keyword_ << ",matcher=" << *matcher_ << "]"; } @@ -60,7 +59,7 @@ bool Predicate::optional() const { return matcher_->optional(); } -const std::string &Predicate::value(const Key& key) const { +const std::string& Predicate::value(const Key& key) const { return matcher_->value(key, keyword_); } @@ -72,16 +71,15 @@ void Predicate::fill(BaseKey& key, const std::string& value) const { matcher_->fill(key, keyword_, value); } -const std::string &Predicate::defaultValue() const { +const std::string& Predicate::defaultValue() const { return matcher_->defaultValue(); } -std::ostream &operator<<(std::ostream &s, const Predicate &x) { +std::ostream& operator<<(std::ostream& s, const Predicate& x) { x.print(s); return s; } - //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 diff --git a/src/fdb5/rules/Predicate.h b/src/fdb5/rules/Predicate.h index 84ae390e1..fbb58f797 100644 --- a/src/fdb5/rules/Predicate.h +++ b/src/fdb5/rules/Predicate.h @@ -17,13 +17,15 @@ #define fdb5_Predicate_H #include -#include #include +#include -#include "eckit/serialisation/Streamable.h" #include "eckit/serialisation/Reanimator.h" +#include "eckit/serialisation/Streamable.h" -namespace metkit { class MarsRequest; } +namespace metkit { +class MarsRequest; +} namespace fdb5 { @@ -37,38 +39,35 @@ class TypesRegistry; class Predicate : public eckit::Streamable { public: // methods - - Predicate(const std::string &keyword, Matcher *matcher); + Predicate(const std::string& keyword, Matcher* matcher); Predicate(eckit::Stream& s); ~Predicate(); bool match(const Key& key) const; - void dump( std::ostream &s, const TypesRegistry ®istry ) const; + void dump(std::ostream& s, const TypesRegistry& registry) const; void fill(BaseKey& key, const std::string& value) const; - const std::string &value(const Key& key) const; + const std::string& value(const Key& key) const; const std::vector& values(const metkit::mars::MarsRequest& rq) const; - const std::string &defaultValue() const; + const std::string& defaultValue() const; bool optional() const; std::string keyword() const; - const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } - static const eckit::ClassSpec& classSpec() { return classSpec_; } + const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } + static const eckit::ClassSpec& classSpec() { return classSpec_; } private: // methods - - friend std::ostream &operator<<(std::ostream &s, const Predicate &x); + friend std::ostream& operator<<(std::ostream& s, const Predicate& x); void encode(eckit::Stream& s) const override; - void print( std::ostream &out ) const; + void print(std::ostream& out) const; private: // members - static eckit::ClassSpec classSpec_; static eckit::Reanimator reanimator_; diff --git a/src/fdb5/rules/Rule.cc b/src/fdb5/rules/Rule.cc index c87baaa2f..e392e81ec 100644 --- a/src/fdb5/rules/Rule.cc +++ b/src/fdb5/rules/Rule.cc @@ -16,27 +16,28 @@ #include "metkit/mars/MarsRequest.h" -#include "fdb5/rules/Predicate.h" -#include "fdb5/rules/Schema.h" #include "fdb5/database/Key.h" #include "fdb5/database/ReadVisitor.h" #include "fdb5/database/WriteVisitor.h" +#include "fdb5/rules/Predicate.h" +#include "fdb5/rules/Schema.h" #include "fdb5/types/Type.h" - namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -eckit::ClassSpec Rule::classSpec_ = { &eckit::Streamable::classSpec(), "Rule", }; +eckit::ClassSpec Rule::classSpec_ = { + &eckit::Streamable::classSpec(), + "Rule", +}; eckit::Reanimator Rule::reanimator_; -Rule::Rule(const Schema &schema, - size_t line, - std::vector &predicates, std::vector &rules, - const std::map &types): - schema_(schema), line_(line) { +Rule::Rule(const Schema& schema, size_t line, std::vector& predicates, std::vector& rules, + const std::map& types) + : schema_(schema) + , line_(line) { std::swap(predicates, predicates_); std::swap(rules, rules_); for (std::map::const_iterator i = types.begin(); i != types.end(); ++i) { @@ -44,31 +45,29 @@ Rule::Rule(const Schema &schema, } } -Rule::Rule(eckit::Stream& s): - Rule(Schema(""), s) { +Rule::Rule(eckit::Stream& s) : Rule(Schema(""), s) { NOTIMP; } -Rule::Rule(const Schema &schema, eckit::Stream& s): - schema_(schema), registry_(s) { +Rule::Rule(const Schema& schema, eckit::Stream& s) : schema_(schema), registry_(s) { size_t numPredicates; size_t numRules; s >> line_; s >> numPredicates; - for (size_t i=0; i < numPredicates; i++) { + for (size_t i = 0; i < numPredicates; i++) { predicates_.push_back(eckit::Reanimator::reanimate(s)); } s >> numRules; - for (size_t i=0; i < numRules; i++) { + for (size_t i = 0; i < numRules; i++) { rules_.push_back(new Rule(schema, s)); } } void Rule::encode(eckit::Stream& s) const { - + registry_.encode(s); s << line_; @@ -83,23 +82,19 @@ void Rule::encode(eckit::Stream& s) const { } Rule::~Rule() { - for (std::vector::iterator i = predicates_.begin(); i != predicates_.end(); ++i ) { + for (std::vector::iterator i = predicates_.begin(); i != predicates_.end(); ++i) { delete *i; } - for (std::vector::iterator i = rules_.begin(); i != rules_.end(); ++i ) { + for (std::vector::iterator i = rules_.begin(); i != rules_.end(); ++i) { delete *i; } } -void Rule::expand( const metkit::mars::MarsRequest &request, - std::vector::const_iterator cur, - size_t depth, - std::vector &keys, - TypedKey& fullComputedKey, - ReadVisitor &visitor) const { +void Rule::expand(const metkit::mars::MarsRequest& request, std::vector::const_iterator cur, size_t depth, + std::vector& keys, TypedKey& fullComputedKey, ReadVisitor& visitor) const { - ASSERT(depth < 3); + ASSERT(depth < 3); if (cur == predicates_.end()) { @@ -108,44 +103,44 @@ void Rule::expand( const metkit::mars::MarsRequest &request, // TODO: join these 2 methods if (rules_.empty()) { ASSERT(depth == 2); /// we have 3 levels ATM - if (!visitor.selectDatum( keys[2], fullComputedKey)) { + if (!visitor.selectDatum(keys[2], fullComputedKey)) { return; // This it not useful } } else { switch (depth) { - case 0: - if (!visitor.selectDatabase(keys[0].canonical(), fullComputedKey)) { + case 0: + if (!visitor.selectDatabase(keys[0].canonical(), fullComputedKey)) { + return; + }; + + // Here we recurse on the database's schema (rather than the master schema) + ASSERT(keys[0] == fullComputedKey); + visitor.databaseSchema().expandSecond(request, visitor, keys[0].canonical()); return; - }; - - // Here we recurse on the database's schema (rather than the master schema) - ASSERT(keys[0] == fullComputedKey); - visitor.databaseSchema().expandSecond(request, visitor, keys[0].canonical()); - return; - case 1: - if (!visitor.selectIndex(keys[1].canonical(), fullComputedKey)) { - return; - } - break; + case 1: + if (!visitor.selectIndex(keys[1].canonical(), fullComputedKey)) { + return; + } + break; - default: - ASSERT(depth == 0 || depth == 1); - break; + default: + ASSERT(depth == 0 || depth == 1); + break; } - for (std::vector::const_iterator i = rules_.begin(); i != rules_.end(); ++i ) { + for (std::vector::const_iterator i = rules_.begin(); i != rules_.end(); ++i) { (*i)->expand(request, visitor, depth + 1, keys, fullComputedKey); } } return; } - std::vector::const_iterator next = cur; + std::vector::const_iterator next = cur; ++next; - const std::string &keyword = (*cur)->keyword(); + const std::string& keyword = (*cur)->keyword(); eckit::StringList values; visitor.values(request, keyword, registry_, values); @@ -168,22 +163,17 @@ void Rule::expand( const metkit::mars::MarsRequest &request, fullComputedKey.pop(keyword); k.pop(keyword); - } - } -void Rule::expand(const metkit::mars::MarsRequest &request, ReadVisitor &visitor, size_t depth, std::vector &keys, TypedKey& fullComputedKey) const { +void Rule::expand(const metkit::mars::MarsRequest& request, ReadVisitor& visitor, size_t depth, + std::vector& keys, TypedKey& fullComputedKey) const { ASSERT(keys.size() == 3); expand(request, predicates_.begin(), depth, keys, fullComputedKey, visitor); } -void Rule::expand( const Key& initialFieldKey, - std::vector::const_iterator cur, - size_t depth, - std::vector &keys, - TypedKey& fullComputedKey, - WriteVisitor &visitor) const { +void Rule::expand(const Key& initialFieldKey, std::vector::const_iterator cur, size_t depth, + std::vector& keys, TypedKey& fullComputedKey, WriteVisitor& visitor) const { static bool matchFirstFdbRule = eckit::Resource("matchFirstFdbRule", true); @@ -201,54 +191,50 @@ void Rule::expand( const Key& initialFieldKey, ASSERT(depth == 2); /// we have 3 levels ATM if (visitor.rule() != 0) { std::ostringstream oss; - oss << "More than one rule matching " - << keys[0] << ", " - << keys[1] << ", " - << keys[2] << " " - << topRule() << " and " - << visitor.rule()->topRule(); + oss << "More than one rule matching " << keys[0] << ", " << keys[1] << ", " << keys[2] << " " + << topRule() << " and " << visitor.rule()->topRule(); throw eckit::SeriousBug(oss.str()); } visitor.rule(this); - visitor.selectDatum( keys[2], fullComputedKey); + visitor.selectDatum(keys[2], fullComputedKey); } else { switch (depth) { - case 0: - if (keys[0] != visitor.prev_[0] /*|| keys[0].registry() != visitor.prev_[0].registry()*/) { - visitor.selectDatabase(keys[0].canonical(), fullComputedKey); - visitor.prev_[0] = keys[0].canonical(); - visitor.prev_[1] = Key{}; - } - - // Here we recurse on the database's schema (rather than the master schema) - visitor.databaseSchema().expandSecond(initialFieldKey, visitor, keys[0].canonical()); - return; - - case 1: - if (keys[1] != visitor.prev_[1] /*|| keys[1].registry() != visitor.prev_[1].registry()*/) { - visitor.selectIndex(keys[1].canonical(), fullComputedKey); - visitor.prev_[1] = keys[1].canonical(); - } - break; - - default: - ASSERT(depth == 0 || depth == 1); - break; + case 0: + if (keys[0] != visitor.prev_[0] /*|| keys[0].registry() != visitor.prev_[0].registry()*/) { + visitor.selectDatabase(keys[0].canonical(), fullComputedKey); + visitor.prev_[0] = keys[0].canonical(); + visitor.prev_[1] = Key{}; + } + + // Here we recurse on the database's schema (rather than the master schema) + visitor.databaseSchema().expandSecond(initialFieldKey, visitor, keys[0].canonical()); + return; + + case 1: + if (keys[1] != visitor.prev_[1] /*|| keys[1].registry() != visitor.prev_[1].registry()*/) { + visitor.selectIndex(keys[1].canonical(), fullComputedKey); + visitor.prev_[1] = keys[1].canonical(); + } + break; + + default: + ASSERT(depth == 0 || depth == 1); + break; } - for (std::vector::const_iterator i = rules_.begin(); i != rules_.end(); ++i ) { + for (std::vector::const_iterator i = rules_.begin(); i != rules_.end(); ++i) { (*i)->expand(initialFieldKey, visitor, depth + 1, keys, fullComputedKey); } } return; } - std::vector::const_iterator next = cur; + std::vector::const_iterator next = cur; ++next; - const std::string &keyword = (*cur)->keyword(); - const std::string &value = (*cur)->value(initialFieldKey); + const std::string& keyword = (*cur)->keyword(); + const std::string& value = (*cur)->value(initialFieldKey); TypedKey& k = keys[depth]; k.push(keyword, value); @@ -261,20 +247,22 @@ void Rule::expand( const Key& initialFieldKey, fullComputedKey.pop(keyword); k.pop(keyword); } -void Rule::expand(const Key& initialFieldKey, WriteVisitor &visitor, size_t depth, std::vector &keys, TypedKey& fullComputedKey) const { +void Rule::expand(const Key& initialFieldKey, WriteVisitor& visitor, size_t depth, std::vector& keys, + TypedKey& fullComputedKey) const { ASSERT(keys.size() == 3); expand(initialFieldKey, predicates_.begin(), depth, keys, fullComputedKey, visitor); } -void Rule::expandFirstLevel(const metkit::mars::MarsRequest& rq, std::vector::const_iterator cur, TypedKey& result, bool& found) const { +void Rule::expandFirstLevel(const metkit::mars::MarsRequest& rq, std::vector::const_iterator cur, + TypedKey& result, bool& found) const { if (cur == predicates_.end()) { found = true; return; } - std::vector::const_iterator next = cur; + std::vector::const_iterator next = cur; ++next; const std::string& keyword = (*cur)->keyword(); @@ -303,8 +291,8 @@ void Rule::expandFirstLevel(const metkit::mars::MarsRequest& request, TypedKey& expandFirstLevel(request, predicates_.begin(), result, done); } - -void Rule::matchFirstLevel( const Key& dbKey, std::vector::const_iterator cur, Key& tmp, std::set& result, const char* missing) const { +void Rule::matchFirstLevel(const Key& dbKey, std::vector::const_iterator cur, Key& tmp, + std::set& result, const char* missing) const { if (cur == predicates_.end()) { if (tmp.match(dbKey)) { @@ -313,16 +301,16 @@ void Rule::matchFirstLevel( const Key& dbKey, std::vector::const_it return; } - std::vector::const_iterator next = cur; + std::vector::const_iterator next = cur; ++next; - const std::string &keyword = (*cur)->keyword(); + const std::string& keyword = (*cur)->keyword(); if (dbKey.find(keyword) == dbKey.end()) { tmp.push(keyword, missing); matchFirstLevel(dbKey, next, tmp, result, missing); } else { - const std::string &value = (*cur)->value(dbKey); + const std::string& value = (*cur)->value(dbKey); tmp.push(keyword, value); @@ -332,25 +320,24 @@ void Rule::matchFirstLevel( const Key& dbKey, std::vector::const_it } tmp.pop(keyword); - } -void Rule::matchFirstLevel(const Key& dbKey, std::set& result, const char* missing) const { +void Rule::matchFirstLevel(const Key& dbKey, std::set& result, const char* missing) const { Key tmp{}; matchFirstLevel(dbKey, predicates_.begin(), tmp, result, missing); } - -void Rule::matchFirstLevel(const metkit::mars::MarsRequest& request, std::vector::const_iterator cur, Key& tmp, std::set& result, const char* missing) const { +void Rule::matchFirstLevel(const metkit::mars::MarsRequest& request, std::vector::const_iterator cur, + Key& tmp, std::set& result, const char* missing) const { if (cur == predicates_.end()) { -// if (tmp.match(request)) { - result.insert(tmp); -// } + // if (tmp.match(request)) { + result.insert(tmp); + // } return; } - std::vector::const_iterator next = cur; + std::vector::const_iterator next = cur; ++next; const std::string& keyword = (*cur)->keyword(); @@ -373,14 +360,13 @@ void Rule::matchFirstLevel(const metkit::mars::MarsRequest& request, std::vector } } -void Rule::matchFirstLevel(const metkit::mars::MarsRequest& request, std::set& result, const char* missing) const { +void Rule::matchFirstLevel(const metkit::mars::MarsRequest& request, std::set& result, const char* missing) const { Key tmp{}; matchFirstLevel(request, predicates_.begin(), tmp, result, missing); } - bool Rule::match(const Key& key) const { - for (std::vector::const_iterator i = predicates_.begin(); i != predicates_.end(); ++i ) { + for (std::vector::const_iterator i = predicates_.begin(); i != predicates_.end(); ++i) { if (!(*i)->match(key)) { return false; } @@ -389,7 +375,7 @@ bool Rule::match(const Key& key) const { } // Find the first rule that matches a list of keys -const Rule* Rule::ruleFor(const std::vector &keys, size_t depth) const { +const Rule* Rule::ruleFor(const std::vector& keys, size_t depth) const { if (depth == keys.size()) { return this; @@ -397,8 +383,8 @@ const Rule* Rule::ruleFor(const std::vector &keys, size_t depth) cons if (match(keys[depth])) { - for (std::vector::const_iterator i = rules_.begin(); i != rules_.end(); ++i ) { - const Rule *r = (*i)->ruleFor(keys, depth + 1); + for (std::vector::const_iterator i = rules_.begin(); i != rules_.end(); ++i) { + const Rule* r = (*i)->ruleFor(keys, depth + 1); if (r) { return r; } @@ -450,16 +436,16 @@ void Rule::fill(BaseKey& key, const eckit::StringList& values) const { ASSERT(it_pred == predicates_.end()); } -void Rule::dump(std::ostream &s, size_t depth) const { +void Rule::dump(std::ostream& s, size_t depth) const { s << "["; - const char *sep = ""; - for (std::vector::const_iterator i = predicates_.begin(); i != predicates_.end(); ++i ) { + const char* sep = ""; + for (std::vector::const_iterator i = predicates_.begin(); i != predicates_.end(); ++i) { s << sep; (*i)->dump(s, registry_); sep = ","; } - for (std::vector::const_iterator i = rules_.begin(); i != rules_.end(); ++i ) { + for (std::vector::const_iterator i = rules_.begin(); i != rules_.end(); ++i) { (*i)->dump(s, depth + 1); } s << "]"; @@ -467,21 +453,21 @@ void Rule::dump(std::ostream &s, size_t depth) const { size_t Rule::depth() const { size_t result = 0; - for (std::vector::const_iterator i = rules_.begin(); i != rules_.end(); ++i ) { + for (std::vector::const_iterator i = rules_.begin(); i != rules_.end(); ++i) { result = std::max(result, (*i)->depth()); } return result + 1; } -void Rule::updateParent(const Rule *parent) { +void Rule::updateParent(const Rule* parent) { parent_ = parent; // if (parent && (®istry_ != &parent->registry_)) { if (parent) { registry_.updateParent(parent_->registry_); } - for (std::vector::iterator i = rules_.begin(); i != rules_.end(); ++i ) { + for (std::vector::iterator i = rules_.begin(); i != rules_.end(); ++i) { // if (&(*i)->registry_ != ®istry_) - (*i)->updateParent(this); + (*i)->updateParent(this); } } @@ -489,12 +475,12 @@ const TypesRegistry& Rule::registry() const { return registry_; } -void Rule::print(std::ostream &out) const { - out << "Rule[line=" << line_ ; +void Rule::print(std::ostream& out) const { + out << "Rule[line=" << line_; out << "]"; } -const Rule &Rule::topRule() const { +const Rule& Rule::topRule() const { if (parent_) { return parent_->topRule(); } else { @@ -502,12 +488,12 @@ const Rule &Rule::topRule() const { } } -const Schema &Rule::schema() const { +const Schema& Rule::schema() const { return schema_; } void Rule::check(const Key& key) const { - for (const auto& pred : predicates_ ) { + for (const auto& pred : predicates_) { auto k = key.find(pred->keyword()); if (k != key.end()) { const std::string& value = (*k).second; @@ -525,7 +511,7 @@ void Rule::check(const Key& key) const { } } -std::ostream &operator<<(std::ostream &s, const Rule &x) { +std::ostream& operator<<(std::ostream& s, const Rule& x) { x.print(s); return s; } diff --git a/src/fdb5/rules/Rule.h b/src/fdb5/rules/Rule.h index fc22c53cd..c3f5efcf8 100644 --- a/src/fdb5/rules/Rule.h +++ b/src/fdb5/rules/Rule.h @@ -17,19 +17,19 @@ #define fdb5_Rule_H #include -#include #include +#include +#include "eckit/serialisation/Reanimator.h" #include "eckit/serialisation/Streamable.h" #include "eckit/types/Types.h" #include "fdb5/types/TypesRegistry.h" -#include "eckit/serialisation/Reanimator.h" namespace metkit { namespace mars { - class MarsRequest; -} +class MarsRequest; } +} // namespace metkit namespace fdb5 { @@ -46,16 +46,11 @@ class TypedKey; class Rule : public eckit::Streamable { public: // methods - /// Takes ownership of vectors - Rule(const Schema &schema, - size_t line, - std::vector &predicates, - std::vector &rules, - const std::map &types - ); + Rule(const Schema& schema, size_t line, std::vector& predicates, std::vector& rules, + const std::map& types); Rule(eckit::Stream& s); - Rule(const Schema &schema, eckit::Stream& s); + Rule(const Schema& schema, eckit::Stream& s); ~Rule(); @@ -63,86 +58,70 @@ class Rule : public eckit::Streamable { eckit::StringList keys(size_t level) const; - void dump(std::ostream &s, size_t depth = 0) const; + void dump(std::ostream& s, size_t depth = 0) const; - void expand(const metkit::mars::MarsRequest &request, - ReadVisitor &Visitor, - size_t depth, - std::vector &keys, - TypedKey& fullComputedKey) const; + void expand(const metkit::mars::MarsRequest& request, ReadVisitor& Visitor, size_t depth, + std::vector& keys, TypedKey& fullComputedKey) const; - void expand(const Key& initialFieldKey, - WriteVisitor &Visitor, - size_t depth, - std::vector &keys, + void expand(const Key& initialFieldKey, WriteVisitor& Visitor, size_t depth, std::vector& keys, TypedKey& fullComputedKey) const; - const Rule* ruleFor(const std::vector &keys, size_t depth) const; + const Rule* ruleFor(const std::vector& keys, size_t depth) const; void fill(BaseKey& key, const eckit::StringList& values) const; - size_t depth() const; - void updateParent(const Rule *parent); + void updateParent(const Rule* parent); - const Rule &topRule() const; + const Rule& topRule() const; - const Schema &schema() const; + const Schema& schema() const; const TypesRegistry& registry() const; const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } - static const eckit::ClassSpec& classSpec() { return classSpec_; } + static const eckit::ClassSpec& classSpec() { return classSpec_; } void check(const Key& key) const; private: // methods + void expand(const metkit::mars::MarsRequest& request, std::vector::const_iterator cur, size_t depth, + std::vector& keys, TypedKey& fullComputedKey, ReadVisitor& Visitor) const; - void expand(const metkit::mars::MarsRequest &request, - std::vector::const_iterator cur, - size_t depth, - std::vector &keys, - TypedKey& fullComputedKey, - ReadVisitor &Visitor) const; - - void expand(const Key& initialFieldKey, - std::vector::const_iterator cur, - size_t depth, - std::vector &keys, - TypedKey& fullComputedKey, - WriteVisitor &Visitor) const; + void expand(const Key& initialFieldKey, std::vector::const_iterator cur, size_t depth, + std::vector& keys, TypedKey& fullComputedKey, WriteVisitor& Visitor) const; - void expandFirstLevel(const metkit::mars::MarsRequest& request, std::vector::const_iterator cur, TypedKey& result, bool& done) const; - void expandFirstLevel(const metkit::mars::MarsRequest& request, TypedKey& result, bool& done) const; + void expandFirstLevel(const metkit::mars::MarsRequest& request, std::vector::const_iterator cur, + TypedKey& result, bool& done) const; + void expandFirstLevel(const metkit::mars::MarsRequest& request, TypedKey& result, bool& done) const; - void matchFirstLevel(const Key& dbKey, std::vector::const_iterator cur, Key& tmp, std::set& result, const char* missing) const; - void matchFirstLevel(const Key& dbKey, std::set& result, const char* missing) const ; - void matchFirstLevel(const metkit::mars::MarsRequest& request, std::vector::const_iterator cur, Key& tmp, std::set& result, const char* missing) const; - void matchFirstLevel(const metkit::mars::MarsRequest& request, std::set& result, const char* missing) const ; + void matchFirstLevel(const Key& dbKey, std::vector::const_iterator cur, Key& tmp, std::set& result, + const char* missing) const; + void matchFirstLevel(const Key& dbKey, std::set& result, const char* missing) const; + void matchFirstLevel(const metkit::mars::MarsRequest& request, std::vector::const_iterator cur, + Key& tmp, std::set& result, const char* missing) const; + void matchFirstLevel(const metkit::mars::MarsRequest& request, std::set& result, const char* missing) const; + void keys(size_t level, size_t depth, eckit::StringList& result, eckit::StringSet& seen) const; - void keys(size_t level, size_t depth, eckit::StringList &result, eckit::StringSet &seen) const; - - friend std::ostream &operator<<(std::ostream &s, const Rule &x); + friend std::ostream& operator<<(std::ostream& s, const Rule& x); void encode(eckit::Stream& s) const override; - void print( std::ostream &out ) const; + void print(std::ostream& out) const; private: // members - static eckit::ClassSpec classSpec_; static eckit::Reanimator reanimator_; const Schema& schema_; const Rule* parent_; - std::vector predicates_; - std::vector rules_; + std::vector predicates_; + std::vector rules_; TypesRegistry registry_; friend class Schema; size_t line_; - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/rules/Schema.cc b/src/fdb5/rules/Schema.cc index a2dbe9a6f..90037209f 100644 --- a/src/fdb5/rules/Schema.cc +++ b/src/fdb5/rules/Schema.cc @@ -25,26 +25,28 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -eckit::ClassSpec Schema::classSpec_ = { &eckit::Streamable::classSpec(), "Schema", }; +eckit::ClassSpec Schema::classSpec_ = { + &eckit::Streamable::classSpec(), + "Schema", +}; eckit::Reanimator Schema::reanimator_; Schema::Schema() = default; -Schema::Schema(const eckit::PathName &path) { +Schema::Schema(const eckit::PathName& path) { load(path); } Schema::Schema(std::istream& s) { load(s); } -Schema::Schema(eckit::Stream& s) : - registry_(s) { +Schema::Schema(eckit::Stream& s) : registry_(s) { size_t numRules; s >> path_; s >> numRules; - for (size_t i=0; i < numRules; i++) { + for (size_t i = 0; i < numRules; i++) { rules_.push_back(new Rule(*this, s)); } @@ -64,13 +66,13 @@ Schema::~Schema() { clear(); } -const Rule* Schema::ruleFor(const Key& dbKey, const Key& idxKey) const { +const Rule* Schema::ruleFor(const Key& dbKey, const Key& idxKey) const { std::vector keys; keys.push_back(dbKey); keys.push_back(idxKey); for (const Rule* rule : rules_) { - const Rule* r = rule->ruleFor(keys , 0); + const Rule* r = rule->ruleFor(keys, 0); if (r) { return r; } @@ -78,16 +80,16 @@ const Rule* Schema::ruleFor(const Key& dbKey, const Key& idxKey) const { return 0; } -void Schema::expand(const metkit::mars::MarsRequest &request, ReadVisitor &visitor) const { +void Schema::expand(const metkit::mars::MarsRequest& request, ReadVisitor& visitor) const { TypedKey fullComputedKey{registry()}; std::vector keys(3, TypedKey{{}, registry()}); for (Rule* r : rules_) { - r->expand(request, visitor, 0, keys, fullComputedKey); + r->expand(request, visitor, 0, keys, fullComputedKey); } } -void Schema::expand(const Key& field, WriteVisitor &visitor) const { +void Schema::expand(const Key& field, WriteVisitor& visitor) const { TypedKey fullComputedKey{registry()}; std::vector keys(3, TypedKey{{}, registry()}); @@ -112,7 +114,7 @@ void Schema::expandSecond(const metkit::mars::MarsRequest& request, ReadVisitor& std::vector keys(3, TypedKey{{}, registry()}); TypedKey fullComputedKey = keys[0] = TypedKey{dbKey, registry()}; - for (std::vector:: const_iterator i = dbRule->rules_.begin(); i != dbRule->rules_.end(); ++i) { + for (std::vector::const_iterator i = dbRule->rules_.begin(); i != dbRule->rules_.end(); ++i) { (*i)->expand(request, visitor, 1, keys, fullComputedKey); } } @@ -131,7 +133,7 @@ void Schema::expandSecond(const Key& field, WriteVisitor& visitor, const Key& db std::vector keys(3, TypedKey{{}, registry()}); TypedKey fullComputedKey = keys[0] = TypedKey{dbKey, registry()}; - for (std::vector:: const_iterator i = dbRule->rules_.begin(); i != dbRule->rules_.end(); ++i) { + for (std::vector::const_iterator i = dbRule->rules_.begin(); i != dbRule->rules_.end(); ++i) { (*i)->expand(field, visitor, 1, keys, fullComputedKey); } } @@ -148,19 +150,20 @@ bool Schema::expandFirstLevel(const metkit::mars::MarsRequest& request, TypedKey return found; } -void Schema::matchFirstLevel(const Key& dbKey, std::set &result, const char* missing) const { +void Schema::matchFirstLevel(const Key& dbKey, std::set& result, const char* missing) const { for (const Rule* rule : rules_) { rule->matchFirstLevel(dbKey, result, missing); } } -void Schema::matchFirstLevel(const metkit::mars::MarsRequest& request, std::set& result, const char* missing) const { +void Schema::matchFirstLevel(const metkit::mars::MarsRequest& request, std::set& result, + const char* missing) const { for (const Rule* rule : rules_) { rule->matchFirstLevel(request, result, missing); } } -void Schema::load(const eckit::PathName &path, bool replace) { +void Schema::load(const eckit::PathName& path, bool replace) { path_ = path; @@ -189,14 +192,14 @@ void Schema::load(std::istream& s, bool replace) { } void Schema::clear() { - for (std::vector::iterator i = rules_.begin(); i != rules_.end(); ++i ) { + for (std::vector::iterator i = rules_.begin(); i != rules_.end(); ++i) { delete *i; } } -void Schema::dump(std::ostream &s) const { +void Schema::dump(std::ostream& s) const { registry_.dump(s); - for (std::vector::const_iterator i = rules_.begin(); i != rules_.end(); ++i ) { + for (std::vector::const_iterator i = rules_.begin(); i != rules_.end(); ++i) { (*i)->dump(s); s << std::endl; } @@ -211,20 +214,19 @@ void Schema::check() { } } -void Schema::print(std::ostream &out) const { +void Schema::print(std::ostream& out) const { out << "Schema[path=" << path_ << "]"; } -const Type &Schema::lookupType(const std::string &keyword) const { +const Type& Schema::lookupType(const std::string& keyword) const { return registry_.lookupType(keyword); } - bool Schema::empty() const { return rules_.empty(); } -const std::string &Schema::path() const { +const std::string& Schema::path() const { return path_; } @@ -232,7 +234,7 @@ const TypesRegistry& Schema::registry() const { return registry_; } -std::ostream &operator<<(std::ostream &s, const Schema &x) { +std::ostream& operator<<(std::ostream& s, const Schema& x) { x.print(s); return s; } diff --git a/src/fdb5/rules/Schema.h b/src/fdb5/rules/Schema.h index db56cc946..5cc2acc3c 100644 --- a/src/fdb5/rules/Schema.h +++ b/src/fdb5/rules/Schema.h @@ -25,8 +25,8 @@ #include #include "eckit/filesystem/PathName.h" -#include "eckit/serialisation/Streamable.h" #include "eckit/serialisation/Reanimator.h" +#include "eckit/serialisation/Streamable.h" #include "fdb5/config/Config.h" #include "fdb5/types/TypesRegistry.h" @@ -48,68 +48,64 @@ class Schema; class Schema : public eckit::Streamable { public: // methods - Schema(); - Schema(const eckit::PathName &path); + Schema(const eckit::PathName& path); Schema(std::istream& s); Schema(eckit::Stream& s); ~Schema(); - void expand(const Key& field, WriteVisitor &visitor) const; - void expand(const metkit::mars::MarsRequest &request, ReadVisitor &visitor) const; + void expand(const Key& field, WriteVisitor& visitor) const; + void expand(const metkit::mars::MarsRequest& request, ReadVisitor& visitor) const; // Each database has its own internal schema. So expand() above results in // expandFurther being called on the relevant schema from the DB, to start // iterating on that schemas rules. - void expandSecond(const Key& field, WriteVisitor &visitor, const Key& dbKey) const; - void expandSecond(const metkit::mars::MarsRequest& request, ReadVisitor &visitor, const Key& dbKey) const; + void expandSecond(const Key& field, WriteVisitor& visitor, const Key& dbKey) const; + void expandSecond(const metkit::mars::MarsRequest& request, ReadVisitor& visitor, const Key& dbKey) const; - bool expandFirstLevel(const metkit::mars::MarsRequest& request, TypedKey& result) const ; - void matchFirstLevel(const Key& dbKey, std::set &result, const char* missing) const ; - void matchFirstLevel(const metkit::mars::MarsRequest& request, std::set& result, const char* missing) const ; + bool expandFirstLevel(const metkit::mars::MarsRequest& request, TypedKey& result) const; + void matchFirstLevel(const Key& dbKey, std::set& result, const char* missing) const; + void matchFirstLevel(const metkit::mars::MarsRequest& request, std::set& result, const char* missing) const; const Rule* ruleFor(const Key& dbKey, const Key& idxKey) const; - void load(const eckit::PathName &path, bool replace = false); + void load(const eckit::PathName& path, bool replace = false); void load(std::istream& s, bool replace = false); - void dump(std::ostream &s) const; + void dump(std::ostream& s) const; bool empty() const; - const Type &lookupType(const std::string &keyword) const; + const Type& lookupType(const std::string& keyword) const; - const std::string &path() const; + const std::string& path() const; const TypesRegistry& registry() const; - + const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } - static const eckit::ClassSpec& classSpec() { return classSpec_; } + static const eckit::ClassSpec& classSpec() { return classSpec_; } private: // methods - void clear(); void check(); - friend std::ostream &operator<<(std::ostream &s, const Schema &x); + friend std::ostream& operator<<(std::ostream& s, const Schema& x); void encode(eckit::Stream& s) const override; - void print( std::ostream &out ) const; + void print(std::ostream& out) const; private: // members - static eckit::ClassSpec classSpec_; static eckit::Reanimator reanimator_; friend void Config::overrideSchema(const eckit::PathName& schemaPath, Schema* schema); TypesRegistry registry_; - - std::vector rules_; - std::string path_; + std::vector rules_; + std::string path_; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/rules/SchemaParser.cc b/src/fdb5/rules/SchemaParser.cc index b15bf72ae..eaf5c26fa 100644 --- a/src/fdb5/rules/SchemaParser.cc +++ b/src/fdb5/rules/SchemaParser.cc @@ -13,15 +13,14 @@ /// @author Tiago Quintino /// @date April 2016 - #include "fdb5/rules/SchemaParser.h" -#include "fdb5/rules/Rule.h" -#include "fdb5/rules/Predicate.h" #include "fdb5/rules/MatchAlways.h" #include "fdb5/rules/MatchAny.h" -#include "fdb5/rules/MatchValue.h" -#include "fdb5/rules/MatchOptional.h" #include "fdb5/rules/MatchHidden.h" +#include "fdb5/rules/MatchOptional.h" +#include "fdb5/rules/MatchValue.h" +#include "fdb5/rules/Predicate.h" +#include "fdb5/rules/Rule.h" #include "fdb5/types/TypesRegistry.h" namespace fdb5 { @@ -33,36 +32,36 @@ std::string SchemaParser::parseIdent(bool value, bool emptyOK) { for (;;) { char c = peek(); switch (c) { - case 0: - case '/': - case '=': - case ',': - case ';': - case ':': - case '[': - case ']': - case '?': - if (s.empty() && !emptyOK) { - throw StreamParser::Error("Syntax error: found '" + std::to_string(c) + "'", line_ + 1); - } - return s; - case '-': - if (s.empty() && !emptyOK) { - throw StreamParser::Error("Syntax error: found '-'", line_ + 1); - } - if (!value) { + case 0: + case '/': + case '=': + case ',': + case ';': + case ':': + case '[': + case ']': + case '?': + if (s.empty() && !emptyOK) { + throw StreamParser::Error("Syntax error: found '" + std::to_string(c) + "'", line_ + 1); + } return s; - } - - default: - consume(c); - s += c; - break; + case '-': + if (s.empty() && !emptyOK) { + throw StreamParser::Error("Syntax error: found '-'", line_ + 1); + } + if (!value) { + return s; + } + + default: + consume(c); + s += c; + break; } } } -Predicate *SchemaParser::parsePredicate(std::map &types) { +Predicate* SchemaParser::parsePredicate(std::map& types) { std::set values; std::string k = parseIdent(false, false); @@ -102,21 +101,21 @@ Predicate *SchemaParser::parsePredicate(std::map &type } switch (values.size()) { - case 0: - return new Predicate(k, new MatchAlways()); - break; + case 0: + return new Predicate(k, new MatchAlways()); + break; - case 1: - return new Predicate(k, new MatchValue(*values.begin())); - break; + case 1: + return new Predicate(k, new MatchValue(*values.begin())); + break; - default: - return new Predicate(k, new MatchAny(values)); - break; + default: + return new Predicate(k, new MatchAny(values)); + break; } } -void SchemaParser::parseTypes(std::map &types) { +void SchemaParser::parseTypes(std::map& types) { for (;;) { std::string name = parseIdent(false, true); if (name.empty()) { @@ -130,9 +129,9 @@ void SchemaParser::parseTypes(std::map &types) { } } -Rule *SchemaParser::parseRule(const Schema &owner) { - std::vector predicates; - std::vector rules; +Rule* SchemaParser::parseRule(const Schema& owner) { + std::vector predicates; + std::vector rules; std::map types; consume('['); @@ -145,19 +144,18 @@ Rule *SchemaParser::parseRule(const Schema &owner) { return new Rule(owner, line, predicates, rules, types); } - for (;;) { char c = peek(); - if ( c == '[') { - while ( c == '[') { + if (c == '[') { + while (c == '[') { rules.push_back(parseRule(owner)); c = peek(); } } else { predicates.push_back(parsePredicate(types)); - while ( (c = peek()) == ',') { + while ((c = peek()) == ',') { consume(c); predicates.push_back(parsePredicate(types)); } @@ -168,16 +166,12 @@ Rule *SchemaParser::parseRule(const Schema &owner) { consume(c); return new Rule(owner, line, predicates, rules, types); } - - } } -SchemaParser::SchemaParser(std::istream &in) : StreamParser(in, true) { -} +SchemaParser::SchemaParser(std::istream& in) : StreamParser(in, true) {} -void SchemaParser::parse(const Schema &owner, - std::vector &result, TypesRegistry ®istry) { +void SchemaParser::parse(const Schema& owner, std::vector& result, TypesRegistry& registry) { char c; std::map types; @@ -194,7 +188,6 @@ void SchemaParser::parse(const Schema &owner, } } - //---------------------------------------------------------------------------------------------------------------------- -} // namespace eckit +} // namespace fdb5 diff --git a/src/fdb5/rules/SchemaParser.h b/src/fdb5/rules/SchemaParser.h index cae777316..29377e9fd 100644 --- a/src/fdb5/rules/SchemaParser.h +++ b/src/fdb5/rules/SchemaParser.h @@ -29,24 +29,21 @@ class TypesRegistry; class SchemaParser : public eckit::StreamParser { public: // methods + SchemaParser(std::istream& in); - SchemaParser(std::istream &in); - - void parse(const Schema &owner, std::vector &, TypesRegistry ®istry); + void parse(const Schema& owner, std::vector&, TypesRegistry& registry); private: // methods - std::string parseIdent(bool value, bool emptyOK); - Rule *parseRule(const Schema &owner); - - Predicate *parsePredicate(std::map &types); - void parseTypes(std::map &); + Rule* parseRule(const Schema& owner); + Predicate* parsePredicate(std::map& types); + void parseTypes(std::map&); }; //---------------------------------------------------------------------------------------------------------------------- -} // namespace eckit +} // namespace fdb5 #endif diff --git a/src/fdb5/toc/AdoptVisitor.cc b/src/fdb5/toc/AdoptVisitor.cc index 386da2f19..dec64cdac 100644 --- a/src/fdb5/toc/AdoptVisitor.cc +++ b/src/fdb5/toc/AdoptVisitor.cc @@ -21,11 +21,12 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -AdoptVisitor::AdoptVisitor(Archiver& owner, const Key& initialFieldKey, const PathName& path, Offset offset, Length length) : - BaseArchiveVisitor(owner, initialFieldKey), - path_(path), - offset_(offset), - length_(length) { +AdoptVisitor::AdoptVisitor(Archiver& owner, const Key& initialFieldKey, const PathName& path, Offset offset, + Length length) + : BaseArchiveVisitor(owner, initialFieldKey) + , path_(path) + , offset_(offset) + , length_(length) { ASSERT(offset_ >= Offset(0)); ASSERT(length_ > Length(0)); } @@ -45,10 +46,7 @@ bool AdoptVisitor::selectDatum(const TypedKey& datumKey, const TypedKey& fullCom void AdoptVisitor::print(std::ostream& out) const { out << "AdoptVisitor[" - << "path=" << path_ - << ",offset=" << offset_ - << ",length=" << length_ - << "]"; + << "path=" << path_ << ",offset=" << offset_ << ",length=" << length_ << "]"; } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/toc/AdoptVisitor.h b/src/fdb5/toc/AdoptVisitor.h index fe2f33726..357b83a2c 100644 --- a/src/fdb5/toc/AdoptVisitor.h +++ b/src/fdb5/toc/AdoptVisitor.h @@ -20,7 +20,9 @@ #include "fdb5/database/BaseArchiveVisitor.h" -namespace metkit { class MarsRequest; } +namespace metkit { +class MarsRequest; +} namespace fdb5 { @@ -32,25 +34,18 @@ class TypedKey; class AdoptVisitor : public BaseArchiveVisitor { public: // methods - - AdoptVisitor(Archiver& owner, - const Key& initialFieldKey, - const eckit::PathName& path, - eckit::Offset offset, + AdoptVisitor(Archiver& owner, const Key& initialFieldKey, const eckit::PathName& path, eckit::Offset offset, eckit::Length length); protected: // methods - bool selectDatum(const TypedKey& datumKey, const TypedKey& fullComputedKey) override; - void print( std::ostream& out ) const override; + void print(std::ostream& out) const override; private: // members - const eckit::PathName path_; eckit::Offset offset_; eckit::Length length_; - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/toc/BTreeIndex.cc b/src/fdb5/toc/BTreeIndex.cc index 604735503..dcd1b4d08 100644 --- a/src/fdb5/toc/BTreeIndex.cc +++ b/src/fdb5/toc/BTreeIndex.cc @@ -8,12 +8,12 @@ * does it submit to any jurisdiction. */ -#include "eckit/log/BigNum.h" #include "eckit/config/Resource.h" +#include "eckit/log/BigNum.h" #include "fdb5/toc/BTreeIndex.h" -#include "fdb5/toc/TocIndex.h" #include "fdb5/toc/FieldRef.h" +#include "fdb5/toc/TocIndex.h" namespace fdb5 { @@ -23,21 +23,18 @@ BTreeIndexVisitor::~BTreeIndexVisitor() {} //---------------------------------------------------------------------------------------------------------------------- -template +template class TBTreeIndex : public BTreeIndex { public: // types - typedef eckit::FixedString BTreeKey; typedef eckit::BTree BTreeStore; public: // methods - - TBTreeIndex(const eckit::PathName &path, bool readOnly, off_t offset ); + TBTreeIndex(const eckit::PathName& path, bool readOnly, off_t offset); ~TBTreeIndex(); private: // methods - virtual bool get(const std::string& key, FieldRef& data) const; virtual bool set(const std::string& key, const FieldRef& data); virtual void flush(); @@ -48,120 +45,108 @@ class TBTreeIndex : public BTreeIndex { virtual void preload(); private: // members - mutable BTreeStore btree_; - }; +template +TBTreeIndex::TBTreeIndex(const eckit::PathName& path, bool readOnly, off_t offset) + : btree_(path, readOnly, offset) {} -template -TBTreeIndex::TBTreeIndex(const eckit::PathName &path, bool readOnly, off_t offset): - btree_( path, readOnly, offset) { -} - -template +template TBTreeIndex::~TBTreeIndex() { btree_.funlock(); } -template -bool TBTreeIndex::get(const std::string& key, FieldRef &data) const { - BTreeKey k (key); +template +bool TBTreeIndex::get(const std::string& key, FieldRef& data) const { + BTreeKey k(key); PAYLOAD payload; bool found = btree_.get(k, payload); - if(found) { + if (found) { data = FieldRef(payload); } return found; } - -template -bool TBTreeIndex::set(const std::string& key, const FieldRef &data) { - BTreeKey k (key); +template +bool TBTreeIndex::set(const std::string& key, const FieldRef& data) { + BTreeKey k(key); PAYLOAD payload(data); return btree_.set(k, payload); } -template +template void TBTreeIndex::flush() { btree_.flush(); } -template +template void TBTreeIndex::sync() { btree_.sync(); } -template +template void TBTreeIndex::flock() { btree_.flock(); } -template +template void TBTreeIndex::funlock() { btree_.funlock(); } -template +template class TBTreeIndexVisitor { - BTreeIndexVisitor &visitor_; + BTreeIndexVisitor& visitor_; + public: - TBTreeIndexVisitor(BTreeIndexVisitor &visitor): - visitor_(visitor) {} + TBTreeIndexVisitor(BTreeIndexVisitor& visitor) : visitor_(visitor) {} // BTree::range() expect a STL like collection - void clear() { - } + void clear() {} - void push_back(const typename TBTreeIndex::BTreeStore::result_type &kv) { + void push_back(const typename TBTreeIndex::BTreeStore::result_type& kv) { visitor_.visit(kv.first, kv.second); } }; -template -void TBTreeIndex::visit(BTreeIndexVisitor &visitor) const { +template +void TBTreeIndex::visit(BTreeIndexVisitor& visitor) const { TBTreeIndexVisitor v(visitor); btree_.range("", "\255", v); } -template +template void TBTreeIndex::preload() { btree_.preload(); } - - //---------------------------------------------------------------------------------------------------------------------- - -#define BTREE(KEYSIZE, RECSIZE, PAYLOAD) \ -struct BTreeIndex_##KEYSIZE##_##RECSIZE##_##PAYLOAD : public TBTreeIndex { \ - BTreeIndex_##KEYSIZE##_##RECSIZE##_##PAYLOAD (const eckit::PathName& path, bool readOnly, off_t offset): \ - TBTreeIndex(path, readOnly, offset){}; \ -}; \ -static BTreeIndexBuilder \ -maker_BTreeIndex_##KEYSIZE##_##RECSIZE##_##PAYLOAD("BTreeIndex_" #KEYSIZE "_" #RECSIZE "_" #PAYLOAD) +#define BTREE(KEYSIZE, RECSIZE, PAYLOAD) \ + struct BTreeIndex_##KEYSIZE##_##RECSIZE##_##PAYLOAD : public TBTreeIndex { \ + BTreeIndex_##KEYSIZE##_##RECSIZE##_##PAYLOAD(const eckit::PathName& path, bool readOnly, off_t offset) \ + : TBTreeIndex(path, readOnly, offset){}; \ + }; \ + static BTreeIndexBuilder \ + maker_BTreeIndex_##KEYSIZE##_##RECSIZE##_##PAYLOAD("BTreeIndex_" #KEYSIZE "_" #RECSIZE "_" #PAYLOAD) BTREE(32, 65536, FieldRefReduced); BTREE(32, 65536, FieldRefFull); BTREE(32, 4194304, FieldRefReduced); - //---------------------------------------------------------------------------------------------------------------------- -BTreeIndex::~BTreeIndex() { -} - +BTreeIndex::~BTreeIndex() {} const std::string& BTreeIndex::defaulType() { static std::string fdbIndexType = eckit::Resource("fdbIndexType;$FDB_INDEX_TYPE", "BTreeIndex"); return fdbIndexType; } -static BTreeIndexBuilder defaultIndex("BTreeIndex"); -static BTreeIndexBuilder PointDBIndex("PointDBIndex"); +static BTreeIndexBuilder defaultIndex("BTreeIndex"); +static BTreeIndexBuilder PointDBIndex("PointDBIndex"); static BTreeIndexBuilder BTreeIndex4MB("BTreeIndex4MB"); //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/toc/BTreeIndex.h b/src/fdb5/toc/BTreeIndex.h index 3b61db572..c4dd0b686 100644 --- a/src/fdb5/toc/BTreeIndex.h +++ b/src/fdb5/toc/BTreeIndex.h @@ -42,7 +42,7 @@ class BTreeIndex { public: virtual ~BTreeIndex(); virtual bool get(const std::string& key, FieldRef& data) const = 0; - virtual bool set(const std::string& key, const FieldRef& data)= 0; + virtual bool set(const std::string& key, const FieldRef& data) = 0; virtual void flush() = 0; virtual void sync() = 0; virtual void visit(BTreeIndexVisitor& visitor) const = 0; @@ -50,46 +50,40 @@ class BTreeIndex { virtual void funlock() = 0; virtual void preload() = 0; - static const std::string& defaulType(); - }; //---------------------------------------------------------------------------------------------------------------------- class BTreeIndexFactory { - virtual BTreeIndex *make(const eckit::PathName& path, bool readOnly, off_t offset) const = 0 ; + virtual BTreeIndex* make(const eckit::PathName& path, bool readOnly, off_t offset) const = 0; protected: - - BTreeIndexFactory(const std::string &); + BTreeIndexFactory(const std::string&); virtual ~BTreeIndexFactory(); std::string name_; public: - - static void list(std::ostream &); - static BTreeIndex *build(const std::string &name, const eckit::PathName& path, bool readOnly, off_t offset); - + static void list(std::ostream&); + static BTreeIndex* build(const std::string& name, const eckit::PathName& path, bool readOnly, off_t offset); }; /// Templated specialisation of the self-registering factory, /// that does the self-registration, and the construction of each object. -template< class T> +template class BTreeIndexBuilder : public BTreeIndexFactory { - BTreeIndex *make(const eckit::PathName& path, bool readOnly, off_t offset) const override { + BTreeIndex* make(const eckit::PathName& path, bool readOnly, off_t offset) const override { return new T(path, readOnly, offset); } public: - BTreeIndexBuilder(const std::string &name) : BTreeIndexFactory(name) {} + BTreeIndexBuilder(const std::string& name) : BTreeIndexFactory(name) {} }; - //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 diff --git a/src/fdb5/toc/EnvVarFileSpaceHandler.cc b/src/fdb5/toc/EnvVarFileSpaceHandler.cc index 0a04013c9..e9795dc25 100644 --- a/src/fdb5/toc/EnvVarFileSpaceHandler.cc +++ b/src/fdb5/toc/EnvVarFileSpaceHandler.cc @@ -15,12 +15,12 @@ #include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" -#include "eckit/thread/AutoLock.h" #include "eckit/io/FileLock.h" +#include "eckit/thread/AutoLock.h" -#include "fdb5/toc/FileSpace.h" -#include "fdb5/database/Key.h" #include "fdb5/LibFdb5.h" +#include "fdb5/database/Key.h" +#include "fdb5/toc/FileSpace.h" using namespace eckit; @@ -28,17 +28,13 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -EnvVarFileSpaceHandler::EnvVarFileSpaceHandler() : - fdbFileSpaceSHandlerEnvVarName_(LibResource("fdbFileSpaceSHandlerEnvVarName;$FDB_FILESPACEHANDLER_ENVVARNAME", "STHOST")) -{ -} +EnvVarFileSpaceHandler::EnvVarFileSpaceHandler() + : fdbFileSpaceSHandlerEnvVarName_(LibResource( + "fdbFileSpaceSHandlerEnvVarName;$FDB_FILESPACEHANDLER_ENVVARNAME", "STHOST")) {} -EnvVarFileSpaceHandler::~EnvVarFileSpaceHandler() { -} +EnvVarFileSpaceHandler::~EnvVarFileSpaceHandler() {} - -PathName EnvVarFileSpaceHandler::select(const Key& key, const FileSpace& fs) const -{ +PathName EnvVarFileSpaceHandler::select(const Key& key, const FileSpace& fs) const { return FileSpaceHandler::lookup("WeightedRandom").selectFileSystem(key, fs); } @@ -46,19 +42,20 @@ eckit::PathName EnvVarFileSpaceHandler::selectFileSystem(const Key& key, const F AutoLock lock(mutex_); - LOG_DEBUG_LIB(LibFdb5) << "Selecting a file system based on environment variable " << fdbFileSpaceSHandlerEnvVarName_ << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "Selecting a file system based on environment variable " + << fdbFileSpaceSHandlerEnvVarName_ << std::endl; - const char* value = ::getenv(fdbFileSpaceSHandlerEnvVarName_.c_str()); - if(value) { + const char* value = ::getenv(fdbFileSpaceSHandlerEnvVarName_.c_str()); + if (value) { eckit::PathName path(value); - if(path.exists()) return path; + if (path.exists()) + return path; std::ostringstream msg; msg << ""; throw ReadError(msg.str(), Here()); - } - else { + } else { std::ostringstream msg; msg << ""; throw UserError(msg.str(), Here()); @@ -69,4 +66,4 @@ static FileSpaceHandlerRegister envvar("envvar"); //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/toc/EnvVarFileSpaceHandler.h b/src/fdb5/toc/EnvVarFileSpaceHandler.h index f0997cede..0564adae8 100644 --- a/src/fdb5/toc/EnvVarFileSpaceHandler.h +++ b/src/fdb5/toc/EnvVarFileSpaceHandler.h @@ -21,8 +21,8 @@ #include "eckit/filesystem/PathName.h" #include "eckit/thread/Mutex.h" -#include "fdb5/toc/FileSpaceHandler.h" #include "fdb5/database/Key.h" +#include "fdb5/toc/FileSpaceHandler.h" namespace fdb5 { @@ -30,10 +30,9 @@ namespace fdb5 { class EnvVarFileSpaceHandler : public FileSpaceHandler { - typedef std::map PathTable; + typedef std::map PathTable; public: // methods - EnvVarFileSpaceHandler(); ~EnvVarFileSpaceHandler() override; @@ -41,7 +40,6 @@ class EnvVarFileSpaceHandler : public FileSpaceHandler { eckit::PathName selectFileSystem(const Key& key, const FileSpace& fs) const override; protected: // methods - void load() const; eckit::PathName append(const std::string& expver, const eckit::PathName& path) const; diff --git a/src/fdb5/toc/ExpverFileSpaceHandler.cc b/src/fdb5/toc/ExpverFileSpaceHandler.cc index 22fe62592..f584f9461 100644 --- a/src/fdb5/toc/ExpverFileSpaceHandler.cc +++ b/src/fdb5/toc/ExpverFileSpaceHandler.cc @@ -10,7 +10,7 @@ #include "ExpverFileSpaceHandler.h" -#include +#include #include #include @@ -19,12 +19,12 @@ #include "eckit/config/Resource.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" -#include "eckit/thread/AutoLock.h" #include "eckit/io/FileLock.h" +#include "eckit/thread/AutoLock.h" -#include "fdb5/toc/FileSpace.h" -#include "fdb5/database/Key.h" #include "fdb5/LibFdb5.h" +#include "fdb5/database/Key.h" +#include "fdb5/toc/FileSpace.h" using namespace eckit; @@ -32,13 +32,11 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -ExpverFileSpaceHandler::ExpverFileSpaceHandler() : - fdbExpverFileSystems_(LibResource("fdbExpverFileSystems;$FDB_EXPVER_FILE", "~fdb/etc/fdb/expver_to_fdb_root.map")) -{ -} +ExpverFileSpaceHandler::ExpverFileSpaceHandler() + : fdbExpverFileSystems_(LibResource("fdbExpverFileSystems;$FDB_EXPVER_FILE", + "~fdb/etc/fdb/expver_to_fdb_root.map")) {} -ExpverFileSpaceHandler::~ExpverFileSpaceHandler() { -} +ExpverFileSpaceHandler::~ExpverFileSpaceHandler() {} void ExpverFileSpaceHandler::load() const { @@ -46,9 +44,9 @@ void ExpverFileSpaceHandler::load() const { std::ifstream in(fdbExpverFileSystems_.localPath()); - if(!in) { + if (!in) { std::ostringstream oss; - oss << fdbExpverFileSystems_ << Log::syserr; + oss << fdbExpverFileSystems_ << Log::syserr; Log::error() << oss.str() << std::endl; throw CantOpenFile(oss.str(), Here()); } @@ -58,37 +56,36 @@ void ExpverFileSpaceHandler::load() const { Tokenizer parse(" "); std::vector s; - while(in.getline(line, sizeof(line))) - { + while (in.getline(line, sizeof(line))) { ++lineNo; s.clear(); - parse(line,s); + parse(line, s); size_t i = 0; - while( i < s.size() ) /* cleanup entries that are empty */ + while (i < s.size()) /* cleanup entries that are empty */ { - if(s[i].length() == 0) - s.erase(s.begin()+i); + if (s[i].length() == 0) + s.erase(s.begin() + i); else i++; } - if(s.size() == 0 || s[0][0] == '#') + if (s.size() == 0 || s[0][0] == '#') continue; - if(s.size() != 2) { + if (s.size() != 2) { std::ostringstream oss; - oss << "Bad line (" << lineNo << ") in configuration file " << fdbExpverFileSystems_ << " -- should have format 'expver filesystem'"; + oss << "Bad line (" << lineNo << ") in configuration file " << fdbExpverFileSystems_ + << " -- should have format 'expver filesystem'"; throw ReadError(oss.str(), Here()); } - table_[ s[0] ] = PathName(s[1]); + table_[s[0]] = PathName(s[1]); } } -eckit::PathName ExpverFileSpaceHandler::append(const std::string& expver, const PathName& path) const -{ +eckit::PathName ExpverFileSpaceHandler::append(const std::string& expver, const PathName& path) const { // obtain exclusive lock to file PathName lockFile = fdbExpverFileSystems_ + ".lock"; @@ -100,45 +97,46 @@ eckit::PathName ExpverFileSpaceHandler::append(const std::string& expver, const std::ifstream fi(fdbExpverFileSystems_.localPath()); - if(!fi) { + if (!fi) { std::ostringstream oss; - oss << fdbExpverFileSystems_ << Log::syserr; + oss << fdbExpverFileSystems_ << Log::syserr; Log::error() << oss.str() << std::endl; throw CantOpenFile(oss.str(), Here()); } - char line[4*1024]; + char line[4 * 1024]; size_t lineNo = 0; Tokenizer parse(" "); std::vector s; - while(fi.getline(line, sizeof(line))) - { + while (fi.getline(line, sizeof(line))) { ++lineNo; s.clear(); - parse(line,s); + parse(line, s); size_t i = 0; - while( i < s.size() ) /* cleanup entries that are empty */ + while (i < s.size()) /* cleanup entries that are empty */ { - if(s[i].length() == 0) - s.erase(s.begin()+i); + if (s[i].length() == 0) + s.erase(s.begin() + i); else i++; } - if(s.size() == 0 || s[0][0] == '#') + if (s.size() == 0 || s[0][0] == '#') continue; - if(s.size() != 2) { + if (s.size() != 2) { std::ostringstream oss; - oss << "Bad line (" << lineNo << ") in configuration file " << fdbExpverFileSystems_ << " -- should have format 'expver filesystem'"; + oss << "Bad line (" << lineNo << ") in configuration file " << fdbExpverFileSystems_ + << " -- should have format 'expver filesystem'"; throw ReadError(oss.str(), Here()); } - if(s[0] == expver) { - LOG_DEBUG_LIB(LibFdb5) << "Found expver " << expver << " " << path << " in " << fdbExpverFileSystems_ << std::endl; + if (s[0] == expver) { + LOG_DEBUG_LIB(LibFdb5) << "Found expver " << expver << " " << path << " in " << fdbExpverFileSystems_ + << std::endl; return PathName(s[1]); } } @@ -147,16 +145,17 @@ eckit::PathName ExpverFileSpaceHandler::append(const std::string& expver, const std::ofstream of(fdbExpverFileSystems_.localPath(), std::ofstream::app); - if(!of) { + if (!of) { std::ostringstream oss; - oss << fdbExpverFileSystems_ << Log::syserr; + oss << fdbExpverFileSystems_ << Log::syserr; Log::error() << oss.str() << std::endl; throw WriteError(oss.str(), Here()); } // append to the file - LOG_DEBUG_LIB(LibFdb5) << "Appending expver " << expver << " " << path << " to " << fdbExpverFileSystems_ << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "Appending expver " << expver << " " << path << " to " << fdbExpverFileSystems_ + << std::endl; of << expver << " " << path << std::endl; @@ -165,15 +164,14 @@ eckit::PathName ExpverFileSpaceHandler::append(const std::string& expver, const return path; } -PathName ExpverFileSpaceHandler::select(const Key& key, const FileSpace& fs) const -{ +PathName ExpverFileSpaceHandler::select(const Key& key, const FileSpace& fs) const { return FileSpaceHandler::lookup("WeightedRandom").selectFileSystem(key, fs); } static bool expver_is_valid(const std::string& str) { LOG_DEBUG_LIB(LibFdb5) << "Validating expver string [" << str << "]" << std::endl; return ((str.size() <= 4) and std::find_if_not(str.begin(), str.end(), isdigit) == str.end()) || - ((str.size() == 4) and std::find_if_not(str.begin(), str.end(), isalnum) == str.end()); + ((str.size() == 4) and std::find_if_not(str.begin(), str.end(), isalnum) == str.end()); } eckit::PathName ExpverFileSpaceHandler::selectFileSystem(const Key& key, const FileSpace& fs) const { @@ -186,23 +184,26 @@ eckit::PathName ExpverFileSpaceHandler::selectFileSystem(const Key& key, const F // check if key is mapped already to a filesystem - if(table_.empty()) load(); + if (table_.empty()) + load(); std::string expver = key.get("expver"); // we can NOT use the type system here because we haven't opened a DB yet - // so we have to do validation directly on string - if(not expver_is_valid(expver)) + // so we have to do validation directly on string + if (not expver_is_valid(expver)) throw eckit::BadValue("Invalid expver value " + expver, Here()); LOG_DEBUG_LIB(LibFdb5) << "Selecting file system for expver [" << expver << "]" << std::endl; PathTable::const_iterator itr = table_.find(expver); - if(itr != table_.end()) { - LOG_DEBUG_LIB(LibFdb5) << "Found expver " << expver << " " << itr->second << " in " << fdbExpverFileSystems_ << std::endl; + if (itr != table_.end()) { + LOG_DEBUG_LIB(LibFdb5) << "Found expver " << expver << " " << itr->second << " in " << fdbExpverFileSystems_ + << std::endl; if (!fdbRootDirectory.empty() && itr->second != fdbRootDirectory) { - Log::warning() << "Existing root directory " << itr->second << " does not match FDB5_ROOT. Using existing" << std::endl; + Log::warning() << "Existing root directory " << itr->second << " does not match FDB5_ROOT. Using existing" + << std::endl; } return itr->second; @@ -217,9 +218,10 @@ eckit::PathName ExpverFileSpaceHandler::selectFileSystem(const Key& key, const F // Before we allow an override, ensure that it is one of the available filesystems. std::vector writable(fs.enabled(ControlIdentifier::Archive)); - if(std::find(writable.begin(), writable.end(), fdbRootDirectory) == writable.end()) { + if (std::find(writable.begin(), writable.end(), fdbRootDirectory) == writable.end()) { std::ostringstream msg; - msg << "FDB root directory " << fdbRootDirectory << " was not in the list of roots supporting archival " << writable; + msg << "FDB root directory " << fdbRootDirectory << " was not in the list of roots supporting archival " + << writable; throw BadParameter(msg.str()); } @@ -235,7 +237,8 @@ eckit::PathName ExpverFileSpaceHandler::selectFileSystem(const Key& key, const F table_[expver] = selected; if (!fdbRootDirectory.empty() && selected != fdbRootDirectory) { - Log::warning() << "Selected root directory " << itr->second << " does not match FDB5_ROOT. Using existing" << std::endl; + Log::warning() << "Selected root directory " << itr->second << " does not match FDB5_ROOT. Using existing" + << std::endl; } return selected; @@ -245,4 +248,4 @@ static FileSpaceHandlerRegister expver("expver"); //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/toc/ExpverFileSpaceHandler.h b/src/fdb5/toc/ExpverFileSpaceHandler.h index d465bad2e..d275e73a4 100644 --- a/src/fdb5/toc/ExpverFileSpaceHandler.h +++ b/src/fdb5/toc/ExpverFileSpaceHandler.h @@ -22,8 +22,8 @@ #include "eckit/filesystem/PathName.h" #include "eckit/thread/Mutex.h" -#include "fdb5/toc/FileSpaceHandler.h" #include "fdb5/database/Key.h" +#include "fdb5/toc/FileSpaceHandler.h" namespace fdb5 { @@ -31,10 +31,9 @@ namespace fdb5 { class ExpverFileSpaceHandler : public FileSpaceHandler { - typedef std::map PathTable; + typedef std::map PathTable; public: // methods - ExpverFileSpaceHandler(); ~ExpverFileSpaceHandler() override; @@ -42,7 +41,6 @@ class ExpverFileSpaceHandler : public FileSpaceHandler { eckit::PathName selectFileSystem(const Key& key, const FileSpace& fs) const override; protected: // methods - void load() const; eckit::PathName append(const std::string& expver, const eckit::PathName& path) const; diff --git a/src/fdb5/toc/FieldRef.cc b/src/fdb5/toc/FieldRef.cc index db500336b..70b4c071d 100644 --- a/src/fdb5/toc/FieldRef.cc +++ b/src/fdb5/toc/FieldRef.cc @@ -14,65 +14,47 @@ #include "eckit/filesystem/URI.h" #include "eckit/serialisation/Stream.h" -#include "fdb5/fdb5_config.h" #include "fdb5/database/Field.h" #include "fdb5/database/UriStore.h" - +#include "fdb5/fdb5_config.h" namespace fdb5 { - //---------------------------------------------------------------------------------------------------------------------- +FieldRefLocation::FieldRefLocation() {} -FieldRefLocation::FieldRefLocation() { -} - - -FieldRefLocation::FieldRefLocation(UriStore &store, const Field& field) { +FieldRefLocation::FieldRefLocation(UriStore& store, const Field& field) { const FieldLocation& loc = field.location(); uriId_ = store.insert(loc.uri()); length_ = loc.length(); offset_ = loc.offset(); - } -void FieldRefLocation::print(std::ostream &s) const { +void FieldRefLocation::print(std::ostream& s) const { s << "FieldRefLocation(pathid=" << uriId_ << ",offset=" << offset_ << ",length=" << length_ << ")"; } -FieldRefReduced::FieldRefReduced() { +FieldRefReduced::FieldRefReduced() {} -} - -FieldRefReduced::FieldRefReduced(const FieldRef &other): - location_(other.location()) { -} +FieldRefReduced::FieldRefReduced(const FieldRef& other) : location_(other.location()) {} -void FieldRefReduced::print(std::ostream &s) const { +void FieldRefReduced::print(std::ostream& s) const { s << location_; } -FieldRef::FieldRef() { -} +FieldRef::FieldRef() {} -FieldRef::FieldRef(UriStore &store, const Field &field): - location_(store, field), - details_(field.details()) { -} +FieldRef::FieldRef(UriStore& store, const Field& field) : location_(store, field), details_(field.details()) {} -FieldRef::FieldRef(const FieldRefReduced& other): - location_(other.location()) { -} +FieldRef::FieldRef(const FieldRefReduced& other) : location_(other.location()) {} -void FieldRef::print(std::ostream &s) const { +void FieldRef::print(std::ostream& s) const { s << location_; } - //---------------------------------------------------------------------------------------------------------------------- - } // namespace fdb5 diff --git a/src/fdb5/toc/FieldRef.h b/src/fdb5/toc/FieldRef.h index 30112ff9c..29fc33ed6 100644 --- a/src/fdb5/toc/FieldRef.h +++ b/src/fdb5/toc/FieldRef.h @@ -17,11 +17,11 @@ #include "eckit/eckit.h" +#include "eckit/filesystem/PathName.h" #include "eckit/io/DataHandle.h" #include "eckit/io/Length.h" -#include "eckit/memory/NonCopyable.h" #include "eckit/io/Offset.h" -#include "eckit/filesystem/PathName.h" +#include "eckit/memory/NonCopyable.h" #include "fdb5/database/FieldDetails.h" @@ -30,82 +30,75 @@ namespace fdb5 { class Field; class UriStore; - //---------------------------------------------------------------------------------------------------------------------- - class FieldRefLocation { public: typedef size_t UriID; FieldRefLocation(); - FieldRefLocation(UriStore &, const Field &); - + FieldRefLocation(UriStore&, const Field&); UriID uriId() const { return uriId_; } - const eckit::Offset &offset() const { return offset_; } - const eckit::Length &length() const { return length_; } + const eckit::Offset& offset() const { return offset_; } + const eckit::Length& length() const { return length_; } protected: - UriID uriId_; - eckit::Offset offset_; - eckit::Length length_; + UriID uriId_; + eckit::Offset offset_; + eckit::Length length_; - void print(std::ostream &s) const; + void print(std::ostream& s) const; - friend std::ostream &operator<<(std::ostream &s, const FieldRefLocation &x) { + friend std::ostream& operator<<(std::ostream& s, const FieldRefLocation& x) { x.print(s); return s; } }; - //---------------------------------------------------------------------------------------------------------------------- - class FieldRef; - class FieldRefReduced { FieldRefLocation location_; + public: FieldRefReduced(); FieldRefReduced(const FieldRef&); const FieldRefLocation& location() const { return location_; } private: // methods + void print(std::ostream& s) const; - void print(std::ostream &s) const; - - friend std::ostream &operator<<(std::ostream &s, const FieldRefReduced &x) { + friend std::ostream& operator<<(std::ostream& s, const FieldRefReduced& x) { x.print(s); return s; } }; -class FieldRef { +class FieldRef { FieldRefLocation location_; FieldDetails details_; -public: +public: FieldRef(); - FieldRef(UriStore &, const Field &); + FieldRef(UriStore&, const Field&); FieldRef(const FieldRefReduced&); FieldRefLocation::UriID uriId() const { return location_.uriId(); } - const eckit::Offset &offset() const { return location_.offset(); } - const eckit::Length &length() const { return location_.length(); } + const eckit::Offset& offset() const { return location_.offset(); } + const eckit::Length& length() const { return location_.length(); } const FieldRefLocation& location() const { return location_; } const FieldDetails& details() const { return details_; } private: // methods + void print(std::ostream& s) const; - void print(std::ostream &s) const; - - friend std::ostream &operator<<(std::ostream &s, const FieldRef &x) { + friend std::ostream& operator<<(std::ostream& s, const FieldRef& x) { x.print(s); return s; } diff --git a/src/fdb5/toc/FileSpace.cc b/src/fdb5/toc/FileSpace.cc index fce73de98..cd11a1c9d 100644 --- a/src/fdb5/toc/FileSpace.cc +++ b/src/fdb5/toc/FileSpace.cc @@ -8,14 +8,14 @@ * does it submit to any jurisdiction. */ -#include "eckit/config/Resource.h" #include "fdb5/toc/FileSpace.h" +#include "eckit/config/Resource.h" #include "eckit/filesystem/StdDir.h" #include "eckit/utils/StringTools.h" -#include "eckit/os/BackTrace.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/FileSpaceStrategies.h" +#include "eckit/os/BackTrace.h" #include "fdb5/LibFdb5.h" #include "fdb5/database/Key.h" @@ -29,15 +29,18 @@ template <> struct VectorPrintSelector { typedef VectorPrintSimple selector; }; -} // namespace eckit +} // namespace eckit namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- FileSpace::FileSpace(const std::string& name, const std::string& re, const std::string& handler, - const std::vector& roots) : - name_(name), handler_(handler), re_(re), roots_(roots) {} + const std::vector& roots) + : name_(name) + , handler_(handler) + , re_(re) + , roots_(roots) {} TocPath FileSpace::filesystem(const Key& key, const eckit::PathName& db) const { // check that the database isn't present already @@ -87,7 +90,8 @@ bool FileSpace::match(const std::string& s) const { eckit::PathName getFullDB(const eckit::PathName& path, const std::string& db) { - static bool searchCaseSensitiveDB = eckit::Resource("fdbSearchCaseSensitiveDB;$FDB_SEARCH_CASESENSITIVE_DB", true); + static bool searchCaseSensitiveDB = + eckit::Resource("fdbSearchCaseSensitiveDB;$FDB_SEARCH_CASESENSITIVE_DB", true); if (searchCaseSensitiveDB) { @@ -97,15 +101,14 @@ eckit::PathName getFullDB(const eckit::PathName& path, const std::string& db) { throw eckit::FailedSystemCall("opendir"); } - // Once readdir_r finally gets deprecated and removed, we may need to + // Once readdir_r finally gets deprecated and removed, we may need to // protecting readdir() as not yet guarranteed thread-safe by POSIX // technically it should only be needed on a per-directory basis // this should be a resursive mutex - // AutoLock lock(mutex_); + // AutoLock lock(mutex_); std::string ldb = eckit::StringTools::lower(db); - for(;;) - { + for (;;) { struct dirent* e = d.dirent(); if (e == nullptr) { break; @@ -120,7 +123,6 @@ eckit::PathName getFullDB(const eckit::PathName& path, const std::string& db) { return path / db; } - bool FileSpace::existsDB(const Key& key, const eckit::PathName& db, TocPath& existsDB) const { unsigned count = 0; bool found = false; @@ -133,7 +135,8 @@ bool FileSpace::existsDB(const Key& key, const eckit::PathName& db, TocPath& exi if (fullDB.exists() && dbToc.exists()) { matchList += (count == 0 ? "" : ", ") + fullDB; - bool allowMultipleDbs = (fullDB / (controlfile_lookup.find(ControlIdentifier::UniqueRoot)->second)).exists(); + bool allowMultipleDbs = + (fullDB / (controlfile_lookup.find(ControlIdentifier::UniqueRoot)->second)).exists(); if (!count || allowMultipleDbs) { // take last existsDB.directory_ = fullDB; existsDB.controlIdentifiers_ = i->controlIdentifiers(); @@ -155,17 +158,15 @@ bool FileSpace::existsDB(const Key& key, const eckit::PathName& db, TocPath& exi void FileSpace::print(std::ostream& out) const { out << "FileSpace(" - << "name=" << name_ << ",handler=" << handler_ << ",regex=" << re_ << ",roots=" << roots_ - << ")"; + << "name=" << name_ << ",handler=" << handler_ << ",regex=" << re_ << ",roots=" << roots_ << ")"; } std::vector FileSpace::roots() const { std::vector result; - std::transform(roots_.begin(), roots_.end(), std::back_inserter(result), - [](const Root& r) { return r.path(); }); + std::transform(roots_.begin(), roots_.end(), std::back_inserter(result), [](const Root& r) { return r.path(); }); return result; } //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/toc/FileSpace.h b/src/fdb5/toc/FileSpace.h index b26fb6ba9..adab57a71 100644 --- a/src/fdb5/toc/FileSpace.h +++ b/src/fdb5/toc/FileSpace.h @@ -16,15 +16,15 @@ #ifndef fdb5_FileSpace_H #define fdb5_FileSpace_H +#include #include #include -#include -#include "eckit/utils/Regex.h" #include "eckit/types/Types.h" +#include "eckit/utils/Regex.h" -#include "fdb5/toc/Root.h" #include "fdb5/api/helpers/ControlIterator.h" +#include "fdb5/toc/Root.h" namespace fdb5 { @@ -42,10 +42,7 @@ struct TocPath { class FileSpace { public: // methods - - FileSpace(const std::string& name, - const std::string& re, - const std::string& handler, + FileSpace(const std::string& name, const std::string& re, const std::string& handler, const std::vector& roots); /// Selects the filesystem from where this Key will be inserted @@ -59,8 +56,8 @@ class FileSpace { std::vector enabled(const ControlIdentifier& controlIdentifier) const; bool match(const std::string& s) const; - - friend std::ostream& operator<<(std::ostream &s, const FileSpace& x) { + + friend std::ostream& operator<<(std::ostream& s, const FileSpace& x) { x.print(s); return s; } @@ -68,13 +65,11 @@ class FileSpace { std::vector roots() const; private: // methods - bool existsDB(const Key& key, const eckit::PathName& db, TocPath& existsDB) const; - void print( std::ostream &out ) const; + void print(std::ostream& out) const; private: // members - typedef std::vector RootVec; std::string name_; diff --git a/src/fdb5/toc/FileSpaceHandler.cc b/src/fdb5/toc/FileSpaceHandler.cc index 8b3c55f1a..5019e8905 100644 --- a/src/fdb5/toc/FileSpaceHandler.cc +++ b/src/fdb5/toc/FileSpaceHandler.cc @@ -12,12 +12,12 @@ #include "fdb5/toc/FileSpaceHandler.h" -#include "eckit/filesystem/FileSpaceStrategies.h" #include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/FileSpaceStrategies.h" #include "eckit/thread/AutoLock.h" -#include "fdb5/toc/FileSpace.h" #include "fdb5/database/Key.h" +#include "fdb5/toc/FileSpace.h" using eckit::Log; @@ -36,10 +36,11 @@ static void init() { //---------------------------------------------------------------------------------------------------------------------- -const FileSpaceHandler& FileSpaceHandlerInstance::get() -{ - if(!instance_) { instance_ = make(); } - return *instance_ ; +const FileSpaceHandler& FileSpaceHandlerInstance::get() { + if (!instance_) { + instance_ = make(); + } + return *instance_; } FileSpaceHandlerInstance::FileSpaceHandlerInstance(const std::string& name) : name_(name) { @@ -50,8 +51,7 @@ FileSpaceHandlerInstance::~FileSpaceHandlerInstance() { FileSpaceHandler::unregist(name_); } -const FileSpaceHandler& FileSpaceHandler::lookup(const std::string& name) -{ +const FileSpaceHandler& FileSpaceHandler::lookup(const std::string& name) { pthread_once(&once, init); eckit::AutoLock lock(local_mutex); @@ -60,7 +60,7 @@ const FileSpaceHandler& FileSpaceHandler::lookup(const std::string& name) if (j == m->end()) { eckit::Log::error() << "No FileSpaceHandler factory for [" << name << "]" << std::endl; eckit::Log::error() << "Available FileSpaceHandler's are:" << std::endl; - for (j = m->begin() ; j != m->end() ; ++j) + for (j = m->begin(); j != m->end(); ++j) eckit::Log::error() << " " << (*j).first << std::endl; throw eckit::SeriousBug(std::string("No FileSpaceHandler called ") + name); } @@ -84,11 +84,9 @@ void FileSpaceHandler::unregist(const std::string& name) { m->erase(name); } -FileSpaceHandler::~FileSpaceHandler() { -} +FileSpaceHandler::~FileSpaceHandler() {} -FileSpaceHandler::FileSpaceHandler() { -} +FileSpaceHandler::FileSpaceHandler() {} //---------------------------------------------------------------------------------------------------------------------- @@ -112,7 +110,7 @@ static FileSpaceHandlerRegister first("First"); struct LeastUsed : public FileSpaceHandler { LeastUsed() {} - eckit::PathName selectFileSystem(const Key&, const FileSpace& fs) const { + eckit::PathName selectFileSystem(const Key&, const FileSpace& fs) const { return eckit::FileSpaceStrategies::leastUsed(fs.enabled(ControlIdentifier::Archive)); } }; @@ -121,7 +119,7 @@ static FileSpaceHandlerRegister leastUsed("LeastUsed"); struct LeastUsedPercent : public FileSpaceHandler { LeastUsedPercent() {} - eckit::PathName selectFileSystem(const Key&, const FileSpace& fs) const { + eckit::PathName selectFileSystem(const Key&, const FileSpace& fs) const { return eckit::FileSpaceStrategies::leastUsedPercent(fs.enabled(ControlIdentifier::Archive)); } }; @@ -130,7 +128,7 @@ static FileSpaceHandlerRegister leastUsedPercent("LeastUsedPer struct RoundRobin : public FileSpaceHandler { RoundRobin() {} - eckit::PathName selectFileSystem(const Key&, const FileSpace& fs) const { + eckit::PathName selectFileSystem(const Key&, const FileSpace& fs) const { return eckit::FileSpaceStrategies::roundRobin(fs.enabled(ControlIdentifier::Archive)); } }; @@ -139,7 +137,7 @@ static FileSpaceHandlerRegister roundRobin("RoundRobin"); struct Random : public FileSpaceHandler { Random() {} - eckit::PathName selectFileSystem(const Key&, const FileSpace& fs) const { + eckit::PathName selectFileSystem(const Key&, const FileSpace& fs) const { return eckit::FileSpaceStrategies::pureRandom(fs.enabled(ControlIdentifier::Archive)); } }; @@ -149,7 +147,7 @@ static FileSpaceHandlerRegister random("Random"); struct WeightedRandom : public FileSpaceHandler { WeightedRandom() {} - eckit::PathName selectFileSystem(const Key&, const FileSpace& fs) const { + eckit::PathName selectFileSystem(const Key&, const FileSpace& fs) const { return eckit::FileSpaceStrategies::weightedRandom(fs.enabled(ControlIdentifier::Archive)); } }; @@ -158,15 +156,15 @@ static FileSpaceHandlerRegister weightedRandom("WeightedRandom") struct WeightedRandomPercent : public FileSpaceHandler { WeightedRandomPercent() {} - eckit::PathName selectFileSystem(const Key&, const FileSpace& fs) const { + eckit::PathName selectFileSystem(const Key&, const FileSpace& fs) const { return eckit::FileSpaceStrategies::weightedRandomPercent(fs.enabled(ControlIdentifier::Archive)); } }; static FileSpaceHandlerRegister weightedRandomPercent("WeightedRandomPercent"); -} +} // namespace detail //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/toc/FileSpaceHandler.h b/src/fdb5/toc/FileSpaceHandler.h index 9d476fb3a..dd514f8fb 100644 --- a/src/fdb5/toc/FileSpaceHandler.h +++ b/src/fdb5/toc/FileSpaceHandler.h @@ -16,9 +16,9 @@ #ifndef fdb5_FileSpaceHandler_H #define fdb5_FileSpaceHandler_H -#include "eckit/utils/Regex.h" -#include "eckit/types/Types.h" #include "eckit/memory/NonCopyable.h" +#include "eckit/types/Types.h" +#include "eckit/utils/Regex.h" #include "fdb5/toc/Root.h" @@ -34,7 +34,6 @@ class FileSpaceHandlerInstance; class FileSpaceHandler : private eckit::NonCopyable { public: // methods - static const FileSpaceHandler& lookup(const std::string& name); static void regist(const std::string& name, FileSpaceHandlerInstance* h); static void unregist(const std::string& name); @@ -44,38 +43,31 @@ class FileSpaceHandler : private eckit::NonCopyable { virtual eckit::PathName selectFileSystem(const Key& key, const FileSpace& fs) const = 0; protected: // methods - FileSpaceHandler(); - }; //---------------------------------------------------------------------------------------------------------------------- class FileSpaceHandlerInstance { public: - const FileSpaceHandler& get(); protected: - FileSpaceHandlerInstance(const std::string& name); virtual ~FileSpaceHandlerInstance(); private: - virtual FileSpaceHandler* make() const = 0; std::string name_; mutable FileSpaceHandler* instance_; - }; -template +template class FileSpaceHandlerRegister : public FileSpaceHandlerInstance { public: - FileSpaceHandlerRegister(const std::string& name) : FileSpaceHandlerInstance(name) { - } + FileSpaceHandlerRegister(const std::string& name) : FileSpaceHandlerInstance(name) {} private: FileSpaceHandler* make() const override { return new T(); } diff --git a/src/fdb5/toc/Root.cc b/src/fdb5/toc/Root.cc index 794ec6f25..69891cbeb 100644 --- a/src/fdb5/toc/Root.cc +++ b/src/fdb5/toc/Root.cc @@ -11,8 +11,8 @@ #include "eckit/log/Log.h" #include "eckit/os/Stat.h" -#include "fdb5/toc/Root.h" #include "fdb5/LibFdb5.h" +#include "fdb5/toc/Root.h" using eckit::Log; using eckit::Stat; @@ -21,36 +21,37 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -Root::Root(const std::string &path, const std::string& filespace, - bool list, bool retrieve, bool archive, bool wipe): - path_(path), - filespace_(filespace), - checked_(false), - exists_(false), - controlIdentifiers_() -{ - if (!list) controlIdentifiers_ |= ControlIdentifier::List; - if (!retrieve) controlIdentifiers_ |= ControlIdentifier::Retrieve; - if (!archive) controlIdentifiers_ |= ControlIdentifier::Archive; - if (!wipe) controlIdentifiers_ |= ControlIdentifier::Wipe; +Root::Root(const std::string& path, const std::string& filespace, bool list, bool retrieve, bool archive, bool wipe) + : path_(path) + , filespace_(filespace) + , checked_(false) + , exists_(false) + , controlIdentifiers_() { + if (!list) + controlIdentifiers_ |= ControlIdentifier::List; + if (!retrieve) + controlIdentifiers_ |= ControlIdentifier::Retrieve; + if (!archive) + controlIdentifiers_ |= ControlIdentifier::Archive; + if (!wipe) + controlIdentifiers_ |= ControlIdentifier::Wipe; } bool Root::exists() const { if (!checked_) { errno = 0; Stat::Struct info; - int err = Stat::stat(path_.asString().c_str(),&info); - if(not err) { + int err = Stat::stat(path_.asString().c_str(), &info); + if (not err) { exists_ = S_ISDIR(info.st_mode); - } - else { + } else { Log::warning() << "FDB root " << path_ << " " << Log::syserr << std::endl; exists_ = false; } LOG_DEBUG_LIB(LibFdb5) << "Root " << *this << (exists_ ? " exists" : " does NOT exists") << std::endl; checked_ = true; } - + return exists_; } @@ -58,19 +59,16 @@ const eckit::PathName& Root::path() const { return path_; } -const std::string& Root::filespace() const -{ +const std::string& Root::filespace() const { return filespace_; } -void Root::print( std::ostream &out ) const { +void Root::print(std::ostream& out) const { out << "Root(" - << "path=" << path_ - << ",controlIdentifiers=" << controlIdentifiers_ - <<")"; + << "path=" << path_ << ",controlIdentifiers=" << controlIdentifiers_ << ")"; } //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/toc/Root.h b/src/fdb5/toc/Root.h index 3a4f1b958..a9d38e86e 100644 --- a/src/fdb5/toc/Root.h +++ b/src/fdb5/toc/Root.h @@ -16,8 +16,8 @@ #ifndef fdb5_Root_H #define fdb5_Root_H -#include #include +#include #include "eckit/filesystem/PathName.h" @@ -29,22 +29,15 @@ class Key; //---------------------------------------------------------------------------------------------------------------------- - -class Root { +class Root { public: // methods - - Root(const std::string& path, - const std::string& filespace, - bool list, - bool retrieve, - bool archive, - bool wipe); + Root(const std::string& path, const std::string& filespace, bool list, bool retrieve, bool archive, bool wipe); // Root(const Root&) = default; // Root& operator=(const Root&) = default; - const eckit::PathName &path() const; + const eckit::PathName& path() const; /// Root exists in the filesystem, use this check to avoid errors when accessing /// This result is computed with a lazy approach and then memoized @@ -55,20 +48,18 @@ class Root { }; const ControlIdentifiers& controlIdentifiers() const { return controlIdentifiers_; } - + const std::string& filespace() const; - friend std::ostream& operator<<(std::ostream &s, const Root& x) { + friend std::ostream& operator<<(std::ostream& s, const Root& x) { x.print(s); return s; } private: // methods - - void print( std::ostream &out ) const; + void print(std::ostream& out) const; private: // members - eckit::PathName path_; std::string filespace_; diff --git a/src/fdb5/toc/RootManager.cc b/src/fdb5/toc/RootManager.cc index cef3a7fc7..17b13a170 100644 --- a/src/fdb5/toc/RootManager.cc +++ b/src/fdb5/toc/RootManager.cc @@ -10,23 +10,23 @@ #include "RootManager.h" -#include #include +#include -#include "eckit/types/Types.h" #include "eckit/config/Resource.h" -#include "eckit/utils/Tokenizer.h" +#include "eckit/filesystem/LocalPathName.h" +#include "eckit/thread/AutoLock.h" +#include "eckit/thread/Mutex.h" +#include "eckit/types/Types.h" #include "eckit/utils/StringTools.h" +#include "eckit/utils/Tokenizer.h" #include "eckit/utils/Translator.h" -#include "eckit/thread/Mutex.h" -#include "eckit/thread/AutoLock.h" -#include "eckit/filesystem/LocalPathName.h" #include "metkit/mars/MarsRequest.h" +#include "fdb5/LibFdb5.h" #include "fdb5/config/Config.h" #include "fdb5/database/Key.h" -#include "fdb5/LibFdb5.h" #include "fdb5/rules/Schema.h" #include "fdb5/toc/FileSpace.h" #include "fdb5/toc/Root.h" @@ -34,13 +34,19 @@ using namespace eckit; namespace fdb5 { - class DbPathNamer; - class FileSpace; -} +class DbPathNamer; +class FileSpace; +} // namespace fdb5 namespace eckit { - template <> struct VectorPrintSelector { typedef VectorPrintSimple selector; }; - template <> struct VectorPrintSelector { typedef VectorPrintSimple selector; }; -} +template <> +struct VectorPrintSelector { + typedef VectorPrintSimple selector; +}; +template <> +struct VectorPrintSelector { + typedef VectorPrintSimple selector; +}; +} // namespace eckit namespace fdb5 { @@ -48,9 +54,7 @@ namespace fdb5 { class DbPathNamer { public: - - DbPathNamer(const std::string& keyregex, const std::string& format) : - format_(format){ + DbPathNamer(const std::string& keyregex, const std::string& format) : format_(format) { crack(keyregex); LOG_DEBUG_LIB(LibFdb5) << "Building " << *this << std::endl; } @@ -59,18 +63,19 @@ class DbPathNamer { /// but partial match for values bool match(const Key& k, const char* missing = 0) const { - if(k.size() != keyregex_.size()) return false; + if (k.size() != keyregex_.size()) + return false; - for(Key::const_iterator i = k.begin(); i != k.end(); ++i) { + for (Key::const_iterator i = k.begin(); i != k.end(); ++i) { std::map::const_iterator j = keyregex_.find(i->first); - if(j == keyregex_.end()) { + if (j == keyregex_.end()) { return false; } - if(!missing || i->second != missing) { - if(!j->second.match(i->second)) { + if (!missing || i->second != missing) { + if (!j->second.match(i->second)) { return false; } } @@ -79,21 +84,16 @@ class DbPathNamer { return true; } - std::string name(const Key& key) const { - return substituteVars(format_, key); - } + std::string name(const Key& key) const { return substituteVars(format_, key); } - std::string namePartial(const Key& key, const char* missing) const { - return substituteVars(format_, key, missing); - } + std::string namePartial(const Key& key, const char* missing) const { return substituteVars(format_, key, missing); } - friend std::ostream& operator<<(std::ostream &s, const DbPathNamer& x) { + friend std::ostream& operator<<(std::ostream& s, const DbPathNamer& x) { x.print(s); return s; } private: // methods - void crack(const std::string& regexstr) { eckit::Tokenizer parse1(","); @@ -107,14 +107,12 @@ class DbPathNamer { eckit::StringList kv; parse2(*i, kv); - if(kv.size() == 2) { + if (kv.size() == 2) { keyregex_[kv[0]] = kv[1]; - } - else { - if(kv.size() == 1) { + } else { + if (kv.size() == 1) { keyregex_[kv[0]] = std::string("[^:/]*"); - } - else { + } else { std::ostringstream msg; msg << "Malformed regex in DbNamer " << regexstr; throw BadValue(msg.str(), Here()); @@ -123,22 +121,19 @@ class DbPathNamer { } } - std::string substituteVars(const std::string& s, const Key& k, const char * missing = 0) const - { + std::string substituteVars(const std::string& s, const Key& k, const char* missing = 0) const { std::string result; size_t len = s.length(); bool var = false; std::string word; - std::map::const_iterator j; + std::map::const_iterator j; - for(size_t i = 0; i < len; i++) - { - switch(s[i]) - { + for (size_t i = 0; i < len; i++) { + switch (s[i]) { case '{': - if(var) { + if (var) { std::ostringstream os; - os << "FDB RootManager substituteVars: unexpected { found in " <second; // we know it exists because it is ensured in match() - } - else - { + } else { + if ((*j).second == missing || (*j).second.empty()) { + result += + keyregex_.find(word)->second; // we know it exists because it is ensured in match() + } else { result += (*j).second; } } - } - else { + } else { std::ostringstream os; - os << "FDB RootManager substituteVars: cannot find a value for '" << word << "' in " < keyregex_; std::string format_; - }; typedef std::vector DbPathNamerTable; @@ -209,7 +201,8 @@ static DbPathNamerMap dbPathNamers; static const DbPathNamerTable& readDbNamers(const Config& config) { - static std::string fdbDbNamesFile = eckit::Resource("fdbDbNamesFile;$FDB_DBNAMES_FILE", "~fdb/etc/fdb/dbnames"); + static std::string fdbDbNamesFile = + eckit::Resource("fdbDbNamesFile;$FDB_DBNAMES_FILE", "~fdb/etc/fdb/dbnames"); eckit::PathName filename(config.expandPath(fdbDbNamesFile)); @@ -226,7 +219,7 @@ static const DbPathNamerTable& readDbNamers(const Config& config) { DbPathNamerTable& table(dbPathNamers[filename]); - if(filename.exists()) { + if (filename.exists()) { LOG_DEBUG_LIB(LibFdb5) << "Loading FDB DBPathNames from " << fdbDbNamesFile << std::endl; @@ -260,8 +253,8 @@ static const DbPathNamerTable& readDbNamers(const Config& config) { switch (s.size()) { case 2: { - const std::string& regex = s[0]; - const std::string& format = s[1]; + const std::string& regex = s[0]; + const std::string& format = s[1]; table.push_back(DbPathNamer(regex, format)); break; @@ -269,7 +262,7 @@ static const DbPathNamerTable& readDbNamers(const Config& config) { default: eckit::Log::warning() << "FDB DBPathNames invalid line ignored: " << line << std::endl; - break; + break; } } } @@ -279,7 +272,6 @@ static const DbPathNamerTable& readDbNamers(const Config& config) { //---------------------------------------------------------------------------------------------------------------------- - typedef std::map FileSpaceMap; eckit::Mutex fileSpacesMutex; @@ -298,7 +290,7 @@ static std::vector readRoots(const eckit::PathName& fdbRootsFile) { return result; } - eckit::Translator str2bool; + eckit::Translator str2bool; eckit::Tokenizer parse(" "); @@ -323,19 +315,19 @@ static std::vector readRoots(const eckit::PathName& fdbRootsFile) { switch (s.size()) { case 4: { - const std::string& path = s[0]; - const std::string& filespace = s[1]; - bool writable = str2bool(s[2]); - bool visit = str2bool(s[3]); + const std::string& path = s[0]; + const std::string& filespace = s[1]; + bool writable = str2bool(s[2]); + bool visit = str2bool(s[3]); // list retrieve archive wipe - result.push_back(Root(path, filespace, visit, visit, writable, writable)); + result.push_back(Root(path, filespace, visit, visit, writable, writable)); break; } - default: - eckit::Log::warning() << "FDB RootManager: Invalid line ignored: " << line << std::endl; - break; + default: + eckit::Log::warning() << "FDB RootManager: Invalid line ignored: " << line << std::endl; + break; } } @@ -346,7 +338,7 @@ static std::vector fileSpaceRoots(const std::vector& all, const std: std::vector roots; - for (std::vector::const_iterator i = all.begin(); i != all.end() ; ++i) { + for (std::vector::const_iterator i = all.begin(); i != all.end(); ++i) { if (i->filespace() == filespace) { roots.push_back(*i); } @@ -403,10 +395,12 @@ static FileSpaceTable parseFileSpacesFile(const eckit::PathName& fdbHome) { return it->second; } - eckit::PathName fdbRootsFile = eckit::Resource("fdbRootsFile;$FDB_ROOTS_FILE", fdbHome / "etc/fdb/roots"); + eckit::PathName fdbRootsFile = + eckit::Resource("fdbRootsFile;$FDB_ROOTS_FILE", fdbHome / "etc/fdb/roots"); std::vector allRoots = readRoots(fdbRootsFile); - eckit::PathName fdbSpacesFile = eckit::Resource("fdbSpacesFile;$FDB_SPACES_FILE", fdbHome / "etc/fdb/spaces"); + eckit::PathName fdbSpacesFile = + eckit::Resource("fdbSpacesFile;$FDB_SPACES_FILE", fdbHome / "etc/fdb/spaces"); std::ifstream in(fdbSpacesFile.localPath()); LOG_DEBUG_LIB(LibFdb5) << "Loading FDB file spaces from " << fdbSpacesFile << std::endl; @@ -440,13 +434,13 @@ static FileSpaceTable parseFileSpacesFile(const eckit::PathName& fdbHome) { switch (s.size()) { case 3: { - const std::string& regex = s[0]; + const std::string& regex = s[0]; const std::string& filespace = s[1]; - const std::string& handler = s[2]; + const std::string& handler = s[2]; std::vector roots = fileSpaceRoots(allRoots, filespace); - if(!roots.size()) { + if (!roots.size()) { std::ostringstream oss; oss << "No roots found for filespace " << filespace; throw UserError(oss.str(), Here()); @@ -456,10 +450,9 @@ static FileSpaceTable parseFileSpacesFile(const eckit::PathName& fdbHome) { break; } - default: - eckit::Log::warning() << "FDB RootManager: Invalid line ignored: " << line << std::endl; - break; - + default: + eckit::Log::warning() << "FDB RootManager: Invalid line ignored: " << line << std::endl; + break; } } @@ -490,12 +483,10 @@ FileSpaceTable RootManager::fileSpaces() { if (!fdbRootDirectory.empty()) { std::vector spaceRoots; - spaceRoots.emplace_back( - Root(fdbRootDirectory, "", true, true, true, true)); + spaceRoots.emplace_back(Root(fdbRootDirectory, "", true, true, true, true)); FileSpaceTable table; - table.emplace_back( - FileSpace("", ".*", "Default", spaceRoots)); + table.emplace_back(FileSpace("", ".*", "Default", spaceRoots)); return table; } @@ -511,33 +502,19 @@ FileSpaceTable RootManager::fileSpaces() { if (space.getBool("marsDisks", false)) { PathName file = config_.expandPath(space.getString("path", "~fdb/etc/disks/fdb")); spaceRoots = parseMarsDisks(file, name); - } - else { + } else { std::vector roots = getSpaceRoots(space); for (const auto& root : roots) { bool writable = root.getBool("writable", true); bool visit = root.getBool("visit", true); - spaceRoots.emplace_back( - Root( - root.getString("path"), - root.getString("name", ""), - root.getBool("list", visit), - root.getBool("retrieve", visit), - root.getBool("archive", writable), - root.getBool("wipe", writable) - ) - ); + spaceRoots.emplace_back(Root(root.getString("path"), root.getString("name", ""), + root.getBool("list", visit), root.getBool("retrieve", visit), + root.getBool("archive", writable), root.getBool("wipe", writable))); } } table.emplace_back( - FileSpace( - name, - space.getString("regex", ".*"), - space.getString("handler", "Default"), - spaceRoots - ) - ); + FileSpace(name, space.getString("regex", ".*"), space.getString("handler", "Default"), spaceRoots)); } return table; } else { @@ -545,37 +522,33 @@ FileSpaceTable RootManager::fileSpaces() { } } - //---------------------------------------------------------------------------------------------------------------------- -RootManager::RootManager(const Config& config) : - dbPathNamers_(readDbNamers(config)), - config_(config) {} +RootManager::RootManager(const Config& config) : dbPathNamers_(readDbNamers(config)), config_(config) {} -std::string RootManager::dbPathName(const Key& key) -{ +std::string RootManager::dbPathName(const Key& key) { std::string dbpath; - for (DbPathNamerTable::const_iterator i = dbPathNamers_.begin(); i != dbPathNamers_.end() ; ++i) { - if(i->match(key)) { + for (DbPathNamerTable::const_iterator i = dbPathNamers_.begin(); i != dbPathNamers_.end(); ++i) { + if (i->match(key)) { dbpath = i->name(key); - LOG_DEBUG_LIB(LibFdb5) << "DbName is " << dbpath << " for key " << key << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "DbName is " << dbpath << " for key " << key << std::endl; return dbpath; } } // default naming convention for DB's dbpath = key.valuesToString(); - LOG_DEBUG_LIB(LibFdb5) << "Using default naming convention for key " << key << " -> " << dbpath << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "Using default naming convention for key " << key << " -> " << dbpath << std::endl; return dbpath; } -std::vector RootManager::possibleDbPathNames(const Key& key, const char* missing) -{ +std::vector RootManager::possibleDbPathNames(const Key& key, const char* missing) { std::vector result; - for (DbPathNamerTable::const_iterator i = dbPathNamers_.begin(); i != dbPathNamers_.end() ; ++i) { - if(i->match(key, missing)) { + for (DbPathNamerTable::const_iterator i = dbPathNamers_.begin(); i != dbPathNamers_.end(); ++i) { + if (i->match(key, missing)) { std::string dbpath = i->namePartial(key, missing); - LOG_DEBUG_LIB(LibFdb5) << "Matched " << *i << " with key " << key << " resulting in dbpath " << dbpath << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "Matched " << *i << " with key " << key << " resulting in dbpath " << dbpath + << std::endl; result.push_back(dbpath); } } @@ -583,7 +556,7 @@ std::vector RootManager::possibleDbPathNames(const Key& key, const // default naming convention for DB's std::ostringstream oss; - const char *sep = ""; + const char* sep = ""; for (auto& k : key.names()) { auto& v = key.get(k); @@ -593,12 +566,11 @@ std::vector RootManager::possibleDbPathNames(const Key& key, const } result.push_back(oss.str()); - LOG_DEBUG_LIB(LibFdb5) << "Using default naming convention for key " << key << " -> " << result.back() << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "Using default naming convention for key " << key << " -> " << result.back() << std::endl; return result; } - TocPath RootManager::directory(const Key& key) { PathName dbpath = dbPathName(key); @@ -609,7 +581,7 @@ TocPath RootManager::directory(const Key& key) { static std::string fdbRootDirectory = eckit::Resource("fdbRootDirectory;$FDB_ROOT_DIRECTORY", ""); - if(!fdbRootDirectory.empty()) { + if (!fdbRootDirectory.empty()) { return TocPath{fdbRootDirectory + "/" + dbpath, ControlIdentifiers{}}; } @@ -617,8 +589,8 @@ TocPath RootManager::directory(const Key& key) { std::string keystr = key.valuesToString(); - for (FileSpaceTable::const_iterator i = spacesTable_.begin(); i != spacesTable_.end() ; ++i) { - if(i->match(keystr)) { + for (FileSpaceTable::const_iterator i = spacesTable_.begin(); i != spacesTable_.end(); ++i) { + if (i->match(keystr)) { TocPath db = i->filesystem(key, dbpath); LOG_DEBUG_LIB(LibFdb5) << "Database directory " << db.directory_ << std::endl; return db; @@ -646,13 +618,14 @@ std::vector RootManager::visitableRoots(const std::set& keys) { for (const std::string& k : keystrings) { if (space.match(k) || k.empty()) { LOG_DEBUG_LIB(LibFdb5) << "MATCH space " << space << std::endl; - space.enabled(ControlIdentifier::List,roots); + space.enabled(ControlIdentifier::List, roots); matched = true; break; } } - if (!matched) LOG_DEBUG_LIB(LibFdb5) << "FAIL to match space " << space << std::endl; + if (!matched) + LOG_DEBUG_LIB(LibFdb5) << "FAIL to match space " << space << std::endl; } LOG_DEBUG_LIB(LibFdb5) << "Visitable Roots " << roots << std::endl; @@ -660,9 +633,8 @@ std::vector RootManager::visitableRoots(const std::set& keys) { return std::vector(roots.begin(), roots.end()); } - std::vector RootManager::visitableRoots(const Key& key) { - return visitableRoots(std::set{ key }); + return visitableRoots(std::set{key}); } std::vector RootManager::visitableRoots(const metkit::mars::MarsRequest& request) { @@ -672,15 +644,14 @@ std::vector RootManager::visitableRoots(const metkit::mars::Mar return visitableRoots(keys); } - std::vector RootManager::canArchiveRoots(const Key& key) { eckit::StringSet roots; std::string k = key.valuesToString(); - for (FileSpaceTable::const_iterator i = spacesTable_.begin(); i != spacesTable_.end() ; ++i) { - if(i->match(k)) { + for (FileSpaceTable::const_iterator i = spacesTable_.begin(); i != spacesTable_.end(); ++i) { + if (i->match(k)) { i->enabled(ControlIdentifier::Archive, roots); } @@ -697,8 +668,8 @@ std::vector RootManager::canMoveToRoots(const Key& key) { std::string k = key.valuesToString(); - for (FileSpaceTable::const_iterator i = spacesTable_.begin(); i != spacesTable_.end() ; ++i) { - if(i->match(k)) { + for (FileSpaceTable::const_iterator i = spacesTable_.begin(); i != spacesTable_.end(); ++i) { + if (i->match(k)) { i->enabled(ControlIdentifier::Wipe, roots); } @@ -711,4 +682,4 @@ std::vector RootManager::canMoveToRoots(const Key& key) { //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/toc/RootManager.h b/src/fdb5/toc/RootManager.h index 30ed7b858..c6cb5f807 100644 --- a/src/fdb5/toc/RootManager.h +++ b/src/fdb5/toc/RootManager.h @@ -16,18 +16,18 @@ #define fdb5_RootManager_H #include "eckit/config/LocalConfiguration.h" -#include "eckit/utils/Regex.h" #include "eckit/filesystem/PathName.h" +#include "eckit/utils/Regex.h" +#include "fdb5/api/helpers/ControlIterator.h" #include "fdb5/config/Config.h" #include "fdb5/toc/FileSpace.h" -#include "fdb5/api/helpers/ControlIterator.h" namespace metkit { namespace mars { - class MarsRequest; -} +class MarsRequest; } +} // namespace metkit namespace fdb5 { @@ -39,10 +39,9 @@ class DbPathNamer; typedef std::vector FileSpaceTable; -class RootManager { +class RootManager { public: // methods - RootManager(const Config& config); /// Uniquely selects a directory where the Key will be put or already exists @@ -65,16 +64,13 @@ class RootManager { std::vector possibleDbPathNames(const Key& key, const char* missing); protected: // methods - virtual FileSpaceTable fileSpaces(); virtual std::vector getSpaceRoots(const eckit::LocalConfiguration& space) = 0; protected: // members - std::vector spacesTable_; private: // members - const std::vector& dbPathNamers_; Config config_; }; @@ -82,28 +78,18 @@ class RootManager { class CatalogueRootManager : public RootManager { public: // methods - - CatalogueRootManager(const Config& config) : - RootManager(config) { - spacesTable_ = fileSpaces(); - } + CatalogueRootManager(const Config& config) : RootManager(config) { spacesTable_ = fileSpaces(); } protected: // methods - std::vector getSpaceRoots(const eckit::LocalConfiguration& space) override; }; class StoreRootManager : public RootManager { public: // methods - - StoreRootManager(const Config& config) : - RootManager(config) { - spacesTable_ = fileSpaces(); - } + StoreRootManager(const Config& config) : RootManager(config) { spacesTable_ = fileSpaces(); } protected: // methods - std::vector getSpaceRoots(const eckit::LocalConfiguration& space) override; }; diff --git a/src/fdb5/toc/TocCatalogue.cc b/src/fdb5/toc/TocCatalogue.cc index 5fa12a5d0..71e4c40ce 100644 --- a/src/fdb5/toc/TocCatalogue.cc +++ b/src/fdb5/toc/TocCatalogue.cc @@ -14,10 +14,10 @@ #include "fdb5/rules/Rule.h" #include "fdb5/toc/RootManager.h" #include "fdb5/toc/TocCatalogue.h" +#include "fdb5/toc/TocMoveVisitor.h" #include "fdb5/toc/TocPurgeVisitor.h" #include "fdb5/toc/TocStats.h" #include "fdb5/toc/TocWipeVisitor.h" -#include "fdb5/toc/TocMoveVisitor.h" using namespace eckit; @@ -25,28 +25,24 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -TocCatalogue::TocCatalogue(const Key& key, const fdb5::Config& config) : - TocCatalogue(key, CatalogueRootManager(config).directory(key), config) { -} +TocCatalogue::TocCatalogue(const Key& key, const fdb5::Config& config) + : TocCatalogue(key, CatalogueRootManager(config).directory(key), config) {} -TocCatalogue::TocCatalogue(const Key& key, const TocPath& tocPath, const fdb5::Config& config) : - CatalogueImpl(key, tocPath.controlIdentifiers_, config), - TocHandler(tocPath.directory_, config) { -} +TocCatalogue::TocCatalogue(const Key& key, const TocPath& tocPath, const fdb5::Config& config) + : CatalogueImpl(key, tocPath.controlIdentifiers_, config) + , TocHandler(tocPath.directory_, config) {} -TocCatalogue::TocCatalogue(const eckit::PathName& directory, const ControlIdentifiers& controlIdentifiers, const fdb5::Config& config) : - CatalogueImpl(Key(), controlIdentifiers, config), - TocHandler(directory, config) { +TocCatalogue::TocCatalogue(const eckit::PathName& directory, const ControlIdentifiers& controlIdentifiers, + const fdb5::Config& config) + : CatalogueImpl(Key(), controlIdentifiers, config) + , TocHandler(directory, config) { // Read the real DB key into the DB base object dbKey_ = databaseKey(); } - -std::vector TocCatalogue::loadIndexes(bool sorted, - std::set* subTocs, - std::vector* indexInSubtoc, - std::vector* remapKeys) const { - return TocHandler::loadIndexes(*this, sorted, subTocs, indexInSubtoc,remapKeys); +std::vector TocCatalogue::loadIndexes(bool sorted, std::set* subTocs, + std::vector* indexInSubtoc, std::vector* remapKeys) const { + return TocHandler::loadIndexes(*this, sorted, subTocs, indexInSubtoc, remapKeys); } bool TocCatalogue::exists() const { @@ -93,19 +89,21 @@ StatsReportVisitor* TocCatalogue::statsReportVisitor() const { return new TocStatsReportVisitor(*this); } -PurgeVisitor *TocCatalogue::purgeVisitor(const Store& store) const { +PurgeVisitor* TocCatalogue::purgeVisitor(const Store& store) const { return new TocPurgeVisitor(*this, store); } -WipeVisitor* TocCatalogue::wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, bool doit, bool porcelain, bool unsafeWipeAll) const { +WipeVisitor* TocCatalogue::wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, + bool doit, bool porcelain, bool unsafeWipeAll) const { return new TocWipeVisitor(*this, store, request, out, doit, porcelain, unsafeWipeAll); } -MoveVisitor* TocCatalogue::moveVisitor(const Store& store, const metkit::mars::MarsRequest& request, const eckit::URI& dest, eckit::Queue& queue) const { +MoveVisitor* TocCatalogue::moveVisitor(const Store& store, const metkit::mars::MarsRequest& request, + const eckit::URI& dest, eckit::Queue& queue) const { return new TocMoveVisitor(*this, store, request, dest, queue); } -void TocCatalogue::maskIndexEntry(const Index &index) const { +void TocCatalogue::maskIndexEntry(const Index& index) const { TocHandler handler(basePath(), config_); handler.writeClearRecord(index); } @@ -114,13 +112,11 @@ std::vector TocCatalogue::indexes(bool sorted) const { return loadIndexes(sorted); } -void TocCatalogue::allMasked(std::set>& metadata, - std::set& data) const { +void TocCatalogue::allMasked(std::set>& metadata, std::set& data) const { enumerateMasked(*this, metadata, data); } -std::string TocCatalogue::type() const -{ +std::string TocCatalogue::type() const { return TocEngine::typeName(); } @@ -132,11 +128,13 @@ void TocCatalogue::remove(const eckit::PathName& path, std::ostream& logAlways, if (path.isDir()) { logVerbose << "rmdir: "; logAlways << path << std::endl; - if (doit) path.rmdir(false); + if (doit) + path.rmdir(false); } else { logVerbose << "Unlinking: "; logAlways << path << std::endl; - if (doit) path.unlink(false); + if (doit) + path.unlink(false); } } diff --git a/src/fdb5/toc/TocCatalogue.h b/src/fdb5/toc/TocCatalogue.h index 9088853bf..98b56669f 100644 --- a/src/fdb5/toc/TocCatalogue.h +++ b/src/fdb5/toc/TocCatalogue.h @@ -20,8 +20,8 @@ #include "fdb5/database/Index.h" #include "fdb5/rules/Schema.h" #include "fdb5/toc/FileSpace.h" -#include "fdb5/toc/TocHandler.h" #include "fdb5/toc/TocEngine.h" +#include "fdb5/toc/TocHandler.h" namespace fdb5 { @@ -32,9 +32,9 @@ namespace fdb5 { class TocCatalogue : public CatalogueImpl, public TocHandler { public: // methods - TocCatalogue(const Key& dbKey, const fdb5::Config& config); - TocCatalogue(const eckit::PathName& directory, const ControlIdentifiers& controlIdentifiers, const fdb5::Config& config); + TocCatalogue(const eckit::PathName& directory, const ControlIdentifiers& controlIdentifiers, + const fdb5::Config& config); ~TocCatalogue() override {} @@ -49,13 +49,11 @@ class TocCatalogue : public CatalogueImpl, public TocHandler { static const std::string DUMP_PARAM_WALKSUBTOC; protected: // methods - TocCatalogue(const Key& dbKey, const TocPath& tocPath, const fdb5::Config& config); std::string type() const override; - std::vector loadIndexes(bool sorted=false, - std::set* subTocs = nullptr, + std::vector loadIndexes(bool sorted = false, std::set* subTocs = nullptr, std::vector* indexInSubtoc = nullptr, std::vector* remapKeys = nullptr) const; @@ -67,26 +65,25 @@ class TocCatalogue : public CatalogueImpl, public TocHandler { StatsReportVisitor* statsReportVisitor() const override; PurgeVisitor* purgeVisitor(const Store& store) const override; - WipeVisitor* wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, bool doit, bool porcelain, bool unsafeWipeAll) const override; - MoveVisitor* moveVisitor(const Store& store, const metkit::mars::MarsRequest& request, const eckit::URI& dest, eckit::Queue& queue) const override; + WipeVisitor* wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, bool doit, + bool porcelain, bool unsafeWipeAll) const override; + MoveVisitor* moveVisitor(const Store& store, const metkit::mars::MarsRequest& request, const eckit::URI& dest, + eckit::Queue& queue) const override; void maskIndexEntry(const Index& index) const override; void loadSchema() override; - std::vector indexes(bool sorted=false) const override; + std::vector indexes(bool sorted = false) const override; - void allMasked(std::set>& metadata, - std::set& data) const override; + void allMasked(std::set>& metadata, std::set& data) const override; // Control access properties of the DB void control(const ControlAction& action, const ControlIdentifiers& identifiers) const override; protected: // members - Key currentIndexKey_; private: // members - friend class TocWipeVisitor; friend class TocMoveVisitor; diff --git a/src/fdb5/toc/TocCatalogueReader.cc b/src/fdb5/toc/TocCatalogueReader.cc index 0649c6873..1fd638b09 100644 --- a/src/fdb5/toc/TocCatalogueReader.cc +++ b/src/fdb5/toc/TocCatalogueReader.cc @@ -13,22 +13,21 @@ #include "eckit/log/Log.h" #include "fdb5/LibFdb5.h" +#include "fdb5/toc/RootManager.h" #include "fdb5/toc/TocCatalogueReader.h" #include "fdb5/toc/TocIndex.h" #include "fdb5/toc/TocStats.h" -#include "fdb5/toc/RootManager.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -TocCatalogueReader::TocCatalogueReader(const Key& dbKey, const fdb5::Config& config) : - TocCatalogue(dbKey, config) { +TocCatalogueReader::TocCatalogueReader(const Key& dbKey, const fdb5::Config& config) : TocCatalogue(dbKey, config) { loadIndexesAndRemap(); } -TocCatalogueReader::TocCatalogueReader(const eckit::URI& uri, const fdb5::Config& config) : - TocCatalogue(uri.path(), ControlIdentifiers{}, config) { +TocCatalogueReader::TocCatalogueReader(const eckit::URI& uri, const fdb5::Config& config) + : TocCatalogue(uri.path(), ControlIdentifiers{}, config) { loadIndexesAndRemap(); } @@ -50,7 +49,7 @@ void TocCatalogueReader::loadIndexesAndRemap() { bool TocCatalogueReader::selectIndex(const Key& idxKey) { - if(currentIndexKey_ == idxKey) { + if (currentIndexKey_ == idxKey) { return true; } @@ -63,8 +62,8 @@ bool TocCatalogueReader::selectIndex(const Key& idxKey) { } } - LOG_DEBUG_LIB(LibFdb5) << "TocCatalogueReader::selectIndex " << idxKey << ", found " - << matching_.size() << " matche(s)" << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "TocCatalogueReader::selectIndex " << idxKey << ", found " << matching_.size() + << " matche(s)" << std::endl; return (matching_.size() != 0); } @@ -87,7 +86,7 @@ bool TocCatalogueReader::open() { return true; } -bool TocCatalogueReader::axis(const std::string &keyword, eckit::StringSet &s) const { +bool TocCatalogueReader::axis(const std::string& keyword, eckit::StringSet& s) const { bool found = false; for (auto m = matching_.begin(); m != matching_.end(); ++m) { if ((*m)->first.axes().has(keyword)) { @@ -123,7 +122,7 @@ bool TocCatalogueReader::retrieve(const Key& key, Field& field) const { return false; } -void TocCatalogueReader::print(std::ostream &out) const { +void TocCatalogueReader::print(std::ostream& out) const { out << "TocCatalogueReader(" << directory() << ")"; } diff --git a/src/fdb5/toc/TocCatalogueReader.h b/src/fdb5/toc/TocCatalogueReader.h index be0c17a29..37480cfaa 100644 --- a/src/fdb5/toc/TocCatalogueReader.h +++ b/src/fdb5/toc/TocCatalogueReader.h @@ -27,7 +27,6 @@ namespace fdb5 { class TocCatalogueReader : public TocCatalogue, public CatalogueReader { public: // methods - TocCatalogueReader(const Key& dbKey, const fdb5::Config& config); TocCatalogueReader(const eckit::URI& uri, const fdb5::Config& config); @@ -37,7 +36,6 @@ class TocCatalogueReader : public TocCatalogue, public CatalogueReader { DbStats stats() const override { return TocHandler::stats(); } private: // methods - void loadIndexesAndRemap(); bool selectIndex(const Key& idxKey) override; void deselectIndex() override; @@ -46,15 +44,14 @@ class TocCatalogueReader : public TocCatalogue, public CatalogueReader { void flush(size_t archivedFields) override {} void clean() override {} void close() override; - - bool axis(const std::string &keyword, eckit::StringSet &s) const override; + + bool axis(const std::string& keyword, eckit::StringSet& s) const override; bool retrieve(const Key& key, Field& field) const override; - void print( std::ostream &out ) const override; + void print(std::ostream& out) const override; private: // members - // Indexes matching current key. If there is a key remapping for a mounted // SubToc, then this is stored alongside std::vector*> matching_; @@ -62,7 +59,6 @@ class TocCatalogueReader : public TocCatalogue, public CatalogueReader { // All indexes // If there is a key remapping for a mounted SubToc, this is stored alongside std::vector> indexes_; - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/toc/TocCatalogueWriter.cc b/src/fdb5/toc/TocCatalogueWriter.cc index 0099af0b2..89ede453a 100644 --- a/src/fdb5/toc/TocCatalogueWriter.cc +++ b/src/fdb5/toc/TocCatalogueWriter.cc @@ -11,18 +11,18 @@ #include "fdb5/fdb5_config.h" #include "eckit/config/Resource.h" -#include "eckit/log/Log.h" -#include "eckit/log/Bytes.h" #include "eckit/io/EmptyHandle.h" +#include "eckit/log/Bytes.h" +#include "eckit/log/Log.h" +#include "fdb5/LibFdb5.h" #include "fdb5/database/EntryVisitMechanism.h" #include "fdb5/io/FDBFileHandle.h" -#include "fdb5/LibFdb5.h" +#include "fdb5/io/LustreSettings.h" +#include "fdb5/toc/RootManager.h" #include "fdb5/toc/TocCatalogueWriter.h" #include "fdb5/toc/TocFieldLocation.h" #include "fdb5/toc/TocIndex.h" -#include "fdb5/toc/RootManager.h" -#include "fdb5/io/LustreSettings.h" using namespace eckit; @@ -30,20 +30,19 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- - -TocCatalogueWriter::TocCatalogueWriter(const Key& dbKey, const fdb5::Config& config) : - TocCatalogue(dbKey, config), - umask_(config.umask()), - archivedLocations_(0) { +TocCatalogueWriter::TocCatalogueWriter(const Key& dbKey, const fdb5::Config& config) + : TocCatalogue(dbKey, config) + , umask_(config.umask()) + , archivedLocations_(0) { writeInitRecord(dbKey); TocCatalogue::loadSchema(); TocCatalogue::checkUID(); } -TocCatalogueWriter::TocCatalogueWriter(const eckit::URI &uri, const fdb5::Config& config) : - TocCatalogue(uri.path(), ControlIdentifiers{}, config), - umask_(config.umask()), - archivedLocations_(0) { +TocCatalogueWriter::TocCatalogueWriter(const eckit::URI& uri, const fdb5::Config& config) + : TocCatalogue(uri.path(), ControlIdentifiers{}, config) + , umask_(config.umask()) + , archivedLocations_(0) { writeInitRecord(TocCatalogue::key()); TocCatalogue::loadSchema(); TocCatalogue::checkUID(); @@ -125,7 +124,7 @@ void TocCatalogueWriter::close() { closeIndexes(); } -void TocCatalogueWriter::index(const Key& key, const eckit::URI &uri, eckit::Offset offset, eckit::Length length) { +void TocCatalogueWriter::index(const Key& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) { archivedLocations_++; if (current_.null()) { @@ -151,16 +150,15 @@ void TocCatalogueWriter::reconsolidateIndexesAndTocs() { class ConsolidateIndexVisitor : public EntryVisitor { public: - ConsolidateIndexVisitor(TocCatalogueWriter& writer) : - writer_(writer) {} + ConsolidateIndexVisitor(TocCatalogueWriter& writer) : writer_(writer) {} ~ConsolidateIndexVisitor() override {} + private: void visitDatum(const Field& field, const Key& datumKey) override { // TODO: Do a sneaky schema.expand() here, prepopulated with the current DB/index/Rule, // to extract the full key, including optional values. const TocFieldLocation& location(static_cast(field.location())); writer_.index(datumKey, location.uri(), location.offset(), location.length()); - } void visitDatum(const Field& field, const std::string& keyFingerprint) override { EntryVisitor::visitDatum(field, keyFingerprint); @@ -188,7 +186,8 @@ void TocCatalogueWriter::reconsolidateIndexesAndTocs() { Log::info() << "Visiting index: " << idx.location().uri() << std::endl; // We need to explicitly mask indexes in the master TOC - if (!indexInSubtoc[i]) maskable_indexes += 1; + if (!indexInSubtoc[i]) + maskable_indexes += 1; } // Flush the new indexes and add relevant entries! @@ -265,7 +264,8 @@ void TocCatalogueWriter::overlayDB(const Catalogue& otherCat, const std::setsecond) { if (variableKeys.find(kv.first) == variableKeys.end()) { std::stringstream ss; - ss << "Key " << kv.first << " not allowed to differ between DBs: " << TocCatalogue::dbKey_ << " : " << otherKey; + ss << "Key " << kv.first << " not allowed to differ between DBs: " << TocCatalogue::dbKey_ << " : " + << otherKey; throw UserError(ss.str(), Here()); } } @@ -303,15 +303,17 @@ bool TocCatalogueWriter::enabled(const ControlIdentifier& controlIdentifier) con return TocCatalogue::enabled(controlIdentifier); } -void TocCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) { +void TocCatalogueWriter::archive(const Key& idxKey, const Key& datumKey, + std::shared_ptr fieldLocation) { archivedLocations_++; if (current_.null()) { ASSERT(!currentIndexKey_.empty()); selectIndex(currentIndexKey_); } else { - // in case of async archival (out of order store/catalogue archival), currentIndexKey_ can differ from the indexKey used for store archival. Reset it - if(currentIndexKey_ != idxKey) { + // in case of async archival (out of order store/catalogue archival), currentIndexKey_ can differ from the + // indexKey used for store archival. Reset it + if (currentIndexKey_ != idxKey) { selectIndex(idxKey); } } @@ -339,7 +341,7 @@ void TocCatalogueWriter::flush(size_t archivedFields) { } eckit::PathName TocCatalogueWriter::generateIndexPath(const Key& key) const { - eckit::PathName tocPath ( directory_ ); + eckit::PathName tocPath(directory_); tocPath /= key.valuesToString(); tocPath = eckit::PathName::unique(tocPath) + ".index"; return tocPath; @@ -351,7 +353,7 @@ eckit::PathName TocCatalogueWriter::generateIndexPath(const Key& key) const { // the data that is indexes thorughout the lifetime of the DBWriter, which can be // compacted later for read performance. void TocCatalogueWriter::flushIndexes() { - for (IndexStore::iterator j = indexes_.begin(); j != indexes_.end(); ++j ) { + for (IndexStore::iterator j = indexes_.begin(); j != indexes_.end(); ++j) { Index& idx = j->second; if (idx.dirty()) { @@ -362,19 +364,18 @@ void TocCatalogueWriter::flushIndexes() { } } - void TocCatalogueWriter::closeIndexes() { - for (IndexStore::iterator j = indexes_.begin(); j != indexes_.end(); ++j ) { + for (IndexStore::iterator j = indexes_.begin(); j != indexes_.end(); ++j) { Index& idx = j->second; idx.close(); } - for (IndexStore::iterator j = fullIndexes_.begin(); j != fullIndexes_.end(); ++j ) { + for (IndexStore::iterator j = fullIndexes_.begin(); j != fullIndexes_.end(); ++j) { Index& idx = j->second; idx.close(); } - indexes_.clear(); // all indexes instances destroyed + indexes_.clear(); // all indexes instances destroyed fullIndexes_.clear(); // all indexes instances destroyed } @@ -414,8 +415,7 @@ void TocCatalogueWriter::compactSubTocIndexes() { } } - -void TocCatalogueWriter::print(std::ostream &out) const { +void TocCatalogueWriter::print(std::ostream& out) const { out << "TocCatalogueWriter(" << directory() << ")"; } diff --git a/src/fdb5/toc/TocCatalogueWriter.h b/src/fdb5/toc/TocCatalogueWriter.h index d428e69ad..3ca790f18 100644 --- a/src/fdb5/toc/TocCatalogueWriter.h +++ b/src/fdb5/toc/TocCatalogueWriter.h @@ -36,14 +36,13 @@ class TocAddIndex; class TocCatalogueWriter : public TocCatalogue, public CatalogueWriter { public: // methods - TocCatalogueWriter(const Key& dbKey, const fdb5::Config& config); TocCatalogueWriter(const eckit::URI& uri, const fdb5::Config& config); ~TocCatalogueWriter() override; /// Used for adopting & indexing external data to the TOC dir - void index(const Key& key, const eckit::URI &uri, eckit::Offset offset, eckit::Length length) override; + void index(const Key& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) override; void reconsolidate() override { reconsolidateIndexesAndTocs(); } @@ -62,7 +61,6 @@ class TocCatalogueWriter : public TocCatalogue, public CatalogueWriter { const TocSerialisationVersion& serialisationVersion() const; protected: // methods - bool selectIndex(const Key& idxKey) override; void deselectIndex() override; @@ -74,10 +72,9 @@ class TocCatalogueWriter : public TocCatalogue, public CatalogueWriter { void archive(const Key& idxKey, const Key& datumKey, std::shared_ptr fieldLocation) override; void reconsolidateIndexesAndTocs(); - void print( std::ostream &out ) const override; + void print(std::ostream& out) const override; private: // methods - void closeIndexes(); void flushIndexes(); void compactSubTocIndexes(); @@ -85,22 +82,20 @@ class TocCatalogueWriter : public TocCatalogue, public CatalogueWriter { eckit::PathName generateIndexPath(const Key& key) const; private: // types + typedef std::map HandleStore; + typedef std::map IndexStore; + typedef std::map PathStore; - typedef std::map< std::string, eckit::DataHandle * > HandleStore; - typedef std::map< Key, Index> IndexStore; - typedef std::map< Key, std::string > PathStore; - -private: // members - - HandleStore handles_; ///< stores the DataHandles being used by the Session +private: // members + HandleStore handles_; ///< stores the DataHandles being used by the Session // If we have multiple flush statements, then the indexes get repeatedly reset. Build and maintain // a full copy of the indexes associated with the process as well, for use when masking out // subtocs. See compactSubTocIndexes. - IndexStore indexes_; - IndexStore fullIndexes_; + IndexStore indexes_; + IndexStore fullIndexes_; - PathStore dataPaths_; + PathStore dataPaths_; Index current_; Index currentFull_; diff --git a/src/fdb5/toc/TocCommon.cc b/src/fdb5/toc/TocCommon.cc index 0bcdd373c..ded556935 100644 --- a/src/fdb5/toc/TocCommon.cc +++ b/src/fdb5/toc/TocCommon.cc @@ -14,13 +14,13 @@ #include #include "eckit/config/Resource.h" -#include "eckit/filesystem/URIManager.h" #include "eckit/filesystem/LocalPathName.h" +#include "eckit/filesystem/URIManager.h" #include "eckit/log/Timer.h" #include "fdb5/LibFdb5.h" -#include "fdb5/toc/RootManager.h" #include "fdb5/io/LustreSettings.h" +#include "fdb5/toc/RootManager.h" namespace fdb5 { @@ -28,16 +28,17 @@ eckit::LocalPathName TocCommon::findRealPath(const eckit::LocalPathName& path) { // realpath only works on existing paths, so work back up the path until // we find one that does, get the realpath on that, then reconstruct. - if (path.exists()) return path.realName(); + if (path.exists()) + return path.realName(); return findRealPath(path.dirName()) / path.baseName(); } -TocCommon::TocCommon(const eckit::PathName& directory) : - directory_(findRealPath(eckit::LocalPathName{directory})), - schemaPath_(directory_ / "schema"), - dbUID_(static_cast(-1)), - userUID_(::getuid()) {} +TocCommon::TocCommon(const eckit::PathName& directory) + : directory_(findRealPath(eckit::LocalPathName{directory})) + , schemaPath_(directory_ / "schema") + , dbUID_(static_cast(-1)) + , userUID_(::getuid()) {} void TocCommon::checkUID() const { static bool fdbOnlyCreatorCanWrite = eckit::Resource("fdbOnlyCreatorCanWrite", true); @@ -49,11 +50,9 @@ void TocCommon::checkUID() const { eckit::Resource>("fdbSuperUsers", "", true); if (dbUID() != userUID_) { - if (std::find(fdbSuperUsers.begin(), fdbSuperUsers.end(), userName(userUID_)) == - fdbSuperUsers.end()) { + if (std::find(fdbSuperUsers.begin(), fdbSuperUsers.end(), userName(userUID_)) == fdbSuperUsers.end()) { std::ostringstream oss; - oss << "Only user '" << userName(dbUID()) - << "' can write to FDB " << directory_ << ", current user is '" + oss << "Only user '" << userName(dbUID()) << "' can write to FDB " << directory_ << ", current user is '" << userName(userUID_) << "'"; throw eckit::UserError(oss.str()); @@ -77,10 +76,9 @@ std::string TocCommon::userName(uid_t uid) { if (p) { return p->pw_name; - } - else { + } else { return eckit::Translator()(uid); } } -} \ No newline at end of file +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/toc/TocCommon.h b/src/fdb5/toc/TocCommon.h index aa9db187d..f17d268f4 100644 --- a/src/fdb5/toc/TocCommon.h +++ b/src/fdb5/toc/TocCommon.h @@ -14,21 +14,20 @@ #pragma once -#include "eckit/filesystem/PathName.h" #include "eckit/filesystem/LocalPathName.h" +#include "eckit/filesystem/PathName.h" #include "eckit/io/FileHandle.h" -#include "eckit/thread/ThreadPool.h" #include "eckit/serialisation/Streamable.h" +#include "eckit/thread/ThreadPool.h" +#include "fdb5/api/helpers/ControlIterator.h" #include "fdb5/config/Config.h" #include "fdb5/database/Key.h" -#include "fdb5/api/helpers/ControlIterator.h" namespace fdb5 { class TocCommon { public: - TocCommon(const eckit::PathName& path); virtual ~TocCommon() {} @@ -42,11 +41,9 @@ class TocCommon { std::string owner() const { return userName(dbUID()); } protected: // methods - virtual uid_t dbUID() const; protected: // members - const eckit::LocalPathName directory_; const eckit::LocalPathName schemaPath_; @@ -54,4 +51,4 @@ class TocCommon { uid_t userUID_; }; -} +} // namespace fdb5 diff --git a/src/fdb5/toc/TocEngine.cc b/src/fdb5/toc/TocEngine.cc index 1562baae3..ce507449d 100644 --- a/src/fdb5/toc/TocEngine.cc +++ b/src/fdb5/toc/TocEngine.cc @@ -35,7 +35,6 @@ using namespace eckit; - namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- @@ -44,26 +43,26 @@ void TocEngine::scan_dbs(const std::string& path, std::list& dbs) c eckit::StdDir d(path.c_str()); - // Once readdir_r finally gets deprecated and removed, we may need to + // Once readdir_r finally gets deprecated and removed, we may need to // protecting readdir() as not yet guarranteed thread-safe by POSIX // technically it should only be needed on a per-directory basis // this should be a resursive mutex - // AutoLock lock(mutex_); + // AutoLock lock(mutex_); - for(;;) - { + for (;;) { struct dirent* e = d.dirent(); if (e == nullptr) { break; } - if(e->d_name[0] == '.') { - if(e->d_name[1] == 0 || (e->d_name[1] =='.' && e->d_name[2] == 0)) + if (e->d_name[0] == '.') { + if (e->d_name[1] == 0 || (e->d_name[1] == '.' && e->d_name[2] == 0)) continue; } std::string full = path; - if (path[path.length()-1] != '/') full += "/"; + if (path[path.length() - 1] != '/') + full += "/"; full += e->d_name; bool do_stat = true; @@ -76,15 +75,14 @@ void TocEngine::scan_dbs(const std::string& path, std::list& dbs) c do_stat = true; } #endif - if(do_stat) { + if (do_stat) { eckit::Stat::Struct info; - if(eckit::Stat::stat(full.c_str(), &info) == 0) - { - if(S_ISDIR(info.st_mode)) { + if (eckit::Stat::stat(full.c_str(), &info) == 0) { + if (S_ISDIR(info.st_mode)) { dbs.push_back(full); } - } - else Log::error() << "Cannot stat " << full << Log::syserr << std::endl; + } else + Log::error() << "Cannot stat " << full << Log::syserr << std::endl; } } } @@ -97,39 +95,33 @@ std::string TocEngine::dbType() const { return TocEngine::typeName(); } -eckit::URI TocEngine::location(const Key& key, const Config& config) const -{ +eckit::URI TocEngine::location(const Key& key, const Config& config) const { return URI("toc", CatalogueRootManager(config).directory(key).directory_); } -bool TocEngine::canHandle(const eckit::URI& uri, const Config& config) const -{ +bool TocEngine::canHandle(const eckit::URI& uri, const Config& config) const { if (uri.scheme() != "toc") return false; eckit::PathName path = uri.path(); - eckit::PathName toc = path / "toc"; + eckit::PathName toc = path / "toc"; return path.isDir() && toc.exists(); } -static void matchKeyToDB(const Key& key, std::set& keys, const char* missing, const Config& config) -{ +static void matchKeyToDB(const Key& key, std::set& keys, const char* missing, const Config& config) { const Schema& schema = config.schema(); schema.matchFirstLevel(key, keys, missing); } -static void matchRequestToDB(const metkit::mars::MarsRequest& rq, std::set& keys, const char* missing, const Config& config) -{ +static void matchRequestToDB(const metkit::mars::MarsRequest& rq, std::set& keys, const char* missing, + const Config& config) { const Schema& schema = config.schema(); schema.matchFirstLevel(rq, keys, missing); } static constexpr const char* regexForMissingValues = "[^:/]*"; - - -std::set TocEngine::databases(const std::set& keys, - const std::vector& roots, +std::set TocEngine::databases(const std::set& keys, const std::vector& roots, const Config& config) const { std::set result; @@ -143,23 +135,23 @@ std::set TocEngine::databases(const std::set& keys, for (std::set::const_iterator i = keys.begin(); i != keys.end(); ++i) { - std::vector dbpaths = CatalogueRootManager(config).possibleDbPathNames(*i, regexForMissingValues); + std::vector dbpaths = + CatalogueRootManager(config).possibleDbPathNames(*i, regexForMissingValues); - for(std::vector::const_iterator dbpath = dbpaths.begin(); dbpath != dbpaths.end(); ++dbpath) { + for (std::vector::const_iterator dbpath = dbpaths.begin(); dbpath != dbpaths.end(); ++dbpath) { std::string regex = "^" + Regex::escape(j->asString()) + "/" + *dbpath + "$"; std::regex reg(regex, std::regex::icase | std::regex::optimize); - LOG_DEBUG_LIB(LibFdb5) << " -> key i " << *i - << " dbpath " << *dbpath - << " pathregex " << regex << std::endl; + LOG_DEBUG_LIB(LibFdb5) << " -> key i " << *i << " dbpath " << *dbpath << " pathregex " << regex + << std::endl; std::smatch m; for (std::list::const_iterator k = dbs.begin(); k != dbs.end(); ++k) { LOG_DEBUG_LIB(LibFdb5) << " -> db " << *k << std::endl; - if(result.find(*k) != result.end()) { + if (result.find(*k) != result.end()) { continue; } @@ -176,9 +168,8 @@ std::set TocEngine::databases(const std::set& keys, return result; } -std::vector TocEngine::databases(const Key& key, - const std::vector& roots, - const Config& config) const { +std::vector TocEngine::databases(const Key& key, const std::vector& roots, + const Config& config) const { std::set keys; @@ -197,7 +188,7 @@ std::vector TocEngine::databases(const Key& key, result.push_back(eckit::URI(TocEngine::typeName(), path)); } } catch (eckit::Exception& e) { - eckit::Log::error() << "Error loading FDB database from " << path << std::endl; + eckit::Log::error() << "Error loading FDB database from " << path << std::endl; eckit::Log::error() << e.what() << std::endl; } } @@ -206,12 +197,11 @@ std::vector TocEngine::databases(const Key& key, } std::vector TocEngine::databases(const metkit::mars::MarsRequest& request, - const std::vector& roots, - const Config& config) const { + const std::vector& roots, const Config& config) const { std::set keys; -// matchRequestToDB(request, keys, regexForMissingValues, config); + // matchRequestToDB(request, keys, regexForMissingValues, config); matchRequestToDB(request, keys, "", config); LOG_DEBUG_LIB(LibFdb5) << "Matched DB schemas for request " << request << " -> keys " << keys << std::endl; @@ -235,7 +225,7 @@ std::vector TocEngine::databases(const metkit::mars::MarsRequest& re } } } catch (eckit::Exception& e) { - eckit::Log::error() << "Error loading FDB database from " << path << std::endl; + eckit::Log::error() << "Error loading FDB database from " << path << std::endl; eckit::Log::error() << e.what() << std::endl; } } @@ -243,18 +233,15 @@ std::vector TocEngine::databases(const metkit::mars::MarsRequest& re return result; } -std::vector TocEngine::visitableLocations(const Key& key, const Config& config) const -{ +std::vector TocEngine::visitableLocations(const Key& key, const Config& config) const { return databases(key, CatalogueRootManager(config).visitableRoots(key), config); } -std::vector TocEngine::visitableLocations(const metkit::mars::MarsRequest& request, const Config& config) const -{ +std::vector TocEngine::visitableLocations(const metkit::mars::MarsRequest& request, const Config& config) const { return databases(request, CatalogueRootManager(config).visitableRoots(request), config); } -void TocEngine::print(std::ostream& out) const -{ +void TocEngine::print(std::ostream& out) const { out << "TocEngine()"; } diff --git a/src/fdb5/toc/TocEngine.h b/src/fdb5/toc/TocEngine.h index e2badef23..aac980132 100644 --- a/src/fdb5/toc/TocEngine.h +++ b/src/fdb5/toc/TocEngine.h @@ -25,14 +25,14 @@ namespace fdb5 { class TocEngine : public fdb5::Engine { public: // methods - static const char* typeName() { return "toc"; } -private: // methods +private: // methods std::set databases(const std::set& keys, const std::vector& dirs, const Config& config) const; - std::vector databases(const Key& key, const std::vector& dirs, const Config& config) const; + std::vector databases(const Key& key, const std::vector& dirs, + const Config& config) const; std::vector databases(const metkit::mars::MarsRequest& rq, const std::vector& dirs, const Config& config) const; @@ -40,7 +40,6 @@ class TocEngine : public fdb5::Engine { void scan_dbs(const std::string& path, std::list& dbs) const; protected: // methods - std::string name() const override; std::string dbType() const override; @@ -50,15 +49,14 @@ class TocEngine : public fdb5::Engine { bool canHandle(const eckit::URI&, const Config& config) const override; std::vector visitableLocations(const Key& key, const Config& config) const override; - std::vector visitableLocations(const metkit::mars::MarsRequest& rq, const Config& config) const override; - - void print( std::ostream &out ) const override; + std::vector visitableLocations(const metkit::mars::MarsRequest& rq, + const Config& config) const override; + void print(std::ostream& out) const override; }; //---------------------------------------------------------------------------------------------------------------------- - } // namespace fdb5 #endif diff --git a/src/fdb5/toc/TocFieldLocation.cc b/src/fdb5/toc/TocFieldLocation.cc index 60c1cc506..7777c4938 100644 --- a/src/fdb5/toc/TocFieldLocation.cc +++ b/src/fdb5/toc/TocFieldLocation.cc @@ -18,36 +18,39 @@ namespace fdb5 { -::eckit::ClassSpec TocFieldLocation::classSpec_ = {&FieldLocation::classSpec(), "TocFieldLocation",}; +::eckit::ClassSpec TocFieldLocation::classSpec_ = { + &FieldLocation::classSpec(), + "TocFieldLocation", +}; ::eckit::Reanimator TocFieldLocation::reanimator_; //---------------------------------------------------------------------------------------------------------------------- -//TocFieldLocation::TocFieldLocation() {} +// TocFieldLocation::TocFieldLocation() {} -TocFieldLocation::TocFieldLocation(const eckit::PathName path, eckit::Offset offset, eckit::Length length, const Key& remapKey) : - FieldLocation(eckit::URI("file", path), offset, length, remapKey) {} +TocFieldLocation::TocFieldLocation(const eckit::PathName path, eckit::Offset offset, eckit::Length length, + const Key& remapKey) + : FieldLocation(eckit::URI("file", path), offset, length, remapKey) {} -TocFieldLocation::TocFieldLocation(const eckit::URI &uri) : - FieldLocation(uri) {} +TocFieldLocation::TocFieldLocation(const eckit::URI& uri) : FieldLocation(uri) {} -TocFieldLocation::TocFieldLocation(const eckit::URI &uri, eckit::Offset offset, eckit::Length length, const Key& remapKey) : - FieldLocation(uri, offset, length, remapKey) {} +TocFieldLocation::TocFieldLocation(const eckit::URI& uri, eckit::Offset offset, eckit::Length length, + const Key& remapKey) + : FieldLocation(uri, offset, length, remapKey) {} -TocFieldLocation::TocFieldLocation(const TocFieldLocation& rhs) : - FieldLocation(rhs.uri_, rhs.offset_, rhs.length_, rhs.remapKey_) {} +TocFieldLocation::TocFieldLocation(const TocFieldLocation& rhs) + : FieldLocation(rhs.uri_, rhs.offset_, rhs.length_, rhs.remapKey_) {} -TocFieldLocation::TocFieldLocation(const UriStore &store, const FieldRef &ref) : - FieldLocation(store.get(ref.uriId()), ref.offset(), ref.length(), Key()) {} +TocFieldLocation::TocFieldLocation(const UriStore& store, const FieldRef& ref) + : FieldLocation(store.get(ref.uriId()), ref.offset(), ref.length(), Key()) {} -TocFieldLocation::TocFieldLocation(eckit::Stream& s) : - FieldLocation(s) {} +TocFieldLocation::TocFieldLocation(eckit::Stream& s) : FieldLocation(s) {} std::shared_ptr TocFieldLocation::make_shared() const { return std::make_shared(std::move(*this)); } -eckit::DataHandle *TocFieldLocation::dataHandle() const { +eckit::DataHandle* TocFieldLocation::dataHandle() const { if (remapKey_.empty()) { return uri_.path().partHandle(offset(), length()); } else { @@ -59,8 +62,9 @@ eckit::DataHandle *TocFieldLocation::dataHandle() const { } } -void TocFieldLocation::print(std::ostream &out) const { - out << "TocFieldLocation[uri=" << uri_ << ",offset=" << offset() << ",length=" << length() << ",remapKey=" << remapKey_ << "]"; +void TocFieldLocation::print(std::ostream& out) const { + out << "TocFieldLocation[uri=" << uri_ << ",offset=" << offset() << ",length=" << length() + << ",remapKey=" << remapKey_ << "]"; } void TocFieldLocation::visit(FieldLocationVisitor& visitor) const { @@ -68,7 +72,8 @@ void TocFieldLocation::visit(FieldLocationVisitor& visitor) const { } void TocFieldLocation::encode(eckit::Stream& s) const { - LOG_DEBUG(LibFdb5::instance().debug(), LibFdb5) << "TocFieldLocation encode URI " << uri_.asRawString() << std::endl; + LOG_DEBUG(LibFdb5::instance().debug(), LibFdb5) + << "TocFieldLocation encode URI " << uri_.asRawString() << std::endl; FieldLocation::encode(s); } diff --git a/src/fdb5/toc/TocFieldLocation.h b/src/fdb5/toc/TocFieldLocation.h index 3ef9af4f2..56c2babcd 100644 --- a/src/fdb5/toc/TocFieldLocation.h +++ b/src/fdb5/toc/TocFieldLocation.h @@ -24,16 +24,14 @@ namespace fdb5 { - //---------------------------------------------------------------------------------------------------------------------- class TocFieldLocation : public FieldLocation { public: - TocFieldLocation(const TocFieldLocation& rhs); TocFieldLocation(const eckit::PathName path, eckit::Offset offset, eckit::Length length, const Key& remapKey); - TocFieldLocation(const eckit::URI &uri); - TocFieldLocation(const eckit::URI &uri, eckit::Offset offset, eckit::Length length, const Key& remapKey); + TocFieldLocation(const eckit::URI& uri); + TocFieldLocation(const eckit::URI& uri, eckit::Offset offset, eckit::Length length, const Key& remapKey); TocFieldLocation(const UriStore& store, const FieldRef& ref); TocFieldLocation(eckit::Stream&); @@ -44,24 +42,19 @@ class TocFieldLocation : public FieldLocation { void visit(FieldLocationVisitor& visitor) const override; public: // For Streamable - - static const eckit::ClassSpec& classSpec() { return classSpec_;} + static const eckit::ClassSpec& classSpec() { return classSpec_; } protected: // For Streamable - const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } void encode(eckit::Stream&) const override; - static eckit::ClassSpec classSpec_; + static eckit::ClassSpec classSpec_; static eckit::Reanimator reanimator_; private: // methods - - void print(std::ostream &out) const override; - + void print(std::ostream& out) const override; }; - //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 diff --git a/src/fdb5/toc/TocHandler.cc b/src/fdb5/toc/TocHandler.cc index 90947c209..4a7d2e5a7 100644 --- a/src/fdb5/toc/TocHandler.cc +++ b/src/fdb5/toc/TocHandler.cc @@ -9,29 +9,29 @@ */ #include -#include #include +#include #include "eckit/config/Resource.h" -#include "eckit/io/FileHandle.h" +#include "eckit/filesystem/PathName.h" #include "eckit/io/FileDescHandle.h" +#include "eckit/io/FileHandle.h" #include "eckit/log/BigNum.h" #include "eckit/log/Log.h" #include "eckit/maths/Functions.h" #include "eckit/serialisation/MemoryStream.h" #include "eckit/thread/AutoLock.h" #include "eckit/thread/StaticMutex.h" -#include "eckit/filesystem/PathName.h" #include "fdb5/LibFdb5.h" +#include "fdb5/api/helpers/ControlIterator.h" #include "fdb5/database/Index.h" +#include "fdb5/io/LustreSettings.h" #include "fdb5/toc/TocCommon.h" #include "fdb5/toc/TocFieldLocation.h" #include "fdb5/toc/TocHandler.h" #include "fdb5/toc/TocIndex.h" #include "fdb5/toc/TocStats.h" -#include "fdb5/api/helpers/ControlIterator.h" -#include "fdb5/io/LustreSettings.h" #if eckit_HAVE_AIO #include @@ -44,46 +44,44 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- namespace { - constexpr const char* retrieve_lock_file = "retrieve.lock"; - constexpr const char* archive_lock_file = "archive.lock"; - constexpr const char* list_lock_file = "list.lock"; - constexpr const char* wipe_lock_file = "wipe.lock"; - constexpr const char* allow_duplicates_file = "duplicates.allow"; -} - -const std::map controlfile_lookup { +constexpr const char* retrieve_lock_file = "retrieve.lock"; +constexpr const char* archive_lock_file = "archive.lock"; +constexpr const char* list_lock_file = "list.lock"; +constexpr const char* wipe_lock_file = "wipe.lock"; +constexpr const char* allow_duplicates_file = "duplicates.allow"; +} // namespace + +const std::map controlfile_lookup{ {ControlIdentifier::Retrieve, retrieve_lock_file}, {ControlIdentifier::Archive, archive_lock_file}, {ControlIdentifier::List, list_lock_file}, {ControlIdentifier::Wipe, wipe_lock_file}, - {ControlIdentifier::UniqueRoot, allow_duplicates_file} -}; + {ControlIdentifier::UniqueRoot, allow_duplicates_file}}; //---------------------------------------------------------------------------------------------------------------------- class TocHandlerCloser { const TocHandler& handler_; - public: - TocHandlerCloser(const TocHandler &handler): handler_(handler) {} - ~TocHandlerCloser() { - handler_.close(); - } + +public: + TocHandlerCloser(const TocHandler& handler) : handler_(handler) {} + ~TocHandlerCloser() { handler_.close(); } }; //---------------------------------------------------------------------------------------------------------------------- class CachedFDProxy { public: // methods - - CachedFDProxy(const eckit::PathName& path, int fd, std::unique_ptr& cached) : - path_(path), - fd_(fd), - cached_(cached.get()) { + CachedFDProxy(const eckit::PathName& path, int fd, std::unique_ptr& cached) + : path_(path) + , fd_(fd) + , cached_(cached.get()) { ASSERT((fd != -1) != (!!cached)); } - long read(void* buf, long len, const char** pdata=nullptr) { - if (pdata && !cached_) throw SeriousBug("Can only return a pointer to data in memory if cached", Here()); + long read(void* buf, long len, const char** pdata = nullptr) { + if (pdata && !cached_) + throw SeriousBug("Can only return a pointer to data in memory if cached", Here()); if (cached_) { if (pdata) { *pdata = reinterpret_cast(cached_->data()) + cached_->position(); @@ -91,7 +89,7 @@ class CachedFDProxy { return cached_->read(buf, len); } else { long ret; - SYSCALL2( ret = ::read(fd_, buf, len), path_); + SYSCALL2(ret = ::read(fd_, buf, len), path_); return ret; } } @@ -118,7 +116,6 @@ class CachedFDProxy { } private: // members - const eckit::PathName& path_; int fd_; MemoryHandle* cached_; @@ -126,23 +123,22 @@ class CachedFDProxy { //---------------------------------------------------------------------------------------------------------------------- -TocHandler::TocHandler(const eckit::PathName& directory, const Config& config) : - TocCommon(directory), - tocPath_(directory_ / "toc"), - dbConfig_(config), - serialisationVersion_(TocSerialisationVersion(config)), - useSubToc_(config.userConfig().getBool("useSubToc", false)), - isSubToc_(false), - preloadBTree_(config.userConfig().getBool("preloadTocBTree", true)), - fd_(-1), - cachedToc_(nullptr), - subTocRead_(nullptr), - count_(0), - enumeratedMaskedEntries_(false), - numSubtocsRaw_(0), - writeMode_(false), - dirty_(false) -{ +TocHandler::TocHandler(const eckit::PathName& directory, const Config& config) + : TocCommon(directory) + , tocPath_(directory_ / "toc") + , dbConfig_(config) + , serialisationVersion_(TocSerialisationVersion(config)) + , useSubToc_(config.userConfig().getBool("useSubToc", false)) + , isSubToc_(false) + , preloadBTree_(config.userConfig().getBool("preloadTocBTree", true)) + , fd_(-1) + , cachedToc_(nullptr) + , subTocRead_(nullptr) + , count_(0) + , enumeratedMaskedEntries_(false) + , numSubtocsRaw_(0) + , writeMode_(false) + , dirty_(false) { // An override to enable using sub tocs without configurations being passed in, for ease // of debugging const char* subTocOverride = ::getenv("FDB5_SUB_TOCS"); @@ -151,23 +147,22 @@ TocHandler::TocHandler(const eckit::PathName& directory, const Config& config) : } } -TocHandler::TocHandler(const eckit::PathName& path, const Key& parentKey, MemoryHandle* cachedToc) : - TocCommon(path.dirName()), - parentKey_(parentKey), - tocPath_(TocCommon::findRealPath(eckit::LocalPathName{path})), - serialisationVersion_(TocSerialisationVersion(dbConfig_)), - useSubToc_(false), - isSubToc_(true), - preloadBTree_(false), - fd_(-1), - cachedToc_(cachedToc), - subTocRead_(nullptr), - count_(0), - enumeratedMaskedEntries_(false), - numSubtocsRaw_(0), - writeMode_(false), - dirty_(false) -{ +TocHandler::TocHandler(const eckit::PathName& path, const Key& parentKey, MemoryHandle* cachedToc) + : TocCommon(path.dirName()) + , parentKey_(parentKey) + , tocPath_(TocCommon::findRealPath(eckit::LocalPathName{path})) + , serialisationVersion_(TocSerialisationVersion(dbConfig_)) + , useSubToc_(false) + , isSubToc_(true) + , preloadBTree_(false) + , fd_(-1) + , cachedToc_(cachedToc) + , subTocRead_(nullptr) + , count_(0) + , enumeratedMaskedEntries_(false) + , numSubtocsRaw_(0) + , writeMode_(false) + , dirty_(false) { if (cachedToc_) { cachedToc_->openForRead(); @@ -178,8 +173,8 @@ TocHandler::TocHandler(const eckit::PathName& path, const Key& parentKey, Memory Key key(databaseKey()); if (!parentKey.empty() && parentKey != key) { - LOG_DEBUG_LIB(LibFdb5) << "Opening (remapped) toc with differing parent key: " - << key << " --> " << parentKey << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "Opening (remapped) toc with differing parent key: " << key << " --> " + << parentKey << std::endl; if (parentKey.size() != key.size()) { std::stringstream ss; @@ -203,7 +198,6 @@ TocHandler::TocHandler(const eckit::PathName& path, const Key& parentKey, Memory } } - TocHandler::~TocHandler() { close(); } @@ -218,21 +212,17 @@ void TocHandler::checkUID() const { return; } - static std::vector fdbSuperUsers = eckit::Resource>("fdbSuperUsers", "", true); + static std::vector fdbSuperUsers = + eckit::Resource>("fdbSuperUsers", "", true); if (dbUID() != userUID_) { - if(std::find(fdbSuperUsers.begin(), fdbSuperUsers.end(), userName(userUID_)) == fdbSuperUsers.end()) { + if (std::find(fdbSuperUsers.begin(), fdbSuperUsers.end(), userName(userUID_)) == fdbSuperUsers.end()) { std::ostringstream oss; - oss << "Only user '" - << userName(dbUID()) + oss << "Only user '" << userName(dbUID()) - << "' can write to FDB " - << directory_ - << ", current user is '" - << userName(userUID_) - << "'"; + << "' can write to FDB " << directory_ << ", current user is '" << userName(userUID_) << "'"; throw eckit::UserError(oss.str()); } @@ -257,11 +247,11 @@ void TocHandler::openForAppend() { #ifdef O_NOATIME // this introduces issues of permissions static bool fdbNoATime = eckit::Resource("fdbNoATime;$FDB_OPEN_NOATIME", false); - if(fdbNoATime) { + if (fdbNoATime) { iomode |= O_NOATIME; } #endif - SYSCALL2((fd_ = ::open( tocPath_.localPath(), iomode, (mode_t)0777 )), tocPath_); + SYSCALL2((fd_ = ::open(tocPath_.localPath(), iomode, (mode_t)0777)), tocPath_); } void TocHandler::openForRead() const { @@ -284,11 +274,11 @@ void TocHandler::openForRead() const { #ifdef O_NOATIME // this introduces issues of permissions static bool fdbNoATime = eckit::Resource("fdbNoATime;$FDB_OPEN_NOATIME", false); - if(fdbNoATime) { + if (fdbNoATime) { iomode |= O_NOATIME; } #endif - SYSCALL2((fd_ = ::open( tocPath_.localPath(), iomode )), tocPath_ ); + SYSCALL2((fd_ = ::open(tocPath_.localPath(), iomode)), tocPath_); eckit::Length tocSize = tocPath_.size(); // The masked subtocs and indexes could be updated each time, so reset this. @@ -296,17 +286,16 @@ void TocHandler::openForRead() const { numSubtocsRaw_ = 0; maskedEntries_.clear(); - if(fdbCacheTocsOnRead) { + if (fdbCacheTocsOnRead) { FileDescHandle toc(fd_, true); // closes the file descriptor AutoClose closer1(toc); fd_ = -1; - bool grow = true; - cachedToc_.reset( new eckit::MemoryHandle(tocSize, grow) ); + cachedToc_.reset(new eckit::MemoryHandle(tocSize, grow)); - long buffersize = 4*1024*1024; + long buffersize = 4 * 1024 * 1024; toc.copyTo(*cachedToc_, buffersize, tocSize, tocReadStats_); cachedToc_->openForRead(); } @@ -317,16 +306,17 @@ void TocHandler::dumpTocCache() const { eckit::Offset offset = cachedToc_->position(); cachedToc_->seek(0); - eckit::PathName tocDumpFile("dump_of_"+tocPath_.baseName()); + eckit::PathName tocDumpFile("dump_of_" + tocPath_.baseName()); eckit::FileHandle dump(eckit::PathName::unique(tocDumpFile)); cachedToc_->copyTo(dump); std::ostringstream ss; - ss << tocPath_.baseName() << " read in " << tocReadStats_.size() << " step" << ((tocReadStats_.size()>1)?"s":"") << std::endl; + ss << tocPath_.baseName() << " read in " << tocReadStats_.size() << " step" + << ((tocReadStats_.size() > 1) ? "s" : "") << std::endl; double time; eckit::Length len; while (tocReadStats_.next(time, len)) { - ss << " step duration: " << (time*1000) << " ms, size: " << len << " bytes"<< std::endl; + ss << " step duration: " << (time * 1000) << " ms, size: " << len << " bytes" << std::endl; } Log::error() << ss.str(); @@ -334,7 +324,7 @@ void TocHandler::dumpTocCache() const { } } -void TocHandler::append(TocRecord &r, size_t payloadSize ) { +void TocHandler::append(TocRecord& r, size_t payloadSize) { ASSERT(fd_ != -1); ASSERT(not cachedToc_); @@ -345,12 +335,12 @@ void TocHandler::append(TocRecord &r, size_t payloadSize ) { size_t roundedSize = roundRecord(r, payloadSize); size_t len; - SYSCALL2( len = ::write(fd_, &r, roundedSize), tocPath_ ); + SYSCALL2(len = ::write(fd_, &r, roundedSize), tocPath_); dirty_ = true; - ASSERT( len == roundedSize); + ASSERT(len == roundedSize); } -void TocHandler::appendBlock(const void *data, size_t size) { +void TocHandler::appendBlock(const void* data, size_t size) { openForAppend(); TocHandlerCloser close(*this); @@ -363,9 +353,9 @@ void TocHandler::appendBlock(const void *data, size_t size) { ASSERT(size % recordRoundSize() == 0); size_t len; - SYSCALL2( len = ::write(fd_, data, size), tocPath_ ); + SYSCALL2(len = ::write(fd_, data, size), tocPath_); dirty_ = true; - ASSERT( len == size ); + ASSERT(len == size); } const TocSerialisationVersion& TocHandler::serialisationVersion() const { @@ -378,7 +368,7 @@ size_t TocHandler::recordRoundSize() { return fdbRoundTocRecords; } -size_t TocHandler::roundRecord(TocRecord &r, size_t payloadSize) { +size_t TocHandler::roundRecord(TocRecord& r, size_t payloadSize) { r.header_.size_ = eckit::round(sizeof(TocRecord::Header) + payloadSize, recordRoundSize()); @@ -387,7 +377,8 @@ size_t TocHandler::roundRecord(TocRecord &r, size_t payloadSize) { // readNext wraps readNextInternal. // readNext reads the next TOC entry from this toc, or from an appropriate subtoc if necessary. -bool TocHandler::readNext( TocRecord &r, bool walkSubTocs, bool hideSubTocEntries, bool hideClearEntries, bool readMasked, const TocRecord** data, size_t* length) const { +bool TocHandler::readNext(TocRecord& r, bool walkSubTocs, bool hideSubTocEntries, bool hideClearEntries, + bool readMasked, const TocRecord** data, size_t* length) const { int len; @@ -423,19 +414,22 @@ bool TocHandler::readNext( TocRecord &r, bool walkSubTocs, bool hideSubTocEntrie } else if (r.header_.tag_ == TocRecord::TOC_INIT) { eckit::MemoryStream s(&r.payload_[0], r.maxPayloadSize); - if (parentKey_.empty()) parentKey_ = Key(s); + if (parentKey_.empty()) + parentKey_ = Key(s); return true; } else if (r.header_.tag_ == TocRecord::TOC_SUB_TOC) { LocalPathName absPath = parseSubTocRecord(r, readMasked); - if (absPath == "") continue; + if (absPath == "") + continue; selectSubTocRead(absPath); if (hideSubTocEntries) { // The first entry in a subtoc must be the init record. Check that - subTocRead_->readNext(r, walkSubTocs, hideSubTocEntries, hideClearEntries, readMasked, data, length); + subTocRead_->readNext(r, walkSubTocs, hideSubTocEntries, hideClearEntries, readMasked, data, + length); ASSERT(r.header_.tag_ == TocRecord::TOC_INIT); } else { return true; // if not hiding the subtoc entries, return them as normal entries! @@ -453,7 +447,7 @@ bool TocHandler::readNext( TocRecord &r, bool walkSubTocs, bool hideSubTocEntrie std::pair key(absPath.baseName(), offset); if (maskedEntries_.find(key) != maskedEntries_.end()) { - if(!readMasked){ + if (!readMasked) { LOG_DEBUG_LIB(LibFdb5) << "Index ignored by mask: " << path << ":" << offset << std::endl; continue; } @@ -488,7 +482,7 @@ bool TocHandler::readNextInternal(TocRecord& r, const TocRecord** data, size_t* return false; } ASSERT(len == sizeof(TocRecord::Header)); - } catch(...) { + } catch (...) { dumpTocCache(); throw; } @@ -497,8 +491,9 @@ bool TocHandler::readNextInternal(TocRecord& r, const TocRecord** data, size_t* long len = proxy.read(&r.payload_, r.header_.size_ - sizeof(TocRecord::Header)); ASSERT(size_t(len) == r.header_.size_ - sizeof(TocRecord::Header)); - if (length) (*length) = len + sizeof(TocRecord::Header); - } catch(...) { + if (length) + (*length) = len + sizeof(TocRecord::Header); + } catch (...) { dumpTocCache(); throw; } @@ -513,14 +508,15 @@ std::vector TocHandler::subTocPaths() const { openForRead(); TocHandlerCloser close(*this); - std::unique_ptr r(new TocRecord(serialisationVersion_.used())); // allocate (large) TocRecord on heap not stack (MARS-779) + std::unique_ptr r( + new TocRecord(serialisationVersion_.used())); // allocate (large) TocRecord on heap not stack (MARS-779) std::vector paths; bool walkSubTocs = true; bool hideSubTocEntries = false; bool hideClearEntries = true; - while ( readNext(*r, walkSubTocs, hideSubTocEntries, hideClearEntries) ) { + while (readNext(*r, walkSubTocs, hideSubTocEntries, hideClearEntries)) { eckit::MemoryStream s(&r->payload_[0], r->maxPayloadSize); std::string path; @@ -536,7 +532,8 @@ std::vector TocHandler::subTocPaths() const { } case TocRecord::TOC_CLEAR: - ASSERT_MSG(r->header_.tag_ != TocRecord::TOC_CLEAR, "The TOC_CLEAR records should have been pre-filtered on the first pass"); + ASSERT_MSG(r->header_.tag_ != TocRecord::TOC_CLEAR, + "The TOC_CLEAR records should have been pre-filtered on the first pass"); break; case TocRecord::TOC_INIT: @@ -570,13 +567,13 @@ void TocHandler::close() const { subTocWrite_->close(); } - if ( fd_ >= 0 ) { + if (fd_ >= 0) { LOG_DEBUG_LIB(LibFdb5) << "Closing TOC " << tocPath_ << std::endl; - if(dirty_) { - SYSCALL2( eckit::fdatasync(fd_), tocPath_ ); + if (dirty_) { + SYSCALL2(eckit::fdatasync(fd_), tocPath_); dirty_ = false; } - SYSCALL2( ::close(fd_), tocPath_ ); + SYSCALL2(::close(fd_), tocPath_); fd_ = -1; writeMode_ = false; } @@ -592,7 +589,8 @@ void TocHandler::allMaskableEntries(Offset startOffset, Offset endOffset, Offset ret = proxy.seek(startOffset); ASSERT(ret == startOffset); - std::unique_ptr r(new TocRecord(serialisationVersion_.used())); // allocate (large) TocRecord on heap not stack (MARS-779) + std::unique_ptr r( + new TocRecord(serialisationVersion_.used())); // allocate (large) TocRecord on heap not stack (MARS-779) while (proxy.position() < endOffset) { @@ -607,7 +605,7 @@ void TocHandler::allMaskableEntries(Offset startOffset, Offset endOffset, s >> path; maskedEntries.emplace(std::pair(path.baseName(), 0)); break; - } + } case TocRecord::TOC_INDEX: s >> path; @@ -654,7 +652,7 @@ eckit::LocalPathName TocHandler::parseSubTocRecord(const TocRecord& r, bool read // Unless readMasked is true, in which case walk it if it exists. std::pair key(absPath.baseName(), 0); if (maskedEntries_.find(key) != maskedEntries_.end()) { - if (!readMasked){ + if (!readMasked) { LOG_DEBUG_LIB(LibFdb5) << "SubToc ignored by mask: " << path << std::endl; return ""; } @@ -684,7 +682,8 @@ class SubtocPreloader { } AutoFDCloser& operator=(const AutoFDCloser&) = delete; ~AutoFDCloser() { - if (fd_ > 0) ::close(fd_); // n.b. ignore return value + if (fd_ > 0) + ::close(fd_); // n.b. ignore return value } }; @@ -694,7 +693,6 @@ class SubtocPreloader { std::vector paths_; public: - explicit SubtocPreloader(const Key& parentKey) : parentKey_(parentKey) {} decltype(subTocReadCache_)&& cache() { @@ -704,7 +702,7 @@ class SubtocPreloader { #ifdef O_NOATIME // this introduces issues of permissions static bool fdbNoATime = eckit::Resource("fdbNoATime;$FDB_OPEN_NOATIME", false); - if(fdbNoATime) { + if (fdbNoATime) { iomode |= O_NOATIME; } #endif @@ -767,7 +765,8 @@ class SubtocPreloader { for (int n = 0; n < aiocbs.size(); ++n) { - if (done[n]) continue; + if (done[n]) + continue; int e = ::aio_error(&aiocbs[n]); if (e == EINPROGRESS) { @@ -790,8 +789,8 @@ class SubtocPreloader { ASSERT(cachedToc->write(buffers[n].data(), aiocbs[n].aio_nbytes) == aiocbs[n].aio_nbytes); } ASSERT(subTocReadCache_.find(paths_[n]) == subTocReadCache_.end()); - subTocReadCache_.emplace(paths_[n], std::make_unique(paths_[n], parentKey_, - cachedToc.release())); + subTocReadCache_.emplace( + paths_[n], std::make_unique(paths_[n], parentKey_, cachedToc.release())); done[n] = true; doneCount++; @@ -802,19 +801,18 @@ class SubtocPreloader { } } #else - NOTIMP; + NOTIMP; #endif // eckit_HAVE_AIO return std::move(subTocReadCache_); } - void addPath(const eckit::LocalPathName& path) { - paths_.push_back(path); - } + void addPath(const eckit::LocalPathName& path) { paths_.push_back(path); } }; void TocHandler::preloadSubTocs(bool readMasked) const { ASSERT(enumeratedMaskedEntries_); - if (numSubtocsRaw_ == 0) return; + if (numSubtocsRaw_ == 0) + return; CachedFDProxy proxy(tocPath_, fd_, cachedToc_); Offset startPosition = proxy.position(); // remember the current position of the file descriptor @@ -824,7 +822,7 @@ void TocHandler::preloadSubTocs(bool readMasked) const { eckit::Timer preloadTimer("subtocs.preload", Log::debug()); { std::unique_ptr r( - new TocRecord(serialisationVersion_.used())); // allocate (large) TocRecord on heap not stack (MARS-779) + new TocRecord(serialisationVersion_.used())); // allocate (large) TocRecord on heap not stack (MARS-779) // n.b. we call databaseKey() directly, as this preload will normally be called before we have walked // the toc at all --> TOC_INIT not yet read --> parentKey_ not yet set. @@ -834,10 +832,10 @@ void TocHandler::preloadSubTocs(bool readMasked) const { switch (r->header_.tag_) { case TocRecord::TOC_SUB_TOC: { - LocalPathName absPath = parseSubTocRecord(*r, readMasked); - if (absPath != "") preloader.addPath(absPath); - } - break; + LocalPathName absPath = parseSubTocRecord(*r, readMasked); + if (absPath != "") + preloader.addPath(absPath); + } break; case TocRecord::TOC_INIT: break; case TocRecord::TOC_INDEX: @@ -870,11 +868,12 @@ void TocHandler::populateMaskedEntriesList() const { maskedEntries_.clear(); - std::unique_ptr r(new TocRecord(serialisationVersion_.used())); // allocate (large) TocRecord on heap not stack (MARS-779) + std::unique_ptr r( + new TocRecord(serialisationVersion_.used())); // allocate (large) TocRecord on heap not stack (MARS-779) size_t countSubTocs = 0; - while ( readNextInternal(*r) ) { + while (readNextInternal(*r)) { eckit::MemoryStream s(&r->payload_[0], r->maxPayloadSize); std::string path; @@ -932,7 +931,7 @@ void TocHandler::writeInitRecord(const Key& key) { eckit::AutoLock lock(local_mutex); - if ( !directory_.exists() ) { + if (!directory_.exists()) { directory_.mkdir(); } @@ -944,11 +943,12 @@ void TocHandler::writeInitRecord(const Key& key) { ASSERT(fd_ == -1); int iomode = O_CREAT | O_RDWR; - SYSCALL2(fd_ = ::open( tocPath_.localPath(), iomode, mode_t(0777) ), tocPath_); + SYSCALL2(fd_ = ::open(tocPath_.localPath(), iomode, mode_t(0777)), tocPath_); TocHandlerCloser closer(*this); - std::unique_ptr r(new TocRecord(serialisationVersion_.used())); // allocate (large) TocRecord on heap not stack (MARS-779) + std::unique_ptr r( + new TocRecord(serialisationVersion_.used())); // allocate (large) TocRecord on heap not stack (MARS-779) size_t len = readNext(*r); if (len == 0) { @@ -959,11 +959,8 @@ void TocHandler::writeInitRecord(const Key& key) { /* Copy schema first */ - LOG_DEBUG_LIB(LibFdb5) << "Copy schema from " - << dbConfig_.schemaPath() - << " to " - << schemaPath_ - << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "Copy schema from " << dbConfig_.schemaPath() << " to " << schemaPath_ + << std::endl; eckit::LocalPathName tmp{eckit::PathName::unique(schemaPath_)}; @@ -984,7 +981,8 @@ void TocHandler::writeInitRecord(const Key& key) { eckit::LocalPathName::rename(tmp, schemaPath_); } - std::unique_ptr r2(new TocRecord(serialisationVersion_.used(), TocRecord::TOC_INIT)); // allocate TocRecord on heap (MARS-779) + std::unique_ptr r2( + new TocRecord(serialisationVersion_.used(), TocRecord::TOC_INIT)); // allocate TocRecord on heap (MARS-779) eckit::MemoryStream s(&r2->payload_[0], r2->maxPayloadSize); s << key; s << isSubToc_; @@ -999,9 +997,11 @@ void TocHandler::writeInitRecord(const Key& key) { } } -void TocHandler::writeClearRecord(const Index &index) { +void TocHandler::writeClearRecord(const Index& index) { - std::unique_ptr r(new TocRecord(serialisationVersion_.used(), TocRecord::TOC_CLEAR)); // allocate (large) TocRecord on heap not stack (MARS-779) + std::unique_ptr r( + new TocRecord(serialisationVersion_.used(), + TocRecord::TOC_CLEAR)); // allocate (large) TocRecord on heap not stack (MARS-779) size_t sz = roundRecord(*r, buildClearRecord(*r, index)); appendBlock(r.get(), sz); @@ -1009,23 +1009,26 @@ void TocHandler::writeClearRecord(const Index &index) { void TocHandler::writeClearAllRecord() { - std::unique_ptr r(new TocRecord(serialisationVersion_.used(), TocRecord::TOC_CLEAR)); // allocate (large) TocRecord on heap not stack (MARS-779) + std::unique_ptr r( + new TocRecord(serialisationVersion_.used(), + TocRecord::TOC_CLEAR)); // allocate (large) TocRecord on heap not stack (MARS-779) eckit::MemoryStream s(&r->payload_[0], r->maxPayloadSize); - s << std::string {"*"}; + s << std::string{"*"}; s << off_t{0}; size_t sz = roundRecord(*r, s.position()); appendBlock(r.get(), sz); } - void TocHandler::writeSubTocRecord(const TocHandler& subToc) { openForAppend(); TocHandlerCloser closer(*this); - std::unique_ptr r(new TocRecord(serialisationVersion_.used(), TocRecord::TOC_SUB_TOC)); // allocate (large) TocRecord on heap not stack (MARS-779) + std::unique_ptr r( + new TocRecord(serialisationVersion_.used(), + TocRecord::TOC_SUB_TOC)); // allocate (large) TocRecord on heap not stack (MARS-779) eckit::MemoryStream s(&r->payload_[0], r->maxPayloadSize); @@ -1041,7 +1044,6 @@ void TocHandler::writeSubTocRecord(const TocHandler& subToc) { LOG_DEBUG_LIB(LibFdb5) << "Write TOC_SUB_TOC " << path << std::endl; } - void TocHandler::writeIndexRecord(const Index& index) { // Toc index writer @@ -1049,11 +1051,13 @@ void TocHandler::writeIndexRecord(const Index& index) { struct WriteToStream : public IndexLocationVisitor { WriteToStream(const Index& index, TocHandler& handler) : index_(index), handler_(handler) {} - virtual void operator() (const IndexLocation& l) { + virtual void operator()(const IndexLocation& l) { const TocIndexLocation& location = reinterpret_cast(l); - std::unique_ptr r(new TocRecord(handler_.serialisationVersion_.used(), TocRecord::TOC_INDEX)); // allocate (large) TocRecord on heap not stack (MARS-779) + std::unique_ptr r( + new TocRecord(handler_.serialisationVersion_.used(), + TocRecord::TOC_INDEX)); // allocate (large) TocRecord on heap not stack (MARS-779) eckit::MemoryStream s(&r->payload_[0], r->maxPayloadSize); @@ -1064,7 +1068,8 @@ void TocHandler::writeIndexRecord(const Index& index) { index_.encode(s, r->header_.serialisationVersion_); handler_.append(*r, s.position()); - LOG_DEBUG_LIB(LibFdb5) << "Write TOC_INDEX " << location.uri().path().baseName() << " - " << location.offset() << " " << index_.type() << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "Write TOC_INDEX " << location.uri().path().baseName() << " - " + << location.offset() << " " << index_.type() << std::endl; } private: @@ -1102,9 +1107,11 @@ void TocHandler::writeIndexRecord(const Index& index) { index.visit(writeVisitor); } -void TocHandler::writeSubTocMaskRecord(const TocHandler &subToc) { +void TocHandler::writeSubTocMaskRecord(const TocHandler& subToc) { - std::unique_ptr r(new TocRecord(serialisationVersion_.used(), TocRecord::TOC_CLEAR)); // allocate (large) TocRecord on heap not stack (MARS-779) + std::unique_ptr r( + new TocRecord(serialisationVersion_.used(), + TocRecord::TOC_CLEAR)); // allocate (large) TocRecord on heap not stack (MARS-779) // We use a relative path to this subtoc if it belongs to the current DB // but an absolute one otherwise (e.g. for fdb-overlay). @@ -1131,13 +1138,14 @@ class HasPath { off_t offset_; public: - HasPath(const eckit::PathName &path, off_t offset): path_(path), offset_(offset) {} + HasPath(const eckit::PathName& path, off_t offset) : path_(path), offset_(offset) {} bool operator()(const Index index) const { const TocIndex* tocidx = dynamic_cast(index.content()); - if(!tocidx) { - throw eckit::NotImplemented("Index is not of TocIndex type -- referencing unknown Index types isn't supported", Here()); + if (!tocidx) { + throw eckit::NotImplemented( + "Index is not of TocIndex type -- referencing unknown Index types isn't supported", Here()); } return (tocidx->path() == path_) && (tocidx->offset() == offset_); @@ -1156,7 +1164,7 @@ uid_t TocHandler::dbUID() const { // Allocate (large) TocRecord on heap not stack (MARS-779) std::unique_ptr r(new TocRecord(serialisationVersion_.used())); - while ( readNext(*r) ) { + while (readNext(*r)) { if (r->header_.tag_ == TocRecord::TOC_INIT) { dbUID_ = r->header_.uid_; return dbUID_; @@ -1174,7 +1182,7 @@ Key TocHandler::databaseKey() { std::unique_ptr r(new TocRecord(serialisationVersion_.used())); bool walkSubTocs = false; - while ( readNext(*r, walkSubTocs) ) { + while (readNext(*r, walkSubTocs)) { if (r->header_.tag_ == TocRecord::TOC_INIT) { eckit::MemoryStream s(&r->payload_[0], r->maxPayloadSize); dbUID_ = r->header_.uid_; @@ -1197,7 +1205,7 @@ size_t TocHandler::numberOfRecords() const { bool walkSubTocs = true; bool hideSubTocEntries = false; bool hideClearEntries = false; - while ( readNext(*r, walkSubTocs, hideSubTocEntries, hideClearEntries) ) { + while (readNext(*r, walkSubTocs, hideSubTocEntries, hideClearEntries)) { count_++; } } @@ -1205,15 +1213,12 @@ size_t TocHandler::numberOfRecords() const { return count_; } -const eckit::LocalPathName& TocHandler::directory() const -{ +const eckit::LocalPathName& TocHandler::directory() const { return directory_; } -std::vector TocHandler::loadIndexes(const Catalogue& catalogue, bool sorted, - std::set* subTocs, - std::vector* indexInSubtoc, - std::vector* remapKeys) const { +std::vector TocHandler::loadIndexes(const Catalogue& catalogue, bool sorted, std::set* subTocs, + std::vector* indexInSubtoc, std::vector* remapKeys) const { std::vector indexes; @@ -1252,7 +1257,7 @@ std::vector TocHandler::loadIndexes(const Catalogue& catalogue, bool sort bool readMasked = false; const TocRecord* pdata; size_t dataLength; - while ( readNext(*r, walkSubTocs, hideSubTocEntries, hideClearEntries, readMasked, &pdata, &dataLength) ) { + while (readNext(*r, walkSubTocs, hideSubTocEntries, hideClearEntries, readMasked, &pdata, &dataLength)) { eckit::MemoryStream s(&r->payload_[0], r->maxPayloadSize); std::string path; @@ -1263,44 +1268,42 @@ std::vector TocHandler::loadIndexes(const Catalogue& catalogue, bool sort count_++; - switch (r->header_.tag_) { - case TocRecord::TOC_INIT: - dbUID_ = r->header_.uid_; - LOG_DEBUG(debug, LibFdb5) << "TocRecord TOC_INIT key is " << Key(s) << std::endl; - break; - - case TocRecord::TOC_INDEX: - indexEntries.emplace_back(IndexEntry{indexEntries.size(), pdata, dataLength, currentDirectory()}); + case TocRecord::TOC_INIT: + dbUID_ = r->header_.uid_; + LOG_DEBUG(debug, LibFdb5) << "TocRecord TOC_INIT key is " << Key(s) << std::endl; + break; - if (subTocs && subTocRead_) { - subTocs->insert(subTocRead_->tocPath()); - } - if (indexInSubtoc) { - indexInSubtoc->push_back(!!subTocRead_); - } - if (remapKeys) { - remapKeys->push_back(currentRemapKey()); - } - break; + case TocRecord::TOC_INDEX: + indexEntries.emplace_back(IndexEntry{indexEntries.size(), pdata, dataLength, currentDirectory()}); - case TocRecord::TOC_CLEAR: - ASSERT_MSG(r->header_.tag_ != TocRecord::TOC_CLEAR, "The TOC_CLEAR records should have been pre-filtered on the first pass"); - break; + if (subTocs && subTocRead_) { + subTocs->insert(subTocRead_->tocPath()); + } + if (indexInSubtoc) { + indexInSubtoc->push_back(!!subTocRead_); + } + if (remapKeys) { + remapKeys->push_back(currentRemapKey()); + } + break; - case TocRecord::TOC_SUB_TOC: - throw eckit::SeriousBug("TOC_SUB_TOC entry should be handled inside readNext"); - break; + case TocRecord::TOC_CLEAR: + ASSERT_MSG(r->header_.tag_ != TocRecord::TOC_CLEAR, + "The TOC_CLEAR records should have been pre-filtered on the first pass"); + break; - default: - std::ostringstream oss; - oss << "Unknown tag in TocRecord " << *r; - throw eckit::SeriousBug(oss.str(), Here()); - break; + case TocRecord::TOC_SUB_TOC: + throw eckit::SeriousBug("TOC_SUB_TOC entry should be handled inside readNext"); + break; + default: + std::ostringstream oss; + oss << "Unknown tag in TocRecord " << *r; + throw eckit::SeriousBug(oss.str(), Here()); + break; } - } // Now construct the index objects (we can parallelise this...) @@ -1316,27 +1319,28 @@ std::vector TocHandler::loadIndexes(const Catalogue& catalogue, bool sort tocindexes.resize(indexEntries.size()); for (int i = 0; i < nthreads; ++i) { - threads.emplace_back(std::async(std::launch::async, [i, &indexEntries, &tocindexes, &nthreads_shadow, debug, &catalogue, this] { - for (int idx = i; idx < indexEntries.size(); idx+=nthreads) { - - const IndexEntry& entry = indexEntries[idx]; - eckit::MemoryStream s(entry.datap->payload_, entry.dataLen - sizeof(TocRecord::Header)); - LocalPathName path; - off_t offset; - std::string type; - s >> path; - s >> offset; - s >> type; - LOG_DEBUG(debug, LibFdb5) << "TocRecord TOC_INDEX " << path << " - " << offset << std::endl; - tocindexes[entry.seqNo] = new TocIndex(s, catalogue, entry.datap->header_.serialisationVersion_, - entry.tocDirectoryName, - entry.tocDirectoryName / path, - offset, preloadBTree_); - } - })); + threads.emplace_back(std::async( + std::launch::async, [i, &indexEntries, &tocindexes, &nthreads_shadow, debug, &catalogue, this] { + for (int idx = i; idx < indexEntries.size(); idx += nthreads) { + + const IndexEntry& entry = indexEntries[idx]; + eckit::MemoryStream s(entry.datap->payload_, entry.dataLen - sizeof(TocRecord::Header)); + LocalPathName path; + off_t offset; + std::string type; + s >> path; + s >> offset; + s >> type; + LOG_DEBUG(debug, LibFdb5) << "TocRecord TOC_INDEX " << path << " - " << offset << std::endl; + tocindexes[entry.seqNo] = + new TocIndex(s, catalogue, entry.datap->header_.serialisationVersion_, + entry.tocDirectoryName, entry.tocDirectoryName / path, offset, preloadBTree_); + } + })); } - for (auto& thread : threads) thread.get(); + for (auto& thread : threads) + thread.get(); indexes.reserve(indexEntries.size()); for (TocIndex* ti : tocindexes) { @@ -1367,7 +1371,6 @@ std::vector TocHandler::loadIndexes(const Catalogue& catalogue, bool sort } return indexes; - } const eckit::LocalPathName& TocHandler::tocPath() const { @@ -1401,7 +1404,7 @@ void TocHandler::dump(std::ostream& out, bool simple, bool walkSubTocs) const { bool hideSubTocEntries = false; bool hideClearEntries = false; - while ( readNext(*r, walkSubTocs, hideSubTocEntries, hideClearEntries) ) { + while (readNext(*r, walkSubTocs, hideSubTocEntries, hideClearEntries)) { eckit::MemoryStream s(&r->payload_[0], r->maxPayloadSize); LocalPathName path; @@ -1422,7 +1425,9 @@ void TocHandler::dump(std::ostream& out, bool simple, bool walkSubTocs) const { s >> isSubToc; } out << " Key: " << key << ", sub-toc: " << (isSubToc ? "yes" : "no"); - if(!simple) { out << std::endl; } + if (!simple) { + out << std::endl; + } break; } @@ -1431,9 +1436,12 @@ void TocHandler::dump(std::ostream& out, bool simple, bool walkSubTocs) const { s >> offset; s >> type; out << " Path: " << path << ", offset: " << offset << ", type: " << type; - if(!simple) { out << std::endl; } - Index index(new TocIndex(s, *(dynamic_cast(this)), r->header_.serialisationVersion_, - currentDirectory(), currentDirectory() / path, offset)); + if (!simple) { + out << std::endl; + } + Index index(new TocIndex(s, *(dynamic_cast(this)), + r->header_.serialisationVersion_, currentDirectory(), + currentDirectory() / path, offset)); index.dump(out, " ", simple); break; } @@ -1460,7 +1468,6 @@ void TocHandler::dump(std::ostream& out, bool simple, bool walkSubTocs) const { } } - void TocHandler::dumpIndexFile(std::ostream& out, const eckit::PathName& indexFile) const { openForRead(); @@ -1473,7 +1480,7 @@ void TocHandler::dumpIndexFile(std::ostream& out, const eckit::PathName& indexFi bool hideSubTocEntries = true; bool hideClearEntries = true; bool readMasked = true; - while ( readNext(*r, walkSubTocs, hideSubTocEntries, hideClearEntries, readMasked) ) { + while (readNext(*r, walkSubTocs, hideSubTocEntries, hideClearEntries, readMasked)) { eckit::MemoryStream s(&r->payload_[0], r->maxPayloadSize); LocalPathName path; @@ -1490,8 +1497,9 @@ void TocHandler::dumpIndexFile(std::ostream& out, const eckit::PathName& indexFi if ((currentDirectory() / path).sameAs(eckit::LocalPathName{indexFile})) { r->dump(out, true); out << std::endl << " Path: " << path << ", offset: " << offset << ", type: " << type; - Index index(new TocIndex(s, *(dynamic_cast(this)), r->header_.serialisationVersion_, - currentDirectory(), currentDirectory() / path, offset)); + Index index(new TocIndex(s, *(dynamic_cast(this)), + r->header_.serialisationVersion_, currentDirectory(), + currentDirectory() / path, offset)); index.dump(out, " ", false, true); } break; @@ -1499,7 +1507,8 @@ void TocHandler::dumpIndexFile(std::ostream& out, const eckit::PathName& indexFi case TocRecord::TOC_SUB_TOC: case TocRecord::TOC_CLEAR: - ASSERT_MSG(r->header_.tag_ != TocRecord::TOC_CLEAR, "The TOC_CLEAR records should have been pre-filtered on the first pass"); + ASSERT_MSG(r->header_.tag_ != TocRecord::TOC_CLEAR, + "The TOC_CLEAR records should have been pre-filtered on the first pass"); break; case TocRecord::TOC_INIT: @@ -1511,27 +1520,23 @@ void TocHandler::dumpIndexFile(std::ostream& out, const eckit::PathName& indexFi } } } - } - std::string TocHandler::dbOwner() const { return userName(dbUID()); } -DbStats TocHandler::stats() const -{ +DbStats TocHandler::stats() const { TocDbStats* stats = new TocDbStats(); - stats->dbCount_ += 1; + stats->dbCount_ += 1; stats->tocRecordsCount_ += numberOfRecords(); - stats->tocFileSize_ += tocFilesSize(); - stats->schemaFileSize_ += schemaPath().size(); + stats->tocFileSize_ += tocFilesSize(); + stats->schemaFileSize_ += schemaPath().size(); return DbStats(stats); } - void TocHandler::enumerateMasked(const Catalogue& catalogue, std::set>& metadata, std::set& data) const { @@ -1582,7 +1587,7 @@ void TocHandler::enumerateMasked(const Catalogue& catalogue, std::set r(new TocRecord(serialisationVersion_.used())); - while ( readNextInternal(*r) ) { + while (readNextInternal(*r)) { if (r->header_.tag_ == TocRecord::TOC_INDEX) { eckit::MemoryStream s(&r->payload_[0], r->maxPayloadSize); @@ -1600,20 +1605,21 @@ void TocHandler::enumerateMasked(const Catalogue& catalogue, std::set key(absPath.baseName(), offset); if (maskedEntries_.find(key) != maskedEntries_.end()) { if (absPath.exists()) { - Index index(new TocIndex(s, *(dynamic_cast(this)), r->header_.serialisationVersion_, directory_, absPath, offset)); - for (const auto& dataURI : index.dataURIs()) data.insert(dataURI); + Index index(new TocIndex(s, *(dynamic_cast(this)), + r->header_.serialisationVersion_, directory_, absPath, offset)); + for (const auto& dataURI : index.dataURIs()) + data.insert(dataURI); } } } } } - size_t TocHandler::tocFilesSize() const { // Get the size of the master toc - size_t size = tocPath().size(); + size_t size = tocPath().size(); // If we have subtocs, we need to get those too! @@ -1627,13 +1633,13 @@ size_t TocHandler::tocFilesSize() const { } std::string TocHandler::userName(long id) const { - struct passwd *p = getpwuid(id); + struct passwd* p = getpwuid(id); - if (p) { - return p->pw_name; - } else { - return eckit::Translator()(id); - } + if (p) { + return p->pw_name; + } else { + return eckit::Translator()(id); + } } const Key& TocHandler::currentRemapKey() const { @@ -1660,7 +1666,7 @@ const LocalPathName& TocHandler::currentTocPath() const { } } -size_t TocHandler::buildIndexRecord(TocRecord& r, const Index &index) { +size_t TocHandler::buildIndexRecord(TocRecord& r, const Index& index) { const IndexLocation& location(index.location()); const TocIndexLocation& tocLoc(reinterpret_cast(location)); @@ -1677,13 +1683,13 @@ size_t TocHandler::buildIndexRecord(TocRecord& r, const Index &index) { return s.position(); } -size_t TocHandler::buildClearRecord(TocRecord &r, const Index &index) { +size_t TocHandler::buildClearRecord(TocRecord& r, const Index& index) { struct TocIndexLocationExtracter : public IndexLocationVisitor { TocIndexLocationExtracter(TocRecord& r) : r_(r), sz_(0) {} - virtual void operator() (const IndexLocation& l) { + virtual void operator()(const IndexLocation& l) { const TocIndexLocation& location = reinterpret_cast(l); @@ -1693,7 +1699,8 @@ size_t TocHandler::buildClearRecord(TocRecord &r, const Index &index) { s << location.offset(); ASSERT(sz_ == 0); sz_ = s.position(); - LOG_DEBUG_LIB(LibFdb5) << "Write TOC_CLEAR " << location.uri().path().baseName() << " - " << location.offset() << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "Write TOC_CLEAR " << location.uri().path().baseName() << " - " + << location.offset() << std::endl; } size_t size() const { return sz_; } @@ -1733,15 +1740,13 @@ size_t TocHandler::buildSubTocMaskRecord(TocRecord& r, const eckit::PathName& pa eckit::MemoryStream s(&r.payload_[0], r.maxPayloadSize); s << path; - s << static_cast(0); // Always use an offset of zero for subtocs + s << static_cast(0); // Always use an offset of zero for subtocs return s.position(); } void TocHandler::control(const ControlAction& action, const ControlIdentifiers& identifiers) const { - - for (ControlIdentifier identifier : identifiers) { auto it = controlfile_lookup.find(identifier); @@ -1750,18 +1755,16 @@ void TocHandler::control(const ControlAction& action, const ControlIdentifiers& const std::string& lock_file(it->second); switch (action) { - case ControlAction::Disable: - createControlFile(lock_file); - break; - - case ControlAction::Enable: - removeControlFile(lock_file); - break; - - default: - eckit::Log::warning() << "Unexpected action: " - << static_cast(action) - << std::endl; + case ControlAction::Disable: + createControlFile(lock_file); + break; + + case ControlAction::Enable: + removeControlFile(lock_file); + break; + + default: + eckit::Log::warning() << "Unexpected action: " << static_cast(action) << std::endl; } } } @@ -1778,13 +1781,11 @@ std::vector TocHandler::lockfilePaths() const { std::vector paths; - for (const auto& name : { retrieve_lock_file, - archive_lock_file, - list_lock_file, - wipe_lock_file }) { + for (const auto& name : {retrieve_lock_file, archive_lock_file, list_lock_file, wipe_lock_file}) { PathName fullPath = fullControlFilePath(name); - if (fullPath.exists()) paths.emplace_back(std::move(fullPath)); + if (fullPath.exists()) + paths.emplace_back(std::move(fullPath)); } return paths; diff --git a/src/fdb5/toc/TocHandler.h b/src/fdb5/toc/TocHandler.h index a933ff39b..d2dbc0281 100644 --- a/src/fdb5/toc/TocHandler.h +++ b/src/fdb5/toc/TocHandler.h @@ -17,26 +17,24 @@ #include #include -#include "eckit/filesystem/PathName.h" #include "eckit/filesystem/LocalPathName.h" +#include "eckit/filesystem/PathName.h" #include "eckit/filesystem/URI.h" #include "eckit/io/Length.h" #include "eckit/io/MemoryHandle.h" #include "eckit/log/Timer.h" #include "fdb5/config/Config.h" -#include "fdb5/database/DbStats.h" #include "fdb5/database/Catalogue.h" +#include "fdb5/database/DbStats.h" #include "fdb5/toc/TocCommon.h" #include "fdb5/toc/TocRecord.h" #include "fdb5/toc/TocSerialisationVersion.h" - - namespace eckit { class Configuration; class MemoryHandle; -} +} // namespace eckit namespace fdb5 { @@ -65,16 +63,14 @@ class TocCopyWatcher : public eckit::TransferWatcher { tocCopyStats_.push_back(std::make_pair(now - time_, len)); time_ = now; } - + public: - TocCopyWatcher(): idx_(0) {} + TocCopyWatcher() : idx_(0) {} - size_t size() const { - return tocCopyStats_.size(); - } + size_t size() const { return tocCopyStats_.size(); } bool next(double& time, eckit::Length& len) { - if (tocCopyStats_.size()>idx_) { + if (tocCopyStats_.size() > idx_) { time = tocCopyStats_[idx_].first; len = tocCopyStats_[idx_].second; @@ -90,7 +86,6 @@ class TocCopyWatcher : public eckit::TransferWatcher { size_t idx_; std::vector> tocCopyStats_; - }; //----------------------------------------------------------------------------- @@ -98,15 +93,14 @@ class TocCopyWatcher : public eckit::TransferWatcher { class TocHandler : public TocCommon, private eckit::NonCopyable { public: // typedefs - typedef std::vector TocVec; - typedef std::vector< eckit::LocalPathName > TocPaths; + typedef std::vector TocPaths; public: // methods - TocHandler( const eckit::PathName &dir, const Config& config); + TocHandler(const eckit::PathName& dir, const Config& config); /// For initialising sub tocs or diagnostic interrogation. - TocHandler(const eckit::PathName& path, const Key& parentKey, eckit::MemoryHandle* cachedToc=nullptr); + TocHandler(const eckit::PathName& path, const Key& parentKey, eckit::MemoryHandle* cachedToc = nullptr); ~TocHandler() override; @@ -114,10 +108,10 @@ class TocHandler : public TocCommon, private eckit::NonCopyable { void checkUID() const override; void writeInitRecord(const Key& tocKey); - void writeClearRecord(const Index &); + void writeClearRecord(const Index&); void writeClearAllRecord(); void writeSubTocRecord(const TocHandler& subToc); - void writeIndexRecord(const Index &); + void writeIndexRecord(const Index&); void writeSubTocMaskRecord(const TocHandler& subToc); void reconsolidateIndexesAndTocs(); @@ -127,10 +121,8 @@ class TocHandler : public TocCommon, private eckit::NonCopyable { /// Return a list of existent indexes. If supplied, also supply a list of associated /// subTocs that were read to get these indexes - std::vector loadIndexes(const Catalogue& catalogue, - bool sorted=false, - std::set* subTocs = nullptr, - std::vector* indexInSubtoc = nullptr, + std::vector loadIndexes(const Catalogue& catalogue, bool sorted = false, + std::set* subTocs = nullptr, std::vector* indexInSubtoc = nullptr, std::vector* remapKeys = nullptr) const; Key databaseKey(); @@ -146,8 +138,7 @@ class TocHandler : public TocCommon, private eckit::NonCopyable { DbStats stats() const; - void enumerateMasked(const Catalogue& catalogue, - std::set>& metadata, + void enumerateMasked(const Catalogue& catalogue, std::set>& metadata, std::set& data) const; std::vector subTocPaths() const; @@ -155,10 +146,8 @@ class TocHandler : public TocCommon, private eckit::NonCopyable { std::vector lockfilePaths() const; protected: // methods - size_t tocFilesSize() const; - // Access and control of locks void control(const ControlAction& action, const ControlIdentifiers& identifiers) const; @@ -166,19 +155,16 @@ class TocHandler : public TocCommon, private eckit::NonCopyable { bool enabled(const ControlIdentifier& controlIdentifier) const; private: // methods - eckit::LocalPathName fullControlFilePath(const std::string& name) const; void createControlFile(const std::string& name) const; void removeControlFile(const std::string& name) const; -protected: // members - +protected: // members mutable Key parentKey_; // Contains the key of the first TOC explored in subtoc chain uid_t dbUID() const override; protected: // methods - // Handle location and remapping information if using a mounted TocCatalogue const eckit::LocalPathName& currentDirectory() const; const eckit::LocalPathName& currentTocPath() const; @@ -193,14 +179,13 @@ class TocHandler : public TocCommon, private eckit::NonCopyable { // Given the payload size, returns the record size - static size_t roundRecord(TocRecord &r, size_t payloadSize); + static size_t roundRecord(TocRecord& r, size_t payloadSize); void appendBlock(const void* data, size_t size); const TocSerialisationVersion& serialisationVersion() const; private: // methods - friend class TocHandlerCloser; void openForAppend(); @@ -218,19 +203,18 @@ class TocHandler : public TocCommon, private eckit::NonCopyable { void populateMaskedEntriesList() const; void preloadSubTocs(bool readMasked) const; - void append(TocRecord &r, size_t payloadSize); + void append(TocRecord& r, size_t payloadSize); // hideSubTocEntries=true returns entries as though only one toc existed (i.e. to hide // the mechanism of subtocs). // readMasked=true will walk subtocs and read indexes even if they are masked. This is // useful for dumping indexes which are cleared, or only referred to in cleared subtocs. - bool readNext(TocRecord &r, bool walkSubTocs = true, bool hideSubTocEntries = true, - bool hideClearEntries = true, bool readMasked = false, - const TocRecord** data=nullptr, size_t* length=nullptr) const; + bool readNext(TocRecord& r, bool walkSubTocs = true, bool hideSubTocEntries = true, bool hideClearEntries = true, + bool readMasked = false, const TocRecord** data = nullptr, size_t* length = nullptr) const; void selectSubTocRead(const eckit::LocalPathName& path) const; - bool readNextInternal(TocRecord &r, const TocRecord** data=nullptr, size_t* length=nullptr) const; + bool readNextInternal(TocRecord& r, const TocRecord** data = nullptr, size_t* length = nullptr) const; std::string userName(long) const; @@ -239,7 +223,6 @@ class TocHandler : public TocCommon, private eckit::NonCopyable { void dumpTocCache() const; private: // members - eckit::LocalPathName tocPath_; Config dbConfig_; @@ -253,7 +236,7 @@ class TocHandler : public TocCommon, private eckit::NonCopyable { // remapping key? Key remapKey_; - mutable int fd_; ///< file descriptor, if zero file is not yet open. + mutable int fd_; ///< file descriptor, if zero file is not yet open. mutable TocCopyWatcher tocReadStats_; mutable std::unique_ptr cachedToc_; ///< this is only for read path @@ -273,7 +256,6 @@ class TocHandler : public TocCommon, private eckit::NonCopyable { mutable bool dirty_; }; - //----------------------------------------------------------------------------- } // namespace fdb5 diff --git a/src/fdb5/toc/TocIndex.cc b/src/fdb5/toc/TocIndex.cc index bedcb4312..8cba8cdd7 100644 --- a/src/fdb5/toc/TocIndex.cc +++ b/src/fdb5/toc/TocIndex.cc @@ -11,11 +11,11 @@ #include "eckit/log/BigNum.h" #include "fdb5/LibFdb5.h" -#include "fdb5/toc/TocStats.h" -#include "fdb5/toc/TocIndex.h" #include "fdb5/toc/BTreeIndex.h" #include "fdb5/toc/FieldRef.h" #include "fdb5/toc/TocFieldLocation.h" +#include "fdb5/toc/TocIndex.h" +#include "fdb5/toc/TocStats.h" namespace fdb5 { @@ -23,11 +23,11 @@ namespace fdb5 { class TocIndexCloser { - const TocIndex &index_; + const TocIndex& index_; bool opened_; public: - TocIndexCloser(const TocIndex &index): index_(index), opened_(index.btree_) { + TocIndexCloser(const TocIndex& index) : index_(index), opened_(index.btree_) { if (!opened_) { const_cast(index_).open(); } @@ -47,26 +47,25 @@ class TocIndexCloser { /// before the type_ members of Index, but Indexs WILL be constructed before /// the members of TocIndex -TocIndex::TocIndex(const Key& key, const Catalogue& catalogue, const eckit::PathName &path, off_t offset, Mode mode, const std::string& type ) : - UriStoreWrapper(path.dirName()), - IndexBase(key, type, catalogue), - btree_(nullptr), - dirty_(false), - mode_(mode), - location_(path, offset), - preloadBTree_(false) { -} - -TocIndex::TocIndex(eckit::Stream &s, const Catalogue& catalogue, const int version, const eckit::PathName &directory, const eckit::PathName &path, - off_t offset, bool preloadBTree): - UriStoreWrapper(directory, s), - IndexBase(s, version, catalogue), - btree_(nullptr), - dirty_(false), - mode_(TocIndex::READ), - location_(path, offset), - preloadBTree_(preloadBTree) { -} +TocIndex::TocIndex(const Key& key, const Catalogue& catalogue, const eckit::PathName& path, off_t offset, Mode mode, + const std::string& type) + : UriStoreWrapper(path.dirName()) + , IndexBase(key, type, catalogue) + , btree_(nullptr) + , dirty_(false) + , mode_(mode) + , location_(path, offset) + , preloadBTree_(false) {} + +TocIndex::TocIndex(eckit::Stream& s, const Catalogue& catalogue, const int version, const eckit::PathName& directory, + const eckit::PathName& path, off_t offset, bool preloadBTree) + : UriStoreWrapper(directory, s) + , IndexBase(s, version, catalogue) + , btree_(nullptr) + , dirty_(false) + , mode_(TocIndex::READ) + , location_(path, offset) + , preloadBTree_(preloadBTree) {} TocIndex::~TocIndex() { close(); @@ -77,27 +76,27 @@ void TocIndex::encode(eckit::Stream& s, const int version) const { IndexBase::encode(s, version); } - -bool TocIndex::get(const Key& key, const Key& remapKey, Field &field) const { +bool TocIndex::get(const Key& key, const Key& remapKey, Field& field) const { ASSERT(btree_); FieldRef ref; bool found = btree_->get(key.valuesToString(), ref); - if ( found ) { + if (found) { const eckit::URI& uri = uris_.get(ref.uriId()); - FieldLocation* loc = FieldLocationFactory::instance().build(uri.scheme(), uri, ref.offset(), ref.length(), remapKey); + FieldLocation* loc = + FieldLocationFactory::instance().build(uri.scheme(), uri, ref.offset(), ref.length(), remapKey); field = Field(std::move(*loc), timestamp_, ref.details()); - delete(loc); + delete (loc); } return found; } - void TocIndex::open() { if (!btree_) { LOG_DEBUG_LIB(LibFdb5) << "Opening " << *this << std::endl; btree_.reset(BTreeIndexFactory::build(type_, location_.path_, mode_ == TocIndex::READ, location_.offset_)); - if (mode_ == TocIndex::READ && preloadBTree_) btree_->preload(); + if (mode_ == TocIndex::READ && preloadBTree_) + btree_->preload(); } } @@ -124,9 +123,9 @@ void TocIndex::close() { } } -void TocIndex::add(const Key& key, const Field &field) { +void TocIndex::add(const Key& key, const Field& field) { ASSERT(btree_); - ASSERT( mode_ == TocIndex::WRITE ); + ASSERT(mode_ == TocIndex::WRITE); FieldRef ref(uris_, field); @@ -134,11 +133,10 @@ void TocIndex::add(const Key& key, const Field &field) { btree_->set(key.valuesToString(), ref); // returns true if replace, false if new insert dirty_ = true; - } void TocIndex::flush() { - ASSERT( mode_ == TocIndex::WRITE ); + ASSERT(mode_ == TocIndex::WRITE); if (dirty_) { axes_.sort(); @@ -150,8 +148,7 @@ void TocIndex::flush() { } } - -void TocIndex::visit(IndexLocationVisitor &visitor) const { +void TocIndex::visit(IndexLocationVisitor& visitor) const { visitor(location_); } @@ -165,12 +162,11 @@ void TocIndex::funlock() const { } class TocIndexVisitor : public BTreeIndexVisitor { - const UriStore &uris_; - EntryVisitor &visitor_; + const UriStore& uris_; + EntryVisitor& visitor_; + public: - TocIndexVisitor(const UriStore &uris, EntryVisitor &visitor): - uris_(uris), - visitor_(visitor) {} + TocIndexVisitor(const UriStore& uris, EntryVisitor& visitor) : uris_(uris), visitor_(visitor) {} void visit(const std::string& keyFingerprint, const FieldRef& ref) { @@ -179,7 +175,7 @@ class TocIndexVisitor : public BTreeIndexVisitor { } }; -void TocIndex::entries(EntryVisitor &visitor) const { +void TocIndex::entries(EntryVisitor& visitor) const { Index instantIndex(const_cast(this)); @@ -191,11 +187,10 @@ void TocIndex::entries(EntryVisitor &visitor) const { } } -void TocIndex::print(std::ostream &out) const { - out << "TocIndex(path=" << location_.path_ << ",offset="<< location_.offset_ << ")"; +void TocIndex::print(std::ostream& out) const { + out << "TocIndex(path=" << location_.path_ << ",offset=" << location_.offset_ << ")"; } - std::string TocIndex::defaulType() { return BTreeIndex::defaulType(); } @@ -208,12 +203,11 @@ bool TocIndex::dirty() const { return dirty_; } - class DumpBTreeVisitor : public BTreeIndexVisitor { std::ostream& out_; std::string indent_; -public: +public: DumpBTreeVisitor(std::ostream& out, const std::string& indent) : out_(out), indent_(indent) {} virtual ~DumpBTreeVisitor() {} @@ -223,11 +217,10 @@ class DumpBTreeVisitor : public BTreeIndexVisitor { } }; - -void TocIndex::dump(std::ostream &out, const char* indent, bool simple, bool dumpFields) const { +void TocIndex::dump(std::ostream& out, const char* indent, bool simple, bool dumpFields) const { out << indent << "Key: " << key_; - if(!simple) { + if (!simple) { out << std::endl; uris_.dump(out, indent); axes_.dump(out, indent); @@ -244,8 +237,7 @@ void TocIndex::dump(std::ostream &out, const char* indent, bool simple, bool dum } } -IndexStats TocIndex::statistics() const -{ +IndexStats TocIndex::statistics() const { IndexStats s(new TocIndexStats()); return s; } diff --git a/src/fdb5/toc/TocIndex.h b/src/fdb5/toc/TocIndex.h index d112c6c4d..4b64917f7 100644 --- a/src/fdb5/toc/TocIndex.h +++ b/src/fdb5/toc/TocIndex.h @@ -34,7 +34,6 @@ namespace fdb5 { class BTreeIndex; - /// FileStoreWrapper exists _only_ so that the uris_ member can be initialised from the stream /// before the Index base class is initialised, for the TocIndex class. This order is required /// to preserve the order that data is stored/read from streams from before the uris_ object @@ -50,31 +49,17 @@ struct UriStoreWrapper { //---------------------------------------------------------------------------------------------------------------------- - -class TocIndex : - private UriStoreWrapper, - public IndexBase { +class TocIndex : private UriStoreWrapper, public IndexBase { public: // types - enum Mode { WRITE, READ }; public: // methods - - TocIndex(const Key& key, - const Catalogue& catalogue, - const eckit::PathName &path, - off_t offset, - Mode mode, + TocIndex(const Key& key, const Catalogue& catalogue, const eckit::PathName& path, off_t offset, Mode mode, const std::string& type = defaulType()); - TocIndex(eckit::Stream &, - const Catalogue& catalogue, - const int version, - const eckit::PathName &directory, - const eckit::PathName &path, - off_t offset, - bool preloadBTree=false); + TocIndex(eckit::Stream&, const Catalogue& catalogue, const int version, const eckit::PathName& directory, + const eckit::PathName& path, off_t offset, bool preloadBTree = false); ~TocIndex() override; @@ -85,9 +70,8 @@ class TocIndex : void flock() const override; void funlock() const override; - -private: // methods +private: // methods const IndexLocation& location() const override { return location_; } const std::vector dataURIs() const override; @@ -99,20 +83,19 @@ class TocIndex : void visit(IndexLocationVisitor& visitor) const override; - bool get(const Key& key, const Key& remapKey, Field& field ) const override; - void add(const Key& key, const Field& field ) override; + bool get(const Key& key, const Key& remapKey, Field& field) const override; + void add(const Key& key, const Field& field) override; void flush() override; void encode(eckit::Stream& s, const int version) const override; void entries(EntryVisitor& visitor) const override; - void print( std::ostream &out ) const override; + void print(std::ostream& out) const override; void dump(std::ostream& out, const char* indent, bool simple = false, bool dumpFields = false) const override; IndexStats statistics() const override; private: // members - - std::unique_ptr btree_; + std::unique_ptr btree_; bool dirty_; @@ -135,7 +118,7 @@ class TocIndex : struct TocIndexFileSort { // Return true if first argument is earlier than the second, and false otherwise. - bool operator() (const Index& lhs, const Index& rhs) { + bool operator()(const Index& lhs, const Index& rhs) { const TocIndex* idx1 = dynamic_cast(lhs.content()); const TocIndex* idx2 = dynamic_cast(rhs.content()); diff --git a/src/fdb5/toc/TocIndexLocation.cc b/src/fdb5/toc/TocIndexLocation.cc index 519d84082..e1cd0e011 100644 --- a/src/fdb5/toc/TocIndexLocation.cc +++ b/src/fdb5/toc/TocIndexLocation.cc @@ -18,17 +18,17 @@ using namespace eckit; - namespace fdb5 { -::eckit::ClassSpec TocIndexLocation::classSpec_ = {&IndexLocation::classSpec(), "TocIndexLocation",}; +::eckit::ClassSpec TocIndexLocation::classSpec_ = { + &IndexLocation::classSpec(), + "TocIndexLocation", +}; ::eckit::Reanimator TocIndexLocation::reanimator_; //---------------------------------------------------------------------------------------------------------------------- -TocIndexLocation::TocIndexLocation(const eckit::PathName& path, off_t offset) : - path_(path), - offset_(offset) {} +TocIndexLocation::TocIndexLocation(const eckit::PathName& path, off_t offset) : path_(path), offset_(offset) {} TocIndexLocation::TocIndexLocation(Stream& s) { s >> path_; @@ -44,8 +44,7 @@ off_t TocIndexLocation::offset() const { return path_; }*/ -URI TocIndexLocation::uri() const -{ +URI TocIndexLocation::uri() const { return URI("toc", path_); } @@ -58,9 +57,8 @@ void TocIndexLocation::encode(Stream& s) const { s << offset_; } -void TocIndexLocation::print(std::ostream &out) const { +void TocIndexLocation::print(std::ostream& out) const { out << "(" << path_ << ":" << offset_ << ")"; - } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/toc/TocIndexLocation.h b/src/fdb5/toc/TocIndexLocation.h index 6f6b15e95..08e7a5c9c 100644 --- a/src/fdb5/toc/TocIndexLocation.h +++ b/src/fdb5/toc/TocIndexLocation.h @@ -21,50 +21,42 @@ #include "fdb5/database/IndexLocation.h" - namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- - class TocIndexLocation : public IndexLocation { public: // methods - TocIndexLocation(const eckit::PathName& path, off_t offset); TocIndexLocation(eckit::Stream&); off_t offset() const; -// eckit::PathName path() const override; + // eckit::PathName path() const override; eckit::URI uri() const override; IndexLocation* clone() const override; public: // For Streamable - - static const eckit::ClassSpec& classSpec() { return classSpec_;} + static const eckit::ClassSpec& classSpec() { return classSpec_; } protected: // For Streamable - const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } void encode(eckit::Stream&) const override; - static eckit::ClassSpec classSpec_; + static eckit::ClassSpec classSpec_; static eckit::Reanimator reanimator_; private: // methods - - void print(std::ostream &out) const override; + void print(std::ostream& out) const override; private: // members - eckit::PathName path_; off_t offset_; private: // friends - friend class TocIndex; }; diff --git a/src/fdb5/toc/TocMoveVisitor.cc b/src/fdb5/toc/TocMoveVisitor.cc index 261bf1686..c9e00a1df 100644 --- a/src/fdb5/toc/TocMoveVisitor.cc +++ b/src/fdb5/toc/TocMoveVisitor.cc @@ -8,9 +8,9 @@ * does it submit to any jurisdiction. */ +#include #include #include -#include #include "eckit/config/Resource.h" #include "eckit/distributed/Message.h" @@ -19,15 +19,15 @@ #include "fdb5/api/helpers/ControlIterator.h" #include "fdb5/database/Catalogue.h" #include "fdb5/database/Store.h" +#include "fdb5/toc/RootManager.h" #include "fdb5/toc/TocCatalogue.h" #include "fdb5/toc/TocMoveVisitor.h" -#include "fdb5/toc/RootManager.h" +#include #include #include -#include #include -#include +#include using namespace eckit; @@ -37,15 +37,13 @@ namespace fdb5 { // TODO: Warnings and errors form inside here back to the user. -TocMoveVisitor::TocMoveVisitor(const TocCatalogue& catalogue, - const Store& store, - const metkit::mars::MarsRequest& request, - const eckit::URI& dest, - eckit::Queue& queue) : - MoveVisitor(request, dest), - catalogue_(catalogue), - store_(store), - queue_(queue) {} +TocMoveVisitor::TocMoveVisitor(const TocCatalogue& catalogue, const Store& store, + const metkit::mars::MarsRequest& request, const eckit::URI& dest, + eckit::Queue& queue) + : MoveVisitor(request, dest) + , catalogue_(catalogue) + , store_(store) + , queue_(queue) {} TocMoveVisitor::~TocMoveVisitor() {} @@ -60,10 +58,10 @@ bool TocMoveVisitor::visitDatabase(const Catalogue& catalogue) { DIR* dirp = ::opendir(catalogue_.basePath().c_str()); struct dirent* dp; while ((dp = readdir(dirp)) != NULL) { - if (strstr( dp->d_name, ".index")) { + if (strstr(dp->d_name, ".index")) { eckit::PathName src_ = PathName(catalogue_.basePath()) / dp->d_name; int fd = ::open(src_.asString().c_str(), O_RDWR); - if(::flock(fd, LOCK_EX)) { + if (::flock(fd, LOCK_EX)) { std::stringstream ss; ss << "Index file " << dp->d_name << " is locked"; throw eckit::UserError(ss.str(), Here()); @@ -77,13 +75,13 @@ bool TocMoveVisitor::visitDatabase(const Catalogue& catalogue) { eckit::PathName destPath = dest_.path(); bool found = false; - - for (const eckit::PathName& root: CatalogueRootManager(catalogue_.config()).canMoveToRoots(catalogue_.key())) { + + for (const eckit::PathName& root : CatalogueRootManager(catalogue_.config()).canMoveToRoots(catalogue_.key())) { if (root.sameAs(destPath)) { eckit::PathName dest_db = destPath / catalogue_.basePath().baseName(true); - if(dest_db.exists()) { + if (dest_db.exists()) { std::stringstream ss; ss << "Target folder already exist!" << std::endl; throw UserError(ss.str(), Here()); @@ -94,7 +92,8 @@ bool TocMoveVisitor::visitDatabase(const Catalogue& catalogue) { } if (!found) { std::stringstream ss; - ss << "Destination " << dest_ << " cannot be used to archive a DB with key: " << catalogue_.key() << std::endl; + ss << "Destination " << dest_ << " cannot be used to archive a DB with key: " << catalogue_.key() + << std::endl; throw eckit::UserError(ss.str(), Here()); } } else { @@ -115,20 +114,18 @@ void TocMoveVisitor::move() { store_.moveTo(catalogue_.key(), catalogue_.config(), dest_, queue_); eckit::PathName destPath = dest_.path(); - for (const eckit::PathName& root: CatalogueRootManager(catalogue_.config()).canMoveToRoots(catalogue_.key())) { + for (const eckit::PathName& root : CatalogueRootManager(catalogue_.config()).canMoveToRoots(catalogue_.key())) { if (root.sameAs(destPath)) { eckit::PathName dest_db = destPath / catalogue_.basePath().baseName(true); - if(!dest_db.exists()) { + if (!dest_db.exists()) { dest_db.mkdir(); } - + DIR* dirp = ::opendir(catalogue_.basePath().c_str()); struct dirent* dp; while ((dp = readdir(dirp)) != NULL) { - if (strstr( dp->d_name, ".index") || - strstr( dp->d_name, "toc.") || - strstr( dp->d_name, "schema")) { + if (strstr(dp->d_name, ".index") || strstr(dp->d_name, "toc.") || strstr(dp->d_name, "schema")) { FileCopy fileCopy(catalogue_.basePath(), dest_db, dp->d_name); queue_.emplace(fileCopy); @@ -141,8 +138,7 @@ void TocMoveVisitor::move() { dirp = ::opendir(catalogue_.basePath().c_str()); while ((dp = readdir(dirp)) != NULL) { - if (strstr( dp->d_name, ".lock") || - strstr( dp->d_name, "duplicates.allow")) { + if (strstr(dp->d_name, ".lock") || strstr(dp->d_name, "duplicates.allow")) { FileCopy fileCopy(catalogue_.basePath(), dest_db, dp->d_name); queue_.emplace(fileCopy); diff --git a/src/fdb5/toc/TocMoveVisitor.h b/src/fdb5/toc/TocMoveVisitor.h index ca18725a7..6093d101f 100644 --- a/src/fdb5/toc/TocMoveVisitor.h +++ b/src/fdb5/toc/TocMoveVisitor.h @@ -25,12 +25,8 @@ namespace fdb5 { class TocMoveVisitor : public MoveVisitor { public: - - TocMoveVisitor(const TocCatalogue& catalogue, - const Store& store, - const metkit::mars::MarsRequest& request, - const eckit::URI& dest, - eckit::Queue& queue); + TocMoveVisitor(const TocCatalogue& catalogue, const Store& store, const metkit::mars::MarsRequest& request, + const eckit::URI& dest, eckit::Queue& queue); ~TocMoveVisitor() override; private: // methods @@ -39,7 +35,6 @@ class TocMoveVisitor : public MoveVisitor { void move(); private: // members - // What are the parameters of the move operation const TocCatalogue& catalogue_; const Store& store_; diff --git a/src/fdb5/toc/TocPurgeVisitor.cc b/src/fdb5/toc/TocPurgeVisitor.cc index 8b49e4c9c..fcd27cb07 100644 --- a/src/fdb5/toc/TocPurgeVisitor.cc +++ b/src/fdb5/toc/TocPurgeVisitor.cc @@ -13,8 +13,8 @@ #include "eckit/log/Bytes.h" #include "eckit/log/Plural.h" -#include "fdb5/toc/TocHandler.h" #include "fdb5/LibFdb5.h" +#include "fdb5/toc/TocHandler.h" using namespace eckit; @@ -22,10 +22,10 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -TocPurgeVisitor::TocPurgeVisitor(const TocCatalogue& catalogue, const Store& store) : - PurgeVisitor(), - TocStatsReportVisitor(catalogue, false), - store_(store) {} +TocPurgeVisitor::TocPurgeVisitor(const TocCatalogue& catalogue, const Store& store) + : PurgeVisitor() + , TocStatsReportVisitor(catalogue, false) + , store_(store) {} TocPurgeVisitor::~TocPurgeVisitor() {} @@ -73,7 +73,8 @@ void TocPurgeVisitor::gatherAuxiliaryURIs() { // Add auxiliary files to the corresponding set eckit::URI uri(store_.type(), eckit::PathName(it.first)); for (const auto& auxURI : store_.getAuxiliaryURIs(uri)) { - if (!store_.auxiliaryURIExists(auxURI)) continue; + if (!store_.auxiliaryURIExists(auxURI)) + continue; // Todo: in future can we just use URIs, not paths? eckit::PathName auxPath = auxURI.path(); if (deletable) { @@ -207,7 +208,7 @@ void TocPurgeVisitor::purge(std::ostream& out, bool porcelain, bool doit) const if (path.dirName().sameAs(directory)) { currentCatalogue->remove(path, logAlways, logVerbose, doit); } - } + } } } diff --git a/src/fdb5/toc/TocPurgeVisitor.h b/src/fdb5/toc/TocPurgeVisitor.h index 96313be8a..7d8212262 100644 --- a/src/fdb5/toc/TocPurgeVisitor.h +++ b/src/fdb5/toc/TocPurgeVisitor.h @@ -16,7 +16,6 @@ #ifndef fdb5_TocPurgeVisitor_H #define fdb5_TocPurgeVisitor_H - #include "fdb5/database/PurgeVisitor.h" #include "fdb5/database/Store.h" #include "fdb5/toc/TocStats.h" @@ -25,10 +24,8 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- - class TocPurgeVisitor : public PurgeVisitor, public TocStatsReportVisitor { public: - TocPurgeVisitor(const TocCatalogue& catalogue, const Store& store); ~TocPurgeVisitor() override; @@ -39,7 +36,6 @@ class TocPurgeVisitor : public PurgeVisitor, public TocStatsReportVisitor { void gatherAuxiliaryURIs() override; private: // members - const Store& store_; std::set deleteAuxFiles_; std::set keepAuxFiles_; diff --git a/src/fdb5/toc/TocRecord.cc b/src/fdb5/toc/TocRecord.cc index 6c5fdc215..4c9207433 100644 --- a/src/fdb5/toc/TocRecord.cc +++ b/src/fdb5/toc/TocRecord.cc @@ -8,39 +8,38 @@ * does it submit to any jurisdiction. */ -#include #include +#include -#include "fdb5/fdb5_version.h" #include "fdb5/LibFdb5.h" +#include "fdb5/fdb5_version.h" #include #include "TocRecord.h" -#include "eckit/memory/Zero.h" -#include "eckit/log/TimeStamp.h" #include "eckit/log/Log.h" +#include "eckit/log/TimeStamp.h" +#include "eckit/memory/Zero.h" #include "eckit/runtime/Main.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -TocRecord::Header::Header(unsigned int serialisationVersion, unsigned char tag): - tag_(tag) { +TocRecord::Header::Header(unsigned int serialisationVersion, unsigned char tag) : tag_(tag) { if (tag_ != TOC_NULL) { eckit::zero(*this); - tag_ = tag; + tag_ = tag; serialisationVersion_ = serialisationVersion; fdbVersion_ = ::fdb5_version_int(); - SYSCALL( ::gettimeofday( ×tamp_, 0 ) ); + SYSCALL(::gettimeofday(×tamp_, 0)); - pid_ = ::getpid(); - uid_ = ::getuid(); + pid_ = ::getpid(); + uid_ = ::getuid(); std::string host = eckit::Main::hostname(); host = host.substr(0, host.find(".")); // guaranteed to be less than 64 chars -- seee RFC 1035 @@ -50,8 +49,7 @@ TocRecord::Header::Header(unsigned int serialisationVersion, unsigned char tag): //---------------------------------------------------------------------------------------------------------------------- -TocRecord::TocRecord(unsigned int serialisationVersion, unsigned char tag): - header_(serialisationVersion, tag) {} +TocRecord::TocRecord(unsigned int serialisationVersion, unsigned char tag) : header_(serialisationVersion, tag) {} void TocRecord::dump(std::ostream& out, bool simple) const { @@ -80,39 +78,26 @@ void TocRecord::dump(std::ostream& out, bool simple) const { std::ostringstream oss; oss << (header_.timestamp_.tv_usec / 1000000.0); - out << " " - << eckit::TimeStamp(header_.timestamp_.tv_sec) - << "." - << std::setw(6) - << std::left - << std::setfill('0') - << oss.str().substr(2) - << std::setfill(' ') - << ", version:" - << header_.serialisationVersion_ - << ", fdb: " - << header_.fdbVersion_ - << ", uid: " - << std::setw(4); - - struct passwd * p = getpwuid(header_.uid_); + out << " " << eckit::TimeStamp(header_.timestamp_.tv_sec) << "." << std::setw(6) << std::left << std::setfill('0') + << oss.str().substr(2) << std::setfill(' ') << ", version:" << header_.serialisationVersion_ + << ", fdb: " << header_.fdbVersion_ << ", uid: " << std::setw(4); + + struct passwd* p = getpwuid(header_.uid_); if (p) { - out << p->pw_name; + out << p->pw_name; } else { out << header_.uid_; } - out << ", pid " - << std::setw(5) - << header_.pid_ - << ", host: " - << header_.hostname_; + out << ", pid " << std::setw(5) << header_.pid_ << ", host: " << header_.hostname_; - if(!simple) { out << std::endl; } + if (!simple) { + out << std::endl; + } } -void TocRecord::print(std::ostream & out) const { +void TocRecord::print(std::ostream& out) const { out << "TocRecord[" << "tag=" << header_.tag_ << "," << "tocVersion=" << header_.serialisationVersion_ << "," diff --git a/src/fdb5/toc/TocRecord.h b/src/fdb5/toc/TocRecord.h index 3cf193664..bc6fa6cf1 100644 --- a/src/fdb5/toc/TocRecord.h +++ b/src/fdb5/toc/TocRecord.h @@ -14,12 +14,12 @@ #ifndef fdb5_TocRecord_H #define fdb5_TocRecord_H -#include #include #include +#include -#include "eckit/types/FixedString.h" #include "eckit/filesystem/PathName.h" +#include "eckit/types/FixedString.h" namespace fdb5 { @@ -36,42 +36,36 @@ struct TocRecord { /// then this use of TOC_CLEAR will have no impact. Therefore this is backward compatible /// and does not require bumping the format version (making roll-out more straightforward). - enum { - TOC_NULL = 0, - TOC_INIT = 't', - TOC_INDEX = 'i', - TOC_CLEAR = 'c', - TOC_SUB_TOC = 's' - }; + enum { TOC_NULL = 0, TOC_INIT = 't', TOC_INDEX = 'i', TOC_CLEAR = 'c', TOC_SUB_TOC = 's' }; static const size_t maxPayloadSize = 1024 * 1024; TocRecord(unsigned int serialisationVersion, unsigned char tag = TOC_NULL); struct Header { - unsigned char tag_; ///< (1) tag identifying the TocRecord type - unsigned char spare_[3]; ///< (3) padding - unsigned int serialisationVersion_; ///< (4) serialisation version of the TocRecord - unsigned int fdbVersion_; ///< (4) version of FDB writing this entry - timeval timestamp_; ///< (16) date & time of entry (in Unix seconds) - pid_t pid_; ///< (4) process PID - uid_t uid_; ///< (4) user ID - eckit::FixedString<64> hostname_; ///< (64) hostname adding entry - size_t size_; ///< (8) Record size + unsigned char tag_; ///< (1) tag identifying the TocRecord type + unsigned char spare_[3]; ///< (3) padding + unsigned int serialisationVersion_; ///< (4) serialisation version of the TocRecord + unsigned int fdbVersion_; ///< (4) version of FDB writing this entry + timeval timestamp_; ///< (16) date & time of entry (in Unix seconds) + pid_t pid_; ///< (4) process PID + uid_t uid_; ///< (4) user ID + eckit::FixedString<64> hostname_; ///< (64) hostname adding entry + size_t size_; ///< (8) Record size Header(unsigned int serialisationVersion, unsigned char tag); }; - Header header_; - unsigned char payload_[maxPayloadSize]; + Header header_; + unsigned char payload_[maxPayloadSize]; static const size_t headerSize = sizeof(Header); void dump(std::ostream& out, bool simple = false) const; - void print(std::ostream &out) const; + void print(std::ostream& out) const; - friend std::ostream &operator<<(std::ostream &s, const TocRecord &r) { + friend std::ostream& operator<<(std::ostream& s, const TocRecord& r) { r.print(s); return s; } diff --git a/src/fdb5/toc/TocSerialisationVersion.cc b/src/fdb5/toc/TocSerialisationVersion.cc index 6a759ab9e..736205fc4 100644 --- a/src/fdb5/toc/TocSerialisationVersion.cc +++ b/src/fdb5/toc/TocSerialisationVersion.cc @@ -19,9 +19,10 @@ static unsigned getUserEnvSerialisationVersion() { static unsigned fdbSerialisationVersion = eckit::Resource("fdbSerialisationVersion;$FDB5_SERIALISATION_VERSION", 0); - + if (fdbSerialisationVersion && fdbSerialisationVersion != TocSerialisationVersion::defaulted()) { - LOG_DEBUG_LIB(LibFdb5) << "fdbSerialisationVersion overidde to version: " << fdbSerialisationVersion << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "fdbSerialisationVersion overidde to version: " << fdbSerialisationVersion + << std::endl; } return fdbSerialisationVersion; // default is 0 (not defined by user/service) } @@ -33,7 +34,8 @@ TocSerialisationVersion::TocSerialisationVersion(const fdb5::Config& config) { } else { static int tocSerialisationVersion = config.getInt("tocSerialisationVersion", 0); if (tocSerialisationVersion && tocSerialisationVersion != TocSerialisationVersion::defaulted()) { - LOG_DEBUG_LIB(LibFdb5) << "tocSerialisationVersion overidde to version: " << tocSerialisationVersion << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "tocSerialisationVersion overidde to version: " << tocSerialisationVersion + << std::endl; used_ = tocSerialisationVersion; } else { used_ = defaulted(); @@ -41,11 +43,10 @@ TocSerialisationVersion::TocSerialisationVersion(const fdb5::Config& config) { } bool valid = check(used_, false); - if(not valid) { + if (not valid) { std::ostringstream msg; - msg << "Unsupported FDB5 toc serialisation version " << envVersion - << " - supported: " << supportedStr() - << std::endl; + msg << "Unsupported FDB5 toc serialisation version " << envVersion << " - supported: " << supportedStr() + << std::endl; throw eckit::BadValue(msg.str(), Here()); } } diff --git a/src/fdb5/toc/TocSerialisationVersion.h b/src/fdb5/toc/TocSerialisationVersion.h index 6e0cfb345..51bb0c535 100644 --- a/src/fdb5/toc/TocSerialisationVersion.h +++ b/src/fdb5/toc/TocSerialisationVersion.h @@ -49,10 +49,8 @@ class TocSerialisationVersion { /// Checks the serialisation version is supported by the software bool check(unsigned int version, bool throwOnFail = true) const; -private: // members - +private: // members unsigned int used_; //< version to be used for serialisation on write - }; } // namespace fdb5 diff --git a/src/fdb5/toc/TocStats.cc b/src/fdb5/toc/TocStats.cc index dde1f2692..b01b8da3f 100644 --- a/src/fdb5/toc/TocStats.cc +++ b/src/fdb5/toc/TocStats.cc @@ -19,29 +19,33 @@ using namespace eckit; namespace fdb5 { -::eckit::ClassSpec TocDbStats::classSpec_ = {&DbStatsContent::classSpec(), "TocDbStats",}; +::eckit::ClassSpec TocDbStats::classSpec_ = { + &DbStatsContent::classSpec(), + "TocDbStats", +}; ::eckit::Reanimator TocDbStats::reanimator_; -::eckit::ClassSpec TocIndexStats::classSpec_ = {&IndexStatsContent::classSpec(), "TocIndexStats",}; +::eckit::ClassSpec TocIndexStats::classSpec_ = { + &IndexStatsContent::classSpec(), + "TocIndexStats", +}; ::eckit::Reanimator TocIndexStats::reanimator_; //---------------------------------------------------------------------------------------------------------------------- -TocDbStats::TocDbStats(): - dbCount_(0), - tocRecordsCount_(0), - tocFileSize_(0), - schemaFileSize_(0), - ownedFilesSize_(0), - adoptedFilesSize_(0), - indexFilesSize_(0), - ownedFilesCount_(0), - adoptedFilesCount_(0), - indexFilesCount_(0) -{ -} +TocDbStats::TocDbStats() + : dbCount_(0) + , tocRecordsCount_(0) + , tocFileSize_(0) + , schemaFileSize_(0) + , ownedFilesSize_(0) + , adoptedFilesSize_(0) + , indexFilesSize_(0) + , ownedFilesCount_(0) + , adoptedFilesCount_(0) + , indexFilesCount_(0) {} -TocDbStats::TocDbStats(Stream &s) { +TocDbStats::TocDbStats(Stream& s) { s >> dbCount_; s >> tocRecordsCount_; @@ -57,7 +61,7 @@ TocDbStats::TocDbStats(Stream &s) { s >> indexFilesCount_; } -TocDbStats& TocDbStats::operator+=(const TocDbStats &rhs) { +TocDbStats& TocDbStats::operator+=(const TocDbStats& rhs) { dbCount_ += rhs.dbCount_; tocRecordsCount_ += rhs.tocRecordsCount_; @@ -73,13 +77,12 @@ TocDbStats& TocDbStats::operator+=(const TocDbStats &rhs) { return *this; } -void TocDbStats::add(const DbStatsContent& rhs) -{ +void TocDbStats::add(const DbStatsContent& rhs) { const TocDbStats& stats = dynamic_cast(rhs); *this += stats; } -void TocDbStats::report(std::ostream &out, const char *indent) const { +void TocDbStats::report(std::ostream& out, const char* indent) const { reportCount(out, "Databases", dbCount_, indent); reportCount(out, "TOC records", tocRecordsCount_, indent); @@ -96,9 +99,9 @@ void TocDbStats::report(std::ostream &out, const char *indent) const { reportBytes(out, "Size of index files", indexFilesSize_, indent); reportBytes(out, "Size of TOC files", tocFileSize_, indent); - reportBytes(out, "Total owned size", tocFileSize_ + schemaFileSize_ + indexFilesSize_ + ownedFilesSize_, indent); - reportBytes(out, "Total size", tocFileSize_ + schemaFileSize_ + indexFilesSize_ + ownedFilesSize_ + adoptedFilesSize_, indent); - + reportBytes(out, "Total owned size", tocFileSize_ + schemaFileSize_ + indexFilesSize_ + ownedFilesSize_, indent); + reportBytes(out, "Total size", + tocFileSize_ + schemaFileSize_ + indexFilesSize_ + ownedFilesSize_ + adoptedFilesSize_, indent); } void TocDbStats::encode(Stream& s) const { @@ -119,11 +122,7 @@ void TocDbStats::encode(Stream& s) const { //---------------------------------------------------------------------------------------------------------------------- -TocIndexStats::TocIndexStats(): - fieldsCount_(0), - duplicatesCount_(0), - fieldsSize_(0), - duplicatesSize_(0) {} +TocIndexStats::TocIndexStats() : fieldsCount_(0), duplicatesCount_(0), fieldsSize_(0), duplicatesSize_(0) {} TocIndexStats::TocIndexStats(Stream& s) { s >> fieldsCount_; @@ -132,8 +131,7 @@ TocIndexStats::TocIndexStats(Stream& s) { s >> duplicatesSize_; } - -TocIndexStats &TocIndexStats::operator+=(const TocIndexStats &rhs) { +TocIndexStats& TocIndexStats::operator+=(const TocIndexStats& rhs) { fieldsCount_ += rhs.fieldsCount_; duplicatesCount_ += rhs.duplicatesCount_; fieldsSize_ += rhs.fieldsSize_; @@ -142,13 +140,12 @@ TocIndexStats &TocIndexStats::operator+=(const TocIndexStats &rhs) { return *this; } -void TocIndexStats::add(const IndexStatsContent& rhs) -{ +void TocIndexStats::add(const IndexStatsContent& rhs) { const TocIndexStats& stats = dynamic_cast(rhs); *this += stats; } -void TocIndexStats::report(std::ostream &out, const char *indent) const { +void TocIndexStats::report(std::ostream& out, const char* indent) const { reportCount(out, "Fields", fieldsCount_, indent); reportBytes(out, "Size of fields", fieldsSize_, indent); reportCount(out, "Duplicated fields ", duplicatesCount_, indent); @@ -166,51 +163,44 @@ void TocIndexStats::encode(Stream& s) const { //---------------------------------------------------------------------------------------------------------------------- -TocDataStats::TocDataStats() { -} +TocDataStats::TocDataStats() {} TocDataStats& TocDataStats::operator+=(const TocDataStats& rhs) { std::set intersect; - std::set_union(allDataFiles_.begin(), - allDataFiles_.end(), - rhs.allDataFiles_.begin(), - rhs.allDataFiles_.end(), - std::insert_iterator< std::set>(intersect, intersect.begin())); + std::set_union(allDataFiles_.begin(), allDataFiles_.end(), rhs.allDataFiles_.begin(), rhs.allDataFiles_.end(), + std::insert_iterator>(intersect, intersect.begin())); std::swap(allDataFiles_, intersect); intersect.clear(); - std::set_union(activeDataFiles_.begin(), - activeDataFiles_.end(), - rhs.activeDataFiles_.begin(), + std::set_union(activeDataFiles_.begin(), activeDataFiles_.end(), rhs.activeDataFiles_.begin(), rhs.activeDataFiles_.end(), - std::insert_iterator< std::set>(intersect, intersect.begin())); + std::insert_iterator>(intersect, intersect.begin())); std::swap(activeDataFiles_, intersect); - for(const auto& [path, size] : rhs.dataUsage_) { + for (const auto& [path, size] : rhs.dataUsage_) { dataUsage_[path] += size; } return *this; } -void TocDataStats::add(const DataStatsContent& rhs) -{ +void TocDataStats::add(const DataStatsContent& rhs) { const TocDataStats& stats = dynamic_cast(rhs); *this += stats; } -void TocDataStats::report(std::ostream &out, const char *indent) const { +void TocDataStats::report(std::ostream& out, const char* indent) const { NOTIMP; } //---------------------------------------------------------------------------------------------------------------------- -TocStatsReportVisitor::TocStatsReportVisitor(const TocCatalogue& catalogue, bool includeReferenced) : - directory_(catalogue.basePath()), - includeReferencedNonOwnedData_(includeReferenced) { +TocStatsReportVisitor::TocStatsReportVisitor(const TocCatalogue& catalogue, bool includeReferenced) + : directory_(catalogue.basePath()) + , includeReferencedNonOwnedData_(includeReferenced) { currentCatalogue_ = &catalogue; dbStats_ = catalogue.stats(); @@ -223,7 +213,6 @@ bool TocStatsReportVisitor::visitDatabase(const Catalogue& catalogue) { return true; } - void TocStatsReportVisitor::visitDatum(const Field& field, const std::string& fieldFingerprint) { TocDbStats* dbStats = new TocDbStats(); @@ -232,8 +221,10 @@ void TocStatsReportVisitor::visitDatum(const Field& field, const std::string& fi if (!includeReferencedNonOwnedData_) { const TocCatalogue* cat = dynamic_cast(currentCatalogue_); - if (!currentIndex_->location().uri().path().dirName().sameAs(cat->basePath())) return; - if (!field.location().uri().path().dirName().sameAs(cat->basePath())) return; + if (!currentIndex_->location().uri().path().dirName().sameAs(cat->basePath())) + return; + if (!field.location().uri().path().dirName().sameAs(cat->basePath())) + return; } // If this index is not yet in the map, then create an entry @@ -251,7 +242,7 @@ void TocStatsReportVisitor::visitDatum(const Field& field, const std::string& fi stats.addFieldsCount(1); stats.addFieldsSize(len); - const eckit::PathName& dataPath = field.location().uri().path(); + const eckit::PathName& dataPath = field.location().uri().path(); const eckit::PathName& indexPath = currentIndex_->location().uri().path(); if (dataPath != lastDataPath_) { @@ -300,7 +291,6 @@ void TocStatsReportVisitor::visitDatum(const Field& field, const std::string& fi void TocStatsReportVisitor::catalogueComplete(const Catalogue& catalogue) {} - DbStats TocStatsReportVisitor::dbStatistics() const { return dbStats_; } @@ -316,5 +306,4 @@ IndexStats TocStatsReportVisitor::indexStatistics() const { //---------------------------------------------------------------------------------------------------------------------- - } // namespace fdb5 diff --git a/src/fdb5/toc/TocStats.h b/src/fdb5/toc/TocStats.h index 80700fc61..7e07d1d15 100644 --- a/src/fdb5/toc/TocStats.h +++ b/src/fdb5/toc/TocStats.h @@ -15,31 +15,28 @@ #ifndef fdb5_TocDbStats_H #define fdb5_TocDbStats_H -#include -#include #include +#include +#include #include "eckit/filesystem/PathName.h" +#include "fdb5/database/DataStats.h" #include "fdb5/database/DbStats.h" +#include "fdb5/database/Index.h" #include "fdb5/database/IndexStats.h" -#include "fdb5/database/DataStats.h" #include "fdb5/database/StatsReportVisitor.h" -#include "fdb5/database/Index.h" #include "fdb5/toc/TocCatalogueReader.h" -#include #include +#include namespace fdb5 { - //---------------------------------------------------------------------------------------------------------------------- - class TocDbStats : public DbStatsContent { public: - TocDbStats(); TocDbStats(eckit::Stream& s); @@ -58,32 +55,27 @@ class TocDbStats : public DbStatsContent { size_t adoptedFilesCount_; size_t indexFilesCount_; - TocDbStats& operator+= (const TocDbStats& rhs); + TocDbStats& operator+=(const TocDbStats& rhs); void add(const DbStatsContent&) override; - void report(std::ostream &out, const char* indent) const override; + void report(std::ostream& out, const char* indent) const override; public: // For Streamable - - static const eckit::ClassSpec& classSpec() { return classSpec_;} + static const eckit::ClassSpec& classSpec() { return classSpec_; } protected: // For Streamable - void encode(eckit::Stream&) const override; const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } - static eckit::ClassSpec classSpec_; + static eckit::ClassSpec classSpec_; static eckit::Reanimator reanimator_; }; - //---------------------------------------------------------------------------------------------------------------------- - class TocIndexStats : public IndexStatsContent { public: - TocIndexStats(); TocIndexStats(eckit::Stream& s); @@ -99,66 +91,69 @@ class TocIndexStats : public IndexStatsContent { size_t fieldsSize() const override { return fieldsSize_; } size_t duplicatesSize() const override { return duplicatesSize_; } - size_t addFieldsCount(size_t i) override { fieldsCount_ += i; return fieldsCount_; } - size_t addDuplicatesCount(size_t i) override { duplicatesCount_ += i; return duplicatesCount_; } - - size_t addFieldsSize(size_t i) override { fieldsSize_ += i; return fieldsSize_; } - size_t addDuplicatesSize(size_t i) override { duplicatesSize_ += i; return duplicatesSize_; } - - TocIndexStats& operator+= (const TocIndexStats& rhs); + size_t addFieldsCount(size_t i) override { + fieldsCount_ += i; + return fieldsCount_; + } + size_t addDuplicatesCount(size_t i) override { + duplicatesCount_ += i; + return duplicatesCount_; + } + + size_t addFieldsSize(size_t i) override { + fieldsSize_ += i; + return fieldsSize_; + } + size_t addDuplicatesSize(size_t i) override { + duplicatesSize_ += i; + return duplicatesSize_; + } + + TocIndexStats& operator+=(const TocIndexStats& rhs); void add(const IndexStatsContent&) override; - void report(std::ostream &out, const char* indent = "") const override; + void report(std::ostream& out, const char* indent = "") const override; public: // For Streamable - - static const eckit::ClassSpec& classSpec() { return classSpec_;} + static const eckit::ClassSpec& classSpec() { return classSpec_; } protected: // For Streamable - void encode(eckit::Stream&) const override; const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } - static eckit::ClassSpec classSpec_; + static eckit::ClassSpec classSpec_; static eckit::Reanimator reanimator_; }; - //---------------------------------------------------------------------------------------------------------------------- - class TocDataStats : public DataStatsContent { public: - TocDataStats(); std::set allDataFiles_; std::set activeDataFiles_; std::map dataUsage_; - TocDataStats& operator+= (const TocDataStats& rhs); + TocDataStats& operator+=(const TocDataStats& rhs); virtual void add(const DataStatsContent&); - virtual void report(std::ostream &out, const char* indent) const; + virtual void report(std::ostream& out, const char* indent) const; }; - //---------------------------------------------------------------------------------------------------------------------- - class TocStatsReportVisitor : public virtual StatsReportVisitor { public: - - TocStatsReportVisitor(const TocCatalogue& catalogue, bool includeReferenced=true); + TocStatsReportVisitor(const TocCatalogue& catalogue, bool includeReferenced = true); ~TocStatsReportVisitor() override; IndexStats indexStatistics() const override; - DbStats dbStatistics() const override; + DbStats dbStatistics() const override; private: // methods - bool visitDatabase(const Catalogue& catalogue) override; void visitDatum(const Field& field, const std::string& keyFingerprint) override; void visitDatum(const Field& field, const Key& datumKey) override { NOTIMP; } @@ -166,9 +161,7 @@ class TocStatsReportVisitor : public virtual StatsReportVisitor { // This visitor is only legit for one DB - so don't reset database void catalogueComplete(const Catalogue& catalogue) override; - protected: // members - eckit::PathName directory_; // Use of unordered_set/unordered_map is significant here from a performance perspective @@ -192,7 +185,6 @@ class TocStatsReportVisitor : public virtual StatsReportVisitor { bool includeReferencedNonOwnedData_; }; - //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 diff --git a/src/fdb5/toc/TocStore.cc b/src/fdb5/toc/TocStore.cc index f834b65bf..eafe0ae2e 100644 --- a/src/fdb5/toc/TocStore.cc +++ b/src/fdb5/toc/TocStore.cc @@ -18,15 +18,15 @@ #include "eckit/io/EmptyHandle.h" #include "fdb5/LibFdb5.h" -#include "fdb5/rules/Rule.h" #include "fdb5/database/FieldLocation.h" -#include "fdb5/toc/TocFieldLocation.h" +#include "fdb5/io/FDBFileHandle.h" +#include "fdb5/io/LustreFileHandle.h" +#include "fdb5/rules/Rule.h" #include "fdb5/toc/RootManager.h" +#include "fdb5/toc/TocFieldLocation.h" #include "fdb5/toc/TocPurgeVisitor.h" #include "fdb5/toc/TocStats.h" #include "fdb5/toc/TocStore.h" -#include "fdb5/io/FDBFileHandle.h" -#include "fdb5/io/LustreFileHandle.h" using namespace eckit; @@ -34,22 +34,21 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -TocStore::TocStore(const Key& key, const Config& config) : - Store(), - TocCommon(StoreRootManager(config).directory(key).directory_), - archivedFields_(0), auxFileExtensions_{auxFileExtensions()} {} +TocStore::TocStore(const Key& key, const Config& config) + : Store() + , TocCommon(StoreRootManager(config).directory(key).directory_) + , archivedFields_(0) + , auxFileExtensions_{auxFileExtensions()} {} eckit::URI TocStore::uri() const { return URI("file", directory_); - } bool TocStore::uriBelongs(const eckit::URI& uri) const { // TODO: assert uri represents a (not necessarily existing) data file return ((uri.scheme() == type()) && (uri.path().dirName().sameAs(directory_))); - } bool TocStore::uriExists(const eckit::URI& uri) const { @@ -63,10 +62,9 @@ bool TocStore::uriExists(const eckit::URI& uri) const { } return p.exists(); - } -bool TocStore::auxiliaryURIExists(const eckit::URI& uri) const { +bool TocStore::auxiliaryURIExists(const eckit::URI& uri) const { ASSERT(uri.scheme() == type()); eckit::PathName p(uri.path()); ASSERT(p.dirName().sameAs(directory_)); @@ -87,7 +85,6 @@ std::vector TocStore::collocatedDataURIs() const { } return res; - } std::set TocStore::asCollocatedDataURIs(const std::vector& uris) const { @@ -98,17 +95,14 @@ std::set TocStore::asCollocatedDataURIs(const std::vector TocStore::archive(const Key& idxKey, const eckit::PathName dataPath = getDataPath(idxKey); - eckit::DataHandle &dh = getDataHandle(dataPath); + eckit::DataHandle& dh = getDataHandle(dataPath); eckit::Offset position = dh.position(); - long len = dh.write( data, length ); + long len = dh.write(data, length); ASSERT(len == length); @@ -156,62 +150,60 @@ void TocStore::remove(const eckit::URI& uri, std::ostream& logAlways, std::ostre if (path.isDir()) { logVerbose << "rmdir: "; logAlways << path << std::endl; - if (doit) path.rmdir(false); + if (doit) + path.rmdir(false); } else { logVerbose << "Unlinking: "; logAlways << path << std::endl; - if (doit) path.unlink(false); + if (doit) + path.unlink(false); } } -eckit::DataHandle *TocStore::getCachedHandle( const eckit::PathName &path ) const { - HandleStore::const_iterator j = handles_.find( path ); - if ( j != handles_.end() ) +eckit::DataHandle* TocStore::getCachedHandle(const eckit::PathName& path) const { + HandleStore::const_iterator j = handles_.find(path); + if (j != handles_.end()) return j->second; else return nullptr; } void TocStore::closeDataHandles() { - for ( HandleStore::iterator j = handles_.begin(); j != handles_.end(); ++j ) { - eckit::DataHandle *dh = j->second; + for (HandleStore::iterator j = handles_.begin(); j != handles_.end(); ++j) { + eckit::DataHandle* dh = j->second; dh->close(); delete dh; } handles_.clear(); } -eckit::DataHandle *TocStore::createFileHandle(const eckit::PathName &path) { +eckit::DataHandle* TocStore::createFileHandle(const eckit::PathName& path) { static size_t sizeBuffer = eckit::Resource("fdbBufferSize", 64 * 1024 * 1024); - if(stripeLustre()) { + if (stripeLustre()) { - LOG_DEBUG_LIB(LibFdb5) << "Creating LustreFileHandle to " << path - << " buffer size " << sizeBuffer - << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "Creating LustreFileHandle to " << path << " buffer size " + << sizeBuffer << std::endl; return new LustreFileHandle(path, sizeBuffer, stripeDataLustreSettings()); } - LOG_DEBUG_LIB(LibFdb5) << "Creating FDBFileHandle to " << path - << " with buffer of " << eckit::Bytes(sizeBuffer) - << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "Creating FDBFileHandle to " << path << " with buffer of " << eckit::Bytes(sizeBuffer) + << std::endl; return new FDBFileHandle(path, sizeBuffer); } -eckit::DataHandle *TocStore::createAsyncHandle(const eckit::PathName &path) { +eckit::DataHandle* TocStore::createAsyncHandle(const eckit::PathName& path) { - static size_t nbBuffers = eckit::Resource("fdbNbAsyncBuffers", 4); + static size_t nbBuffers = eckit::Resource("fdbNbAsyncBuffers", 4); static size_t sizeBuffer = eckit::Resource("fdbSizeAsyncBuffer", 64 * 1024 * 1024); - if(stripeLustre()) { + if (stripeLustre()) { - LOG_DEBUG_LIB(LibFdb5) << "Creating LustreFileHandle to " << path - << " with " << nbBuffers - << " buffer each with " << eckit::Bytes(sizeBuffer) - << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "Creating LustreFileHandle to " << path << " with " << nbBuffers + << " buffer each with " << eckit::Bytes(sizeBuffer) << std::endl; return new LustreFileHandle(path, nbBuffers, sizeBuffer, stripeDataLustreSettings()); } @@ -219,22 +211,22 @@ eckit::DataHandle *TocStore::createAsyncHandle(const eckit::PathName &path) { return new eckit::AIOHandle(path, nbBuffers, sizeBuffer); } -eckit::DataHandle *TocStore::createDataHandle(const eckit::PathName &path) { +eckit::DataHandle* TocStore::createDataHandle(const eckit::PathName& path) { static bool fdbWriteToNull = eckit::Resource("fdbWriteToNull;$FDB_WRITE_TO_NULL", false); - if(fdbWriteToNull) + if (fdbWriteToNull) return new eckit::EmptyHandle(); static bool fdbAsyncWrite = eckit::Resource("fdbAsyncWrite;$FDB_ASYNC_WRITE", false); - if(fdbAsyncWrite) + if (fdbAsyncWrite) return createAsyncHandle(path); return createFileHandle(path); } -eckit::DataHandle& TocStore::getDataHandle( const eckit::PathName &path ) { - eckit::DataHandle *dh = getCachedHandle(path); - if ( !dh ) { +eckit::DataHandle& TocStore::getDataHandle(const eckit::PathName& path) { + eckit::DataHandle* dh = getCachedHandle(path); + if (!dh) { dh = createDataHandle(path); ASSERT(dh); handles_[path] = dh; @@ -245,20 +237,20 @@ eckit::DataHandle& TocStore::getDataHandle( const eckit::PathName &path ) { eckit::PathName TocStore::generateDataPath(const Key& key) const { - eckit::PathName dpath ( directory_ ); - dpath /= key.valuesToString(); + eckit::PathName dpath(directory_); + dpath /= key.valuesToString(); dpath = eckit::PathName::unique(dpath) + ".data"; return dpath; } eckit::PathName TocStore::getDataPath(const Key& key) const { PathStore::const_iterator j = dataPaths_.find(key); - if ( j != dataPaths_.end() ) + if (j != dataPaths_.end()) return j->second; eckit::PathName dataPath = generateDataPath(key); - dataPaths_[ key ] = dataPath; + dataPaths_[key] = dataPath; return dataPath; } @@ -266,7 +258,7 @@ eckit::PathName TocStore::getDataPath(const Key& key) const { void TocStore::flushDataHandles() { for (HandleStore::iterator j = handles_.begin(); j != handles_.end(); ++j) { - eckit::DataHandle *dh = j->second; + eckit::DataHandle* dh = j->second; dh->flush(); } } @@ -274,7 +266,7 @@ void TocStore::flushDataHandles() { bool TocStore::canMoveTo(const Key& key, const Config& config, const eckit::URI& dest) const { if (dest.scheme().empty() || dest.scheme() == "toc" || dest.scheme() == "file" || dest.scheme() == "unix") { eckit::PathName destPath = dest.path(); - for (const eckit::PathName& root: StoreRootManager(config).canMoveToRoots(key)) { + for (const eckit::PathName& root : StoreRootManager(config).canMoveToRoots(key)) { if (root.sameAs(destPath)) { return true; } @@ -319,10 +311,11 @@ std::set TocStore::auxFileExtensions() const { return extensions; } -void TocStore::moveTo(const Key& key, const Config& config, const eckit::URI& dest, eckit::Queue& queue) const { +void TocStore::moveTo(const Key& key, const Config& config, const eckit::URI& dest, + eckit::Queue& queue) const { eckit::PathName destPath = dest.path(); - for (const eckit::PathName& root: StoreRootManager(config).canMoveToRoots(key)) { + for (const eckit::PathName& root : StoreRootManager(config).canMoveToRoots(key)) { if (root.sameAs(destPath)) { eckit::PathName src_db = directory_; eckit::PathName dest_db = destPath / key.valuesToString(); @@ -332,9 +325,10 @@ void TocStore::moveTo(const Key& key, const Config& config, const eckit::URI& de struct dirent* dp; std::multimap> files; while ((dp = ::readdir(dirp)) != NULL) { - if (strstr( dp->d_name, ".data")) { + if (strstr(dp->d_name, ".data")) { eckit::PathName file(src_db / dp->d_name); - if ((file.extension() == ".data") || auxFileExtensions_.find(file.extension()) != auxFileExtensions_.end()) { + if ((file.extension() == ".data") || + auxFileExtensions_.find(file.extension()) != auxFileExtensions_.end()) { struct stat fileStat; SYSCALL(::stat(file.asString().c_str(), &fileStat)); files.emplace(fileStat.st_size, new FileCopy(src_db.path(), dest_db, dp->d_name)); @@ -357,7 +351,7 @@ void TocStore::remove(const Key& key) const { DIR* dirp = ::opendir(src_db.asString().c_str()); struct dirent* dp; while ((dp = ::readdir(dirp)) != NULL) { - if (strstr( dp->d_name, ".data")) { + if (strstr(dp->d_name, ".data")) { eckit::PathName dataFile = src_db / dp->d_name; LOG_DEBUG_LIB(LibFdb5) << "Removing " << dataFile << std::endl; dataFile.unlink(false); @@ -366,7 +360,7 @@ void TocStore::remove(const Key& key) const { closedir(dirp); } -void TocStore::print(std::ostream &out) const { +void TocStore::print(std::ostream& out) const { out << "TocStore(" << directory_ << ")"; } diff --git a/src/fdb5/toc/TocStore.h b/src/fdb5/toc/TocStore.h index 4a28260f8..b433b63c9 100644 --- a/src/fdb5/toc/TocStore.h +++ b/src/fdb5/toc/TocStore.h @@ -31,7 +31,6 @@ namespace fdb5 { class TocStore : public Store, public TocCommon { public: // methods - TocStore(const Key& key, const Config& config); ~TocStore() override {} @@ -49,7 +48,8 @@ class TocStore : public Store, public TocCommon { void checkUID() const override { TocCommon::checkUID(); } bool canMoveTo(const Key& key, const Config& config, const eckit::URI& dest) const override; - void moveTo(const Key& key, const Config& config, const eckit::URI& dest, eckit::Queue& queue) const override; + void moveTo(const Key& key, const Config& config, const eckit::URI& dest, + eckit::Queue& queue) const override; void remove(const Key& key) const override; std::vector getAuxiliaryURIs(const eckit::URI&) const override; @@ -57,48 +57,44 @@ class TocStore : public Store, public TocCommon { std::set auxFileExtensions() const; protected: // methods - std::string type() const override { return "file"; } bool exists() const override; eckit::DataHandle* retrieve(Field& field) const override; - std::unique_ptr archive(const Key& idxKey, const void *data, eckit::Length length) override; + std::unique_ptr archive(const Key& idxKey, const void* data, eckit::Length length) override; void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override; - eckit::DataHandle *getCachedHandle( const eckit::PathName &path ) const; + eckit::DataHandle* getCachedHandle(const eckit::PathName& path) const; void closeDataHandles(); - eckit::DataHandle *createFileHandle(const eckit::PathName &path); - eckit::DataHandle *createAsyncHandle(const eckit::PathName &path); - eckit::DataHandle *createDataHandle(const eckit::PathName &path); - eckit::DataHandle& getDataHandle( const eckit::PathName &path ); + eckit::DataHandle* createFileHandle(const eckit::PathName& path); + eckit::DataHandle* createAsyncHandle(const eckit::PathName& path); + eckit::DataHandle* createDataHandle(const eckit::PathName& path); + eckit::DataHandle& getDataHandle(const eckit::PathName& path); eckit::PathName generateDataPath(const Key& key) const; eckit::PathName getDataPath(const Key& key) const; void flushDataHandles(); - void print( std::ostream &out ) const override; + void print(std::ostream& out) const override; private: // methods eckit::URI getAuxiliaryURI(const eckit::URI&, const std::string& ext) const; private: // types + typedef std::map HandleStore; + typedef std::map PathStore; - typedef std::map< std::string, eckit::DataHandle * > HandleStore; - typedef std::map< Key, std::string > PathStore; - -private: // members +private: // members + HandleStore handles_; ///< stores the DataHandles being used by the Session - HandleStore handles_; ///< stores the DataHandles being used by the Session - - mutable PathStore dataPaths_; + mutable PathStore dataPaths_; size_t archivedFields_; std::set auxFileExtensions_; - }; //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 -#endif //fdb5_TocStore_H +#endif // fdb5_TocStore_H diff --git a/src/fdb5/toc/TocWipeVisitor.cc b/src/fdb5/toc/TocWipeVisitor.cc index bf2a13086..d1436e667 100644 --- a/src/fdb5/toc/TocWipeVisitor.cc +++ b/src/fdb5/toc/TocWipeVisitor.cc @@ -8,11 +8,10 @@ * does it submit to any jurisdiction. */ - #include +#include #include #include -#include #include #include @@ -34,13 +33,10 @@ namespace { class StdDir { eckit::PathName path_; - DIR *d_; + DIR* d_; public: - - StdDir(const eckit::PathName& p) : - path_(p), - d_(opendir(p.localPath())) { + StdDir(const eckit::PathName& p) : path_(p), d_(opendir(p.localPath())) { if (!d_) { std::stringstream ss; @@ -49,7 +45,10 @@ class StdDir { } } - ~StdDir() { if (d_) closedir(d_); } + ~StdDir() { + if (d_) + closedir(d_); + } void children(std::vector& paths) { @@ -60,7 +59,8 @@ class StdDir { while ((e = readdir(d_)) != nullptr) { if (e->d_name[0] == '.') { - if (e->d_name[1] == '\0' || (e->d_name[1] == '.' && e->d_name[2] == '\0')) continue; + if (e->d_name[1] == '\0' || (e->d_name[1] == '.' && e->d_name[2] == '\0')) + continue; } eckit::PathName p(path_ / e->d_name); @@ -78,28 +78,23 @@ class StdDir { } } }; -} +} // namespace //---------------------------------------------------------------------------------------------------------------------- // TODO: Warnings and errors form inside here back to the user. -TocWipeVisitor::TocWipeVisitor(const TocCatalogue& catalogue, - const Store& store, - const metkit::mars::MarsRequest& request, - std::ostream& out, - bool doit, - bool porcelain, - bool unsafeWipeAll) : - WipeVisitor(request, out, doit, porcelain, unsafeWipeAll), - catalogue_(catalogue), - store_(store), - tocPath_(""), - schemaPath_("") {} +TocWipeVisitor::TocWipeVisitor(const TocCatalogue& catalogue, const Store& store, + const metkit::mars::MarsRequest& request, std::ostream& out, bool doit, bool porcelain, + bool unsafeWipeAll) + : WipeVisitor(request, out, doit, porcelain, unsafeWipeAll) + , catalogue_(catalogue) + , store_(store) + , tocPath_("") + , schemaPath_("") {} TocWipeVisitor::~TocWipeVisitor() {} - bool TocWipeVisitor::visitDatabase(const Catalogue& catalogue) { // Overall checks @@ -167,10 +162,12 @@ bool TocWipeVisitor::visitIndex(const Index& index) { for (const eckit::URI& uri : store_.asCollocatedDataURIs(indexDataPaths)) { if (include) { if (!store_.uriBelongs(uri)) { - Log::error() << "Index to be deleted has pointers to fields that don't belong to the configured store." << std::endl; + Log::error() << "Index to be deleted has pointers to fields that don't belong to the configured store." + << std::endl; Log::error() << "Configured Store URI: " << store_.uri().asString() << std::endl; Log::error() << "Pointed Store unit URI: " << uri.asString() << std::endl; - Log::error() << "Impossible to delete such fields. Index deletion aborted to avoid leaking fields." << std::endl; + Log::error() << "Impossible to delete such fields. Index deletion aborted to avoid leaking fields." + << std::endl; NOTIMP; } dataPaths_.insert(eckit::PathName(uri.path())); @@ -192,14 +189,15 @@ std::vector TocWipeVisitor::getAuxiliaryPaths(const eckit::URI& // todo: in future, we should be using URIs, not paths. std::vector paths; for (const auto& auxURI : store_.getAuxiliaryURIs(dataURI)) { - if (store_.auxiliaryURIExists(auxURI)) paths.push_back(auxURI.path()); + if (store_.auxiliaryURIExists(auxURI)) + paths.push_back(auxURI.path()); } return paths; } void TocWipeVisitor::addMaskedPaths() { - //ASSERT(indexRequest_.empty()); + // ASSERT(indexRequest_.empty()); std::set> metadata; std::set data; @@ -245,11 +243,14 @@ void TocWipeVisitor::ensureSafePaths() { // Very explicitly ensure that we cannot delete anything marked as safe - if (safePaths_.find(tocPath_) != safePaths_.end()) tocPath_ = ""; - if (safePaths_.find(schemaPath_) != safePaths_.end()) schemaPath_ = ""; + if (safePaths_.find(tocPath_) != safePaths_.end()) + tocPath_ = ""; + if (safePaths_.find(schemaPath_) != safePaths_.end()) + schemaPath_ = ""; for (const auto& p : safePaths_) { - for (std::set* s : {&subtocPaths_, &lockfilePaths_, &indexPaths_, &dataPaths_, &auxiliaryDataPaths_}) { + for (std::set* s : + {&subtocPaths_, &lockfilePaths_, &indexPaths_, &dataPaths_, &auxiliaryDataPaths_}) { s->erase(p); } } @@ -261,30 +262,29 @@ void TocWipeVisitor::calculateResidualPaths() { // NB: Not needed for auxiliaryDataPaths_ as their existence is checked in getAuxiliaryPaths() for (std::set* fileset : {&subtocPaths_, &lockfilePaths_, &indexPaths_}) { - for (std::set::iterator it = fileset->begin(); it != fileset->end(); ) { + for (std::set::iterator it = fileset->begin(); it != fileset->end();) { if (it->exists()) { ++it; } else { fileset->erase(it++); } - } } for (std::set* fileset : {&dataPaths_}) { - for (std::set::iterator it = fileset->begin(); it != fileset->end(); ) { + for (std::set::iterator it = fileset->begin(); it != fileset->end();) { if (store_.uriExists(eckit::URI(store_.type(), *it))) { ++it; } else { fileset->erase(it++); } - } } - if (tocPath_.asString().size() && !tocPath_.exists()) tocPath_ = ""; + if (tocPath_.asString().size() && !tocPath_.exists()) + tocPath_ = ""; if (schemaPath_.asString().size() && !schemaPath_.exists()) schemaPath_ = ""; @@ -297,7 +297,8 @@ void TocWipeVisitor::calculateResidualPaths() { deletePaths.insert(indexPaths_.begin(), indexPaths_.end()); if (store_.type() == "file") deletePaths.insert(dataPaths_.begin(), dataPaths_.end()); - if (tocPath_.asString().size()) deletePaths.insert(tocPath_); + if (tocPath_.asString().size()) + deletePaths.insert(tocPath_); if (schemaPath_.asString().size()) deletePaths.insert(schemaPath_); @@ -313,8 +314,7 @@ void TocWipeVisitor::calculateResidualPaths() { // First we check if there are paths marked to delete that don't exist. This is an error std::set paths; - std::set_difference(deletePaths.begin(), deletePaths.end(), - allPaths.begin(), allPaths.end(), + std::set_difference(deletePaths.begin(), deletePaths.end(), allPaths.begin(), allPaths.end(), std::inserter(paths, paths.begin())); if (!paths.empty()) { @@ -322,25 +322,27 @@ void TocWipeVisitor::calculateResidualPaths() { for (const auto& p : paths) { Log::error() << " - " << p << std::endl; } - throw SeriousBug("Path to delete should be in existing path set. Are multiple wipe commands running simultaneously?", Here()); + throw SeriousBug( + "Path to delete should be in existing path set. Are multiple wipe commands running simultaneously?", + Here()); } - std::set_difference(allPaths.begin(), allPaths.end(), - deletePaths.begin(), deletePaths.end(), + std::set_difference(allPaths.begin(), allPaths.end(), deletePaths.begin(), deletePaths.end(), std::inserter(residualPaths_, residualPaths_.begin())); } - // if the store uses a backend other than POSIX (file), repeat the algorithm specialized + // if the store uses a backend other than POSIX (file), repeat the algorithm specialized // for its store units - if (store_.type() == "file") return; + if (store_.type() == "file") + return; std::vector allCollocatedDataURIs(store_.collocatedDataURIs()); std::vector allDataPathsVector; for (const auto& u : allCollocatedDataURIs) { allDataPathsVector.push_back(eckit::PathName(u.path())); } - + std::set allDataPaths(allDataPathsVector.begin(), allDataPathsVector.end()); ASSERT(residualDataPaths_.empty()); @@ -350,8 +352,7 @@ void TocWipeVisitor::calculateResidualPaths() { // First we check if there are paths marked to delete that don't exist. This is an error std::set paths; - std::set_difference(dataPaths_.begin(), dataPaths_.end(), - allDataPaths.begin(), allDataPaths.end(), + std::set_difference(dataPaths_.begin(), dataPaths_.end(), allDataPaths.begin(), allDataPaths.end(), std::inserter(paths, paths.begin())); if (!paths.empty()) { @@ -359,61 +360,67 @@ void TocWipeVisitor::calculateResidualPaths() { for (const auto& p : paths) { Log::error() << " - " << p << std::endl; } - throw SeriousBug("Store unit path to delete should be in existing path set. Are multiple wipe commands running simultaneously?", Here()); + throw SeriousBug( + "Store unit path to delete should be in existing path set. Are multiple wipe commands running " + "simultaneously?", + Here()); } - std::set_difference(allDataPaths.begin(), allDataPaths.end(), - dataPaths_.begin(), dataPaths_.end(), + std::set_difference(allDataPaths.begin(), allDataPaths.end(), dataPaths_.begin(), dataPaths_.end(), std::inserter(residualDataPaths_, residualDataPaths_.begin())); } - } bool TocWipeVisitor::anythingToWipe() const { - return (!subtocPaths_.empty() || !lockfilePaths_.empty() || !indexPaths_.empty() || - !dataPaths_.empty() || !indexesToMask_.empty() || - tocPath_.asString().size() || schemaPath_.asString().size()); + return (!subtocPaths_.empty() || !lockfilePaths_.empty() || !indexPaths_.empty() || !dataPaths_.empty() || + !indexesToMask_.empty() || tocPath_.asString().size() || schemaPath_.asString().size()); } void TocWipeVisitor::report(bool wipeAll) { ASSERT(anythingToWipe()); - out_ << "FDB owner: " << catalogue_.owner() << std::endl - << std::endl; + out_ << "FDB owner: " << catalogue_.owner() << std::endl << std::endl; out_ << "Toc files to delete:" << std::endl; - if (!tocPath_.asString().size() && subtocPaths_.empty()) out_ << " - NONE -" << std::endl; - if (tocPath_.asString().size()) out_ << " " << tocPath_ << std::endl; + if (!tocPath_.asString().size() && subtocPaths_.empty()) + out_ << " - NONE -" << std::endl; + if (tocPath_.asString().size()) + out_ << " " << tocPath_ << std::endl; for (const auto& f : subtocPaths_) { out_ << " " << f << std::endl; } out_ << std::endl; out_ << "Control files to delete:" << std::endl; - if (!schemaPath_.asString().size() && lockfilePaths_.empty()) out_ << " - NONE -" << std::endl; - if (schemaPath_.asString().size()) out_ << " " << schemaPath_ << std::endl; + if (!schemaPath_.asString().size() && lockfilePaths_.empty()) + out_ << " - NONE -" << std::endl; + if (schemaPath_.asString().size()) + out_ << " " << schemaPath_ << std::endl; for (const auto& f : lockfilePaths_) { out_ << " " << f << std::endl; } out_ << std::endl; out_ << "Index files to delete: " << std::endl; - if (indexPaths_.empty()) out_ << " - NONE -" << std::endl; + if (indexPaths_.empty()) + out_ << " - NONE -" << std::endl; for (const auto& f : indexPaths_) { out_ << " " << f << std::endl; } out_ << std::endl; out_ << "Data files to delete: " << std::endl; - if (dataPaths_.empty()) out_ << " - NONE -" << std::endl; + if (dataPaths_.empty()) + out_ << " - NONE -" << std::endl; for (const auto& f : dataPaths_) { out_ << " " << f << std::endl; } out_ << std::endl; out_ << "Auxiliary files to delete: " << std::endl; - if (auxiliaryDataPaths_.empty()) out_ << " - NONE -" << std::endl; + if (auxiliaryDataPaths_.empty()) + out_ << " - NONE -" << std::endl; for (const auto& f : auxiliaryDataPaths_) { out_ << " " << f << std::endl; } @@ -430,7 +437,8 @@ void TocWipeVisitor::report(bool wipeAll) { } out_ << "Protected files (explicitly untouched):" << std::endl; - if (safePaths_.empty()) out_ << " - NONE - " << std::endl; + if (safePaths_.empty()) + out_ << " - NONE - " << std::endl; for (const auto& f : safePaths_) { out_ << " " << f << std::endl; } @@ -438,7 +446,8 @@ void TocWipeVisitor::report(bool wipeAll) { if (!safePaths_.empty()) { out_ << "Indexes to mask:" << std::endl; - if (indexesToMask_.empty()) out_ << " - NONE - " << std::endl; + if (indexesToMask_.empty()) + out_ << " - NONE - " << std::endl; for (const auto& i : indexesToMask_) { out_ << " " << i.location() << std::endl; } @@ -460,9 +469,8 @@ void TocWipeVisitor::wipe(bool wipeAll) { // into a state we don't like. if (wipeAll && doit_) { - catalogue_.control(ControlAction::Disable, ControlIdentifier::List | - ControlIdentifier::Retrieve | - ControlIdentifier::Archive); + catalogue_.control(ControlAction::Disable, + ControlIdentifier::List | ControlIdentifier::Retrieve | ControlIdentifier::Archive); ASSERT(!catalogue_.enabled(ControlIdentifier::List)); ASSERT(!catalogue_.enabled(ControlIdentifier::Retrieve)); @@ -481,7 +489,8 @@ void TocWipeVisitor::wipe(bool wipeAll) { for (const auto& index : indexesToMask_) { logVerbose << "Index to mask: "; logAlways << index << std::endl; - if (doit_) catalogue_.maskIndexEntry(index); + if (doit_) + catalogue_.maskIndexEntry(index); } } @@ -517,14 +526,13 @@ void TocWipeVisitor::wipe(bool wipeAll) { } if (wipeAll && store_.type() != "file") - /// @todo: if the store is holding catalogue information (e.g. daos KVs) it + /// @todo: if the store is holding catalogue information (e.g. daos KVs) it /// should not be removed store_.remove(store_.uri(), logAlways, logVerbose, doit_); - for (const std::set& pathset : {indexPaths_, - std::set{schemaPath_}, subtocPaths_, - std::set{tocPath_}, lockfilePaths_, - (wipeAll ? std::set{catalogue_.basePath()} : std::set{})}) { + for (const std::set& pathset : + {indexPaths_, std::set{schemaPath_}, subtocPaths_, std::set{tocPath_}, lockfilePaths_, + (wipeAll ? std::set{catalogue_.basePath()} : std::set{})}) { for (const PathName& path : pathset) { if (path.exists()) { @@ -534,7 +542,6 @@ void TocWipeVisitor::wipe(bool wipeAll) { } } - void TocWipeVisitor::catalogueComplete(const Catalogue& catalogue) { WipeVisitor::catalogueComplete(catalogue); @@ -557,24 +564,26 @@ void TocWipeVisitor::catalogueComplete(const Catalogue& catalogue) { ensureSafePaths(); if (anythingToWipe()) { - if (wipeAll) calculateResidualPaths(); + if (wipeAll) + calculateResidualPaths(); - if (!porcelain_) report(wipeAll); + if (!porcelain_) + report(wipeAll); // This is here as it needs to run whatever combination of doit/porcelain/... if (wipeAll && !residualPaths_.empty()) { out_ << "Unexpected files present in directory: " << std::endl; - for (const auto& p : residualPaths_) out_ << " " << p << std::endl; + for (const auto& p : residualPaths_) + out_ << " " << p << std::endl; out_ << std::endl; - } if (wipeAll && !residualDataPaths_.empty()) { out_ << "Unexpected store units present in store: " << std::endl; - for (const auto& p : residualDataPaths_) out_ << " " << store_.type() << "://" << p << std::endl; + for (const auto& p : residualDataPaths_) + out_ << " " << store_.type() << "://" << p << std::endl; out_ << std::endl; - } if (wipeAll && (!residualPaths_.empty() || !residualDataPaths_.empty())) { if (!unsafeWipeAll_) { @@ -584,11 +593,11 @@ void TocWipeVisitor::catalogueComplete(const Catalogue& catalogue) { } } - if (doit_ || porcelain_) wipe(wipeAll); + if (doit_ || porcelain_) + wipe(wipeAll); } } - //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 diff --git a/src/fdb5/toc/TocWipeVisitor.h b/src/fdb5/toc/TocWipeVisitor.h index c8df9f5d2..042de93f1 100644 --- a/src/fdb5/toc/TocWipeVisitor.h +++ b/src/fdb5/toc/TocWipeVisitor.h @@ -14,7 +14,6 @@ #ifndef fdb5_TocWipeVisitor_H #define fdb5_TocWipeVisitor_H - #include "fdb5/database/WipeVisitor.h" #include "fdb5/toc/TocCatalogue.h" @@ -25,18 +24,11 @@ namespace fdb5 { class TocWipeVisitor : public WipeVisitor { public: - - TocWipeVisitor(const TocCatalogue& catalogue, - const Store& store, - const metkit::mars::MarsRequest& request, - std::ostream& out, - bool doit, - bool porcelain, - bool unsafeWipeAll); + TocWipeVisitor(const TocCatalogue& catalogue, const Store& store, const metkit::mars::MarsRequest& request, + std::ostream& out, bool doit, bool porcelain, bool unsafeWipeAll); ~TocWipeVisitor() override; private: // methods - bool visitDatabase(const Catalogue& catalogue) override; bool visitIndex(const Index& index) override; void catalogueComplete(const Catalogue& catalogue) override; @@ -53,7 +45,6 @@ class TocWipeVisitor : public WipeVisitor { void wipe(bool wipeAll); private: // members - // What are the parameters of the wipe operation const TocCatalogue& catalogue_; const Store& store_; diff --git a/src/fdb5/tools/FDBInspect.cc b/src/fdb5/tools/FDBInspect.cc index 3b18a4792..2ea7e7549 100644 --- a/src/fdb5/tools/FDBInspect.cc +++ b/src/fdb5/tools/FDBInspect.cc @@ -186,5 +186,4 @@ void FDBInspect::usage(const std::string &tool) const { } // namespace fdb5 - #endif diff --git a/src/fdb5/tools/FDBInspect.h b/src/fdb5/tools/FDBInspect.h index d84a41e00..e4ee6192f 100644 --- a/src/fdb5/tools/FDBInspect.h +++ b/src/fdb5/tools/FDBInspect.h @@ -28,23 +28,20 @@ namespace fdb5 { class FDBInspect : public FDBTool { protected: // methods + FDBInspect(int argc, char** argv, std::string minimunKeys = std::string()); - FDBInspect(int argc, char **argv, std::string minimunKeys = std::string()); + virtual void usage(const std::string& tool) const; - virtual void usage(const std::string &tool) const; - - virtual void init(const eckit::option::CmdArgs &args); + virtual void init(const eckit::option::CmdArgs& args); virtual void execute(const eckit::option::CmdArgs& args); bool fail() const; private: // methods - virtual void process(const eckit::PathName&, const eckit::option::CmdArgs& args) = 0; protected: // members - // minimum set of keys needed to execute a query // these are used for safety to restrict the search space for the visitors std::vector minimumKeys_; diff --git a/src/fdb5/tools/FDBLock.cc b/src/fdb5/tools/FDBLock.cc index 3531b9966..7ddee0352 100644 --- a/src/fdb5/tools/FDBLock.cc +++ b/src/fdb5/tools/FDBLock.cc @@ -21,21 +21,20 @@ using namespace eckit::option; using namespace eckit; - namespace fdb5 { namespace tools { //---------------------------------------------------------------------------------------------------------------------- -FDBLock::FDBLock(int argc, char **argv, bool unlock) : - FDBVisitTool(argc, argv, "class,expver,stream,date,time"), - unlock_(unlock), - list_(false), - retrieve_(false), - archive_(false), - wipe_(false) { +FDBLock::FDBLock(int argc, char** argv, bool unlock) + : FDBVisitTool(argc, argv, "class,expver,stream,date,time") + , unlock_(unlock) + , list_(false) + , retrieve_(false) + , archive_(false) + , wipe_(false) { - std::string prefix = std::string(unlock?"Unl":"L") + "ock matching databases for "; + std::string prefix = std::string(unlock ? "Unl" : "L") + "ock matching databases for "; options_.push_back(new SimpleOption("list", prefix + "listing")); options_.push_back(new SimpleOption("retrieve", prefix + "retrieval")); @@ -43,10 +42,8 @@ FDBLock::FDBLock(int argc, char **argv, bool unlock) : options_.push_back(new SimpleOption("wipe", prefix + "wipe")); } - FDBLock::~FDBLock() {} - void FDBLock::init(const CmdArgs& args) { FDBVisitTool::init(args); @@ -60,10 +57,8 @@ void FDBLock::init(const CmdArgs& args) { ss << "No identifier specified to (un)lock."; throw UserError(ss.str(), Here()); } - } - void FDBLock::execute(const CmdArgs& args) { FDB fdb(config(args)); @@ -71,10 +66,14 @@ void FDBLock::execute(const CmdArgs& args) { ControlAction action = unlock_ ? ControlAction::Enable : ControlAction::Disable; ControlIdentifiers identifiers; - if (list_) identifiers |= ControlIdentifier::List; - if (retrieve_) identifiers |= ControlIdentifier::Retrieve; - if (archive_) identifiers |= ControlIdentifier::Archive; - if (wipe_) identifiers |= ControlIdentifier::Wipe; + if (list_) + identifiers |= ControlIdentifier::List; + if (retrieve_) + identifiers |= ControlIdentifier::Retrieve; + if (archive_) + identifiers |= ControlIdentifier::Archive; + if (wipe_) + identifiers |= ControlIdentifier::Wipe; for (const FDBToolRequest& request : requests("read")) { @@ -86,11 +85,16 @@ void FDBLock::execute(const CmdArgs& args) { Log::info() << "Database: " << elem.key << std::endl << " location: " << elem.location.asString() << std::endl; - if (!elem.controlIdentifiers.enabled(ControlIdentifier::Retrieve)) Log::info() << " retrieve: LOCKED" << std::endl; - if (!elem.controlIdentifiers.enabled(ControlIdentifier::Archive)) Log::info() << " archive: LOCKED" << std::endl; - if (!elem.controlIdentifiers.enabled(ControlIdentifier::List)) Log::info() << " list: LOCKED" << std::endl; - if (!elem.controlIdentifiers.enabled(ControlIdentifier::Wipe)) Log::info() << " wipe: LOCKED" << std::endl; - if (!elem.controlIdentifiers.enabled(ControlIdentifier::UniqueRoot)) Log::info() << " multi-root: PERMITTED" << std::endl; + if (!elem.controlIdentifiers.enabled(ControlIdentifier::Retrieve)) + Log::info() << " retrieve: LOCKED" << std::endl; + if (!elem.controlIdentifiers.enabled(ControlIdentifier::Archive)) + Log::info() << " archive: LOCKED" << std::endl; + if (!elem.controlIdentifiers.enabled(ControlIdentifier::List)) + Log::info() << " list: LOCKED" << std::endl; + if (!elem.controlIdentifiers.enabled(ControlIdentifier::Wipe)) + Log::info() << " wipe: LOCKED" << std::endl; + if (!elem.controlIdentifiers.enabled(ControlIdentifier::UniqueRoot)) + Log::info() << " multi-root: PERMITTED" << std::endl; count++; } @@ -107,4 +111,3 @@ void FDBLock::execute(const CmdArgs& args) { } // namespace tools } // namespace fdb5 - diff --git a/src/fdb5/tools/FDBLock.h b/src/fdb5/tools/FDBLock.h index 6b26c5754..c5356d5e9 100644 --- a/src/fdb5/tools/FDBLock.h +++ b/src/fdb5/tools/FDBLock.h @@ -19,7 +19,6 @@ using namespace eckit::option; using namespace eckit; - namespace fdb5 { namespace tools { @@ -27,17 +26,14 @@ namespace tools { class FDBLock : public FDBVisitTool { public: // methods - - FDBLock(int argc, char **argv, bool unlock=false); + FDBLock(int argc, char** argv, bool unlock = false); ~FDBLock() override; private: // methods - void execute(const CmdArgs& args) override; - void init(const CmdArgs &args) override; + void init(const CmdArgs& args) override; private: // members - bool unlock_; bool list_; diff --git a/src/fdb5/tools/FDBTool.cc b/src/fdb5/tools/FDBTool.cc index 92a5af1a4..c99dcdea1 100644 --- a/src/fdb5/tools/FDBTool.cc +++ b/src/fdb5/tools/FDBTool.cc @@ -31,13 +31,11 @@ static void usage(const std::string& tool) { } void FDBTool::run() { - if(needsConfig_) { + if (needsConfig_) { options_.push_back(new eckit::option::SimpleOption("config", "FDB configuration filename")); } - eckit::option::CmdArgs args(&fdb5::usage, options_, numberOfPositionalArguments(), - minimumPositionalArguments()); - + eckit::option::CmdArgs args(&fdb5::usage, options_, numberOfPositionalArguments(), minimumPositionalArguments()); init(args); execute(args); @@ -78,10 +76,8 @@ void FDBTool::finish(const eckit::option::CmdArgs&) {} FDBToolException::FDBToolException(const std::string& w) : Exception(w) {} -FDBToolException::FDBToolException(const std::string& w, const eckit::CodeLocation& l) : - Exception(w, l) {} - +FDBToolException::FDBToolException(const std::string& w, const eckit::CodeLocation& l) : Exception(w, l) {} //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/tools/FDBTool.h b/src/fdb5/tools/FDBTool.h index 14050d0c0..2514ea530 100644 --- a/src/fdb5/tools/FDBTool.h +++ b/src/fdb5/tools/FDBTool.h @@ -18,19 +18,19 @@ #include -#include "eckit/runtime/Tool.h" #include "eckit/config/Configuration.h" #include "eckit/config/LocalConfiguration.h" #include "eckit/exception/Exceptions.h" +#include "eckit/runtime/Tool.h" #include "fdb5/config/Config.h" namespace eckit { - namespace option { - class Option; - class CmdArgs; - } -} +namespace option { +class Option; +class CmdArgs; +} // namespace option +} // namespace eckit namespace fdb5 { @@ -42,30 +42,26 @@ namespace fdb5 { class FDBTool : public eckit::Tool { protected: // methods - - FDBTool(int argc, char **argv); + FDBTool(int argc, char** argv); ~FDBTool() override {} void run() override; - Config config(const eckit::option::CmdArgs& args, const eckit::Configuration& userConfig = eckit::LocalConfiguration()) const; + Config config(const eckit::option::CmdArgs& args, + const eckit::Configuration& userConfig = eckit::LocalConfiguration()) const; public: // methods - - virtual void usage(const std::string &tool) const; + virtual void usage(const std::string& tool) const; protected: // members - - std::vector options_; + std::vector options_; /// Set this to false in tool subclass if your tool does not require access to 'config.yaml' bool needsConfig_{true}; protected: // methods - virtual void init(const eckit::option::CmdArgs& args); virtual void finish(const eckit::option::CmdArgs& args); private: // methods - virtual void execute(const eckit::option::CmdArgs& args) = 0; virtual int numberOfPositionalArguments() const { return -1; } @@ -74,14 +70,12 @@ class FDBTool : public eckit::Tool { //---------------------------------------------------------------------------------------------------------------------- - class FDBToolException : public eckit::Exception { public: FDBToolException(const std::string&); FDBToolException(const std::string&, const eckit::CodeLocation&); }; - //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 diff --git a/src/fdb5/tools/FDBVisitTool.cc b/src/fdb5/tools/FDBVisitTool.cc index 04c6cce9d..fee3c5544 100644 --- a/src/fdb5/tools/FDBVisitTool.cc +++ b/src/fdb5/tools/FDBVisitTool.cc @@ -8,18 +8,16 @@ * does it submit to any jurisdiction. */ - #include "eckit/config/Resource.h" +#include "eckit/option/CmdArgs.h" #include "eckit/option/SimpleOption.h" #include "eckit/option/VectorOption.h" -#include "eckit/option/CmdArgs.h" -#include "metkit/mars/MarsParser.h" #include "metkit/mars/MarsExpension.h" +#include "metkit/mars/MarsParser.h" #include "fdb5/tools/FDBVisitTool.h" - using namespace eckit; using namespace eckit::option; @@ -28,19 +26,17 @@ namespace tools { //---------------------------------------------------------------------------------------------------------------------- - -FDBVisitTool::FDBVisitTool(int argc, char **argv, std::string minimumKeys) : - FDBTool(argc, argv), - fail_(true), - all_(false), - raw_(false) { +FDBVisitTool::FDBVisitTool(int argc, char** argv, std::string minimumKeys) + : FDBTool(argc, argv) + , fail_(true) + , all_(false) + , raw_(false) { minimumKeys_ = Resource>("FDBInspectMinimumKeys", minimumKeys, true); - if(minimumKeys_.size() != 0) { - options_.push_back(new VectorOption("minimum-keys", - "Use these keywords as a minimum set which *must* be specified", - 0, ",")); + if (minimumKeys_.size() != 0) { + options_.push_back(new VectorOption( + "minimum-keys", "Use these keywords as a minimum set which *must* be specified", 0, ",")); } // Don't apply MarsExpension to the parsed requests. This relies on the user @@ -48,9 +44,8 @@ FDBVisitTool::FDBVisitTool(int argc, char **argv, std::string minimumKeys) : options_.push_back(new SimpleOption("raw", "Don't apply (contextual) expansion and checking on requests.")); // Be able to turn ignore-errors off - options_.push_back( - new SimpleOption("ignore-errors", - "Ignore errors (report them as warnings) and continue processing wherever possible")); + options_.push_back(new SimpleOption( + "ignore-errors", "Ignore errors (report them as warnings) and continue processing wherever possible")); } FDBVisitTool::~FDBVisitTool() {} @@ -117,17 +112,16 @@ std::vector FDBVisitTool::requests(const std::string& verb) cons return requests; } -void FDBVisitTool::usage(const std::string &tool) const { +void FDBVisitTool::usage(const std::string& tool) const { // derived classes should provide this type of usage information ... - Log::info() << "Usage: " << tool << " [options] [request1] [request2] ..." << std::endl - << std::endl; + Log::info() << "Usage: " << tool << " [options] [request1] [request2] ..." << std::endl << std::endl; Log::info() << "Examples:" << std::endl - << "=========" << std::endl << std::endl - << tool << " class=rd,expver=xywz,stream=oper,date=20190603,time=00" + << "=========" << std::endl << std::endl + << tool << " class=rd,expver=xywz,stream=oper,date=20190603,time=00" << std::endl << std::endl; FDBTool::usage(tool); diff --git a/src/fdb5/tools/FDBVisitTool.h b/src/fdb5/tools/FDBVisitTool.h index 5def65793..588c663ea 100644 --- a/src/fdb5/tools/FDBVisitTool.h +++ b/src/fdb5/tools/FDBVisitTool.h @@ -14,8 +14,8 @@ #ifndef fdb5_tools_FDBVisitTool_H #define fdb5_tools_FDBVisitTool_H -#include "fdb5/tools/FDBTool.h" #include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/tools/FDBTool.h" namespace fdb5 { namespace tools { @@ -25,22 +25,20 @@ namespace tools { class FDBVisitTool : public FDBTool { protected: // methods - - FDBVisitTool(int argc, char **argv, std::string minimunKeys = std::string()); + FDBVisitTool(int argc, char** argv, std::string minimunKeys = std::string()); ~FDBVisitTool() override; void usage(const std::string& tool) const override; - void init(const eckit::option::CmdArgs &args) override; + void init(const eckit::option::CmdArgs& args) override; void run() override; bool fail() const; - std::vector requests(const std::string& verb="retrieve") const; + std::vector requests(const std::string& verb = "retrieve") const; private: // members - // minimum set of keys needed to execute a query // these are used for safety to restrict the search space for the visitors std::vector minimumKeys_; diff --git a/src/fdb5/tools/fdb-axes.cc b/src/fdb5/tools/fdb-axes.cc index ed71f215e..ae471afdb 100644 --- a/src/fdb5/tools/fdb-axes.cc +++ b/src/fdb5/tools/fdb-axes.cc @@ -8,9 +8,9 @@ * does it submit to any jurisdiction. */ +#include "eckit/log/JSON.h" #include "eckit/option/CmdArgs.h" #include "eckit/option/SimpleOption.h" -#include "eckit/log/JSON.h" #include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" @@ -19,7 +19,6 @@ using namespace eckit; using namespace option; - namespace fdb5::tools { //---------------------------------------------------------------------------------------------------------------------- @@ -27,22 +26,17 @@ namespace fdb5::tools { class FDBAxisTest : public FDBVisitTool { public: // methods - - FDBAxisTest(int argc, char **argv) : - FDBVisitTool(argc, argv, "class,expver"), - level_(3), - json_(false) { - options_.push_back(new SimpleOption("level", "Specify how many levels of the keys should be should be explored")); + FDBAxisTest(int argc, char** argv) : FDBVisitTool(argc, argv, "class,expver"), level_(3), json_(false) { + options_.push_back( + new SimpleOption("level", "Specify how many levels of the keys should be should be explored")); options_.push_back(new SimpleOption("json", "Output available fields in JSON form")); } private: // methods - void execute(const CmdArgs& args) final; - void init(const CmdArgs &args) final; + void init(const CmdArgs& args) final; private: // members - int level_; bool json_; }; @@ -58,9 +52,9 @@ void FDBAxisTest::execute(const CmdArgs& args) { FDB fdb(config(args)); for (const FDBToolRequest& request : requests()) { -// Timer taxes("axes"); + // Timer taxes("axes"); auto r = fdb.axes(request, level_); -// taxes.stop(); + // taxes.stop(); if (json_) { JSON json(Log::info()); @@ -76,8 +70,7 @@ void FDBAxisTest::execute(const CmdArgs& args) { } // namespace fdb5::tools -int main(int argc, char **argv) { +int main(int argc, char** argv) { fdb5::tools::FDBAxisTest app(argc, argv); return app.start(); } - diff --git a/src/fdb5/tools/fdb-copy.cc b/src/fdb5/tools/fdb-copy.cc index f4e927dac..3cf6f4df3 100644 --- a/src/fdb5/tools/fdb-copy.cc +++ b/src/fdb5/tools/fdb-copy.cc @@ -11,27 +11,27 @@ #include #include +#include "eckit/filesystem/PathName.h" #include "eckit/option/CmdArgs.h" #include "eckit/option/SimpleOption.h" -#include "eckit/filesystem/PathName.h" -#include "metkit/mars/MarsRequest.h" -#include "metkit/mars/MarsParser.h" #include "metkit/mars/MarsExpension.h" +#include "metkit/mars/MarsParser.h" +#include "metkit/mars/MarsRequest.h" #include "fdb5/api/FDB.h" -#include "fdb5/message/MessageArchiver.h" #include "fdb5/io/HandleGatherer.h" +#include "fdb5/message/MessageArchiver.h" #include "fdb5/tools/FDBTool.h" using namespace eckit::option; class FDBCopy : public fdb5::FDBTool { - virtual void execute(const CmdArgs &args); - virtual void usage(const std::string &tool) const; + virtual void execute(const CmdArgs& args); + virtual void usage(const std::string& tool) const; - public: - FDBCopy(int argc, char **argv): fdb5::FDBTool(argc, argv) { +public: + FDBCopy(int argc, char** argv) : fdb5::FDBTool(argc, argv) { options_.push_back(new SimpleOption("verbose", "Print verbose output")); options_.push_back(new SimpleOption("raw", "Process the MARS request without expansion")); options_.push_back(new SimpleOption("sort", "Sort fields according to location on input storage")); @@ -40,7 +40,7 @@ class FDBCopy : public fdb5::FDBTool { } }; -void FDBCopy::usage(const std::string &tool) const { +void FDBCopy::usage(const std::string& tool) const { eckit::Log::info() << std::endl << "Usage: " << tool << " --from --to " << std::endl; fdb5::FDBTool::usage(tool); } @@ -64,8 +64,7 @@ static std::vector readRequest(const CmdArgs& args) { if (args.getBool("raw", false)) { for (auto r : parsedRequests) requests.push_back(r); - } - else { + } else { const bool inherit = false; metkit::mars::MarsExpension expand(inherit); auto expanded = expand.expand(parsedRequests); @@ -77,11 +76,11 @@ static std::vector readRequest(const CmdArgs& args) { void FDBCopy::execute(const CmdArgs& args) { - bool verbose = args.getBool("verbose", false); + bool verbose = args.getBool("verbose", false); std::string from; args.get("from", from); - if(from.empty()) { + if (from.empty()) { throw eckit::UserError("Missing --from parameter"); } @@ -91,7 +90,7 @@ void FDBCopy::execute(const CmdArgs& args) { throw eckit::UserError("Missing --to parameter"); } - fdb5::Config readConfig = fdb5::Config::make(eckit::PathName(from)); + fdb5::Config readConfig = fdb5::Config::make(eckit::PathName(from)); fdb5::Config writeConfig = fdb5::Config::make(eckit::PathName(to)); std::vector requests = readRequest(args); @@ -101,7 +100,7 @@ void FDBCopy::execute(const CmdArgs& args) { fdb5::HandleGatherer handles(sort); fdb5::FDB fdbRead(readConfig); - + for (const auto& request : requests) { eckit::Log::info() << request << std::endl; handles.add(fdbRead.retrieve(request)); @@ -112,9 +111,8 @@ void FDBCopy::execute(const CmdArgs& args) { fdb5::MessageArchiver fdbWriter(fdb5::Key(), false, verbose, writeConfig); fdbWriter.archive(*dh); } - -int main(int argc, char **argv) { + +int main(int argc, char** argv) { FDBCopy app(argc, argv); return app.start(); } - diff --git a/src/fdb5/tools/fdb-dump-index.cc b/src/fdb5/tools/fdb-dump-index.cc index 332ea1392..a1be735a2 100644 --- a/src/fdb5/tools/fdb-dump-index.cc +++ b/src/fdb5/tools/fdb-dump-index.cc @@ -21,24 +21,19 @@ using namespace eckit; class FDBDumpIndex : public fdb5::FDBTool { - public: // methods +public: // methods + FDBDumpIndex(int argc, char** argv) : fdb5::FDBTool(argc, argv) {} - FDBDumpIndex(int argc, char **argv) : - fdb5::FDBTool(argc, argv) {} - - private: // methods - - virtual void usage(const std::string &tool) const; +private: // methods + virtual void usage(const std::string& tool) const; virtual void execute(const option::CmdArgs& args); }; -void FDBDumpIndex::usage(const std::string &tool) const { - Log::info() << std::endl - << "Usage: " << tool << " [path1] [path2] ..." << std::endl; +void FDBDumpIndex::usage(const std::string& tool) const { + Log::info() << std::endl << "Usage: " << tool << " [path1] [path2] ..." << std::endl; fdb5::FDBTool::usage(tool); } - void FDBDumpIndex::execute(const option::CmdArgs& args) { // n.b. We don't just open the toc, then check if in the list of indexes, as there @@ -57,8 +52,7 @@ void FDBDumpIndex::execute(const option::CmdArgs& args) { //---------------------------------------------------------------------------------------------------------------------- -int main(int argc, char **argv) { +int main(int argc, char** argv) { FDBDumpIndex app(argc, argv); return app.start(); } - diff --git a/src/fdb5/tools/fdb-dump-toc.cc b/src/fdb5/tools/fdb-dump-toc.cc index b2ab727bf..23b1457c8 100644 --- a/src/fdb5/tools/fdb-dump-toc.cc +++ b/src/fdb5/tools/fdb-dump-toc.cc @@ -20,27 +20,24 @@ using namespace eckit; class FDBDumpToc : public fdb5::FDBTool { - public: // methods - - FDBDumpToc(int argc, char **argv) : fdb5::FDBTool(argc, argv) { +public: // methods + FDBDumpToc(int argc, char** argv) : fdb5::FDBTool(argc, argv) { // FDBDumpToc does not require to read the configuration needsConfig_ = false; - options_.push_back(new eckit::option::SimpleOption("walk", "Walk subtocs rather than show simple entries")); + options_.push_back( + new eckit::option::SimpleOption("walk", "Walk subtocs rather than show simple entries")); } - private: // methods - - virtual void usage(const std::string &tool) const; +private: // methods + virtual void usage(const std::string& tool) const; virtual void execute(const eckit::option::CmdArgs& args); }; -void FDBDumpToc::usage(const std::string &tool) const { - Log::info() << std::endl - << "Usage: " << tool << " [path1] [path2] ..." << std::endl; +void FDBDumpToc::usage(const std::string& tool) const { + Log::info() << std::endl << "Usage: " << tool << " [path1] [path2] ..." << std::endl; fdb5::FDBTool::usage(tool); } - void FDBDumpToc::execute(const eckit::option::CmdArgs& args) { bool walkSubTocs = args.getBool("walk", false); @@ -57,8 +54,7 @@ void FDBDumpToc::execute(const eckit::option::CmdArgs& args) { //---------------------------------------------------------------------------------------------------------------------- -int main(int argc, char **argv) { +int main(int argc, char** argv) { FDBDumpToc app(argc, argv); return app.start(); } - diff --git a/src/fdb5/tools/fdb-dump.cc b/src/fdb5/tools/fdb-dump.cc index 9b5e30f5e..22d850b2a 100644 --- a/src/fdb5/tools/fdb-dump.cc +++ b/src/fdb5/tools/fdb-dump.cc @@ -26,21 +26,16 @@ namespace tools { class FDBDump : public FDBVisitTool { public: // methods - - FDBDump(int argc, char **argv) : - FDBVisitTool(argc, argv), - simple_(false) { + FDBDump(int argc, char** argv) : FDBVisitTool(argc, argv), simple_(false) { options_.push_back(new SimpleOption("simple", "Dump one (simpler) record per line")); } private: // methods - void execute(const CmdArgs& args) override; - void init(const CmdArgs &args) override; + void init(const CmdArgs& args) override; private: // members - bool simple_; }; @@ -51,7 +46,6 @@ void FDBDump::init(const CmdArgs& args) { simple_ = args.getBool("simple", false); } - void FDBDump::execute(const CmdArgs& args) { FDB fdb(config(args)); @@ -80,8 +74,7 @@ void FDBDump::execute(const CmdArgs& args) { } // namespace tools } // namespace fdb5 -int main(int argc, char **argv) { +int main(int argc, char** argv) { fdb5::tools::FDBDump app(argc, argv); return app.start(); } - diff --git a/src/fdb5/tools/fdb-hammer.cc b/src/fdb5/tools/fdb-hammer.cc index 8257d61bc..dfbf467a0 100644 --- a/src/fdb5/tools/fdb-hammer.cc +++ b/src/fdb5/tools/fdb-hammer.cc @@ -8,56 +8,52 @@ * does it submit to any jurisdiction. */ -#include +#include #include -#include #include #include +#include #include -#include +#include #include "eccodes.h" -#include "eckit/config/Resource.h" #include "eckit/config/LocalConfiguration.h" +#include "eckit/config/Resource.h" #include "eckit/io/DataHandle.h" -#include "eckit/io/StdFile.h" -#include "eckit/io/MemoryHandle.h" #include "eckit/io/EmptyHandle.h" +#include "eckit/io/MemoryHandle.h" +#include "eckit/io/StdFile.h" #include "eckit/option/CmdArgs.h" #include "eckit/option/SimpleOption.h" -#include "fdb5/message/MessageArchiver.h" +#include "fdb5/api/helpers/FDBToolRequest.h" #include "fdb5/io/HandleGatherer.h" +#include "fdb5/message/MessageArchiver.h" #include "fdb5/tools/FDBTool.h" -#include "fdb5/api/helpers/FDBToolRequest.h" // This list is currently sufficient to get to nparams=200 of levtype=ml,type=fc -const std::unordered_set AWKWARD_PARAMS {11, 12, 13, 14, 15, 16, 49, 51, 52, 61, 121, 122, 146, 147, 169, 175, 176, 177, 179, 189, 201, 202}; - +const std::unordered_set AWKWARD_PARAMS{11, 12, 13, 14, 15, 16, 49, 51, 52, 61, 121, + 122, 146, 147, 169, 175, 176, 177, 179, 189, 201, 202}; using namespace eckit; - class FDBHammer : public fdb5::FDBTool { - void usage(const std::string &tool) const override; + void usage(const std::string& tool) const override; - void init(const eckit::option::CmdArgs &args) override; + void init(const eckit::option::CmdArgs& args) override; int minimumPositionalArguments() const override { return 1; } - void execute(const eckit::option::CmdArgs &args) override; + void execute(const eckit::option::CmdArgs& args) override; void executeRead(const eckit::option::CmdArgs& args); void executeWrite(const eckit::option::CmdArgs& args); void executeList(const eckit::option::CmdArgs& args); public: - - FDBHammer(int argc, char **argv) : - fdb5::FDBTool(argc, argv), - verbose_(false) { + FDBHammer(int argc, char** argv) : fdb5::FDBTool(argc, argv), verbose_(false) { options_.push_back(new eckit::option::SimpleOption("expver", "Reset expver on data")); options_.push_back(new eckit::option::SimpleOption("class", "Reset class on data")); @@ -80,13 +76,16 @@ class FDBHammer : public fdb5::FDBTool { bool verbose_; }; -void FDBHammer::usage(const std::string &tool) const { - eckit::Log::info() << std::endl << "Usage: " << tool << " [--statistics] [--read] [--list] --nsteps= --nensembles= --nlevels= --nparams= --expver= " << std::endl; +void FDBHammer::usage(const std::string& tool) const { + eckit::Log::info() << std::endl + << "Usage: " << tool + << " [--statistics] [--read] [--list] --nsteps= --nensembles= " + "--nlevels= --nparams= --expver= " + << std::endl; fdb5::FDBTool::usage(tool); } -void FDBHammer::init(const eckit::option::CmdArgs& args) -{ +void FDBHammer::init(const eckit::option::CmdArgs& args) { FDBTool::init(args); ASSERT(args.has("expver")); @@ -98,7 +97,7 @@ void FDBHammer::init(const eckit::option::CmdArgs& args) verbose_ = args.getBool("verbose", false); } -void FDBHammer::execute(const eckit::option::CmdArgs &args) { +void FDBHammer::execute(const eckit::option::CmdArgs& args) { if (args.getBool("read", false)) { executeRead(args); @@ -109,7 +108,7 @@ void FDBHammer::execute(const eckit::option::CmdArgs &args) { } } -void FDBHammer::executeWrite(const eckit::option::CmdArgs &args) { +void FDBHammer::executeWrite(const eckit::option::CmdArgs& args) { eckit::AutoStdFile fin(args(0)); @@ -126,12 +125,12 @@ void FDBHammer::executeWrite(const eckit::option::CmdArgs &args) { bool delay = args.getBool("delay", false); - const char* buffer = nullptr; size_t size = 0; eckit::LocalConfiguration userConfig{}; - if (!args.has("disable-subtocs")) userConfig.set("useSubToc", true); + if (!args.has("disable-subtocs")) + userConfig.set("useSubToc", true); if (delay) { std::random_device rd; @@ -162,21 +161,19 @@ void FDBHammer::executeWrite(const eckit::option::CmdArgs &args) { for (size_t member = 1; member <= nensembles; ++member) { if (args.has("nensembles")) { - CODES_CHECK(codes_set_long(handle, "number", member+number-1), 0); + CODES_CHECK(codes_set_long(handle, "number", member + number - 1), 0); } for (size_t step = 0; step < nsteps; ++step) { CODES_CHECK(codes_set_long(handle, "step", step), 0); for (size_t lev = 1; lev <= nlevels; ++lev) { - CODES_CHECK(codes_set_long(handle, "level", lev+level-1), 0); + CODES_CHECK(codes_set_long(handle, "level", lev + level - 1), 0); for (size_t param = 1, real_param = 1; param <= nparams; ++param, ++real_param) { // GRIB API only allows us to use certain parameters while (AWKWARD_PARAMS.find(real_param) != AWKWARD_PARAMS.end()) { real_param++; } - Log::info() << "Member: " << member - << ", step: " << step - << ", level: " << level + Log::info() << "Member: " << member << ", step: " << step << ", level: " << level << ", param: " << real_param << std::endl; CODES_CHECK(codes_set_long(handle, "paramId", real_param), 0); @@ -188,7 +185,8 @@ void FDBHammer::executeWrite(const eckit::option::CmdArgs &args) { MemoryHandle dh(buffer, size); - if (member == 1 && step == 0 && lev == 1 && param == 1) gettimeofday(&tval_before_io, NULL); + if (member == 1 && step == 0 && lev == 1 && param == 1) + gettimeofday(&tval_before_io, NULL); archiver.archive(dh); writeCount++; bytesWritten += size; @@ -200,7 +198,8 @@ void FDBHammer::executeWrite(const eckit::option::CmdArgs &args) { gribTimer.stop(); elapsed_grib += gribTimer.elapsed(); archiver.flush(); - if (member == nensembles && step == (nsteps - 1)) gettimeofday(&tval_after_io, NULL); + if (member == nensembles && step == (nsteps - 1)) + gettimeofday(&tval_after_io, NULL); gribTimer.start(); } } @@ -220,20 +219,13 @@ void FDBHammer::executeWrite(const eckit::option::CmdArgs &args) { Log::info() << "Total rate: " << double(bytesWritten) / timer.elapsed() << " bytes / s" << std::endl; Log::info() << "Total rate: " << double(bytesWritten) / (timer.elapsed() * 1024 * 1024) << " MB / s" << std::endl; - Log::info() << "Timestamp before first IO: " << - (long int)tval_before_io.tv_sec << "." << - std::setw(6) << std::setfill('0') << - (long int)tval_before_io.tv_usec << std::endl; - Log::info() << "Timestamp after last IO: " << - (long int)tval_after_io.tv_sec << "." << - std::setw(6) << std::setfill('0') << - (long int)tval_after_io.tv_usec << std::endl; - + Log::info() << "Timestamp before first IO: " << (long int)tval_before_io.tv_sec << "." << std::setw(6) + << std::setfill('0') << (long int)tval_before_io.tv_usec << std::endl; + Log::info() << "Timestamp after last IO: " << (long int)tval_after_io.tv_sec << "." << std::setw(6) + << std::setfill('0') << (long int)tval_after_io.tv_usec << std::endl; } - -void FDBHammer::executeRead(const eckit::option::CmdArgs &args) { - +void FDBHammer::executeRead(const eckit::option::CmdArgs& args) { fdb5::MessageDecoder decoder; std::vector requests = decoder.messageToRequests(args(0)); @@ -253,7 +245,8 @@ void FDBHammer::executeRead(const eckit::option::CmdArgs &args) { request.setValue("optimised", "on"); eckit::LocalConfiguration userConfig{}; - if (!args.has("disable-subtocs")) userConfig.set("useSubToc", true); + if (!args.has("disable-subtocs")) + userConfig.set("useSubToc", true); struct timeval tval_before_io, tval_after_io; eckit::Timer timer; @@ -265,12 +258,12 @@ void FDBHammer::executeRead(const eckit::option::CmdArgs &args) { for (size_t member = 1; member <= nensembles; ++member) { if (args.has("nensembles")) { - request.setValue("number", member+number-1); + request.setValue("number", member + number - 1); } for (size_t step = 0; step < nsteps; ++step) { request.setValue("step", step); for (size_t lev = 1; lev <= nlevels; ++lev) { - request.setValue("levelist", lev+level-1); + request.setValue("levelist", lev + level - 1); for (size_t param = 1, real_param = 1; param <= nparams; ++param, ++real_param) { // GRIB API only allows us to use certain parameters while (AWKWARD_PARAMS.find(real_param) != AWKWARD_PARAMS.end()) { @@ -278,12 +271,11 @@ void FDBHammer::executeRead(const eckit::option::CmdArgs &args) { } request.setValue("param", real_param); - Log::info() << "Member: " << member - << ", step: " << step - << ", level: " << level + Log::info() << "Member: " << member << ", step: " << step << ", level: " << level << ", param: " << real_param << std::endl; - if (member == 1 && step == 0 && lev == 1 && param == 1) gettimeofday(&tval_before_io, NULL); + if (member == 1 && step == 0 && lev == 1 && param == 1) + gettimeofday(&tval_before_io, NULL); handles.add(fdb.retrieve(request)); fieldsRead++; } @@ -305,22 +297,16 @@ void FDBHammer::executeRead(const eckit::option::CmdArgs &args) { Log::info() << "Total rate: " << double(total) / timer.elapsed() << " bytes / s" << std::endl; Log::info() << "Total rate: " << double(total) / (timer.elapsed() * 1024 * 1024) << " MB / s" << std::endl; - Log::info() << "Timestamp before first IO: " << - (long int)tval_before_io.tv_sec << "." << - std::setw(6) << std::setfill('0') << - (long int)tval_before_io.tv_usec << std::endl; - Log::info() << "Timestamp after last IO: " << - (long int)tval_after_io.tv_sec << "." << - std::setw(6) << std::setfill('0') << - (long int)tval_after_io.tv_usec << std::endl; - + Log::info() << "Timestamp before first IO: " << (long int)tval_before_io.tv_sec << "." << std::setw(6) + << std::setfill('0') << (long int)tval_before_io.tv_usec << std::endl; + Log::info() << "Timestamp after last IO: " << (long int)tval_after_io.tv_sec << "." << std::setw(6) + << std::setfill('0') << (long int)tval_after_io.tv_usec << std::endl; } +void FDBHammer::executeList(const eckit::option::CmdArgs& args) { -void FDBHammer::executeList(const eckit::option::CmdArgs &args) { - - - std::vector minimumKeys = eckit::Resource>("FDBInspectMinimumKeys", "class,expver", true); + std::vector minimumKeys = + eckit::Resource>("FDBInspectMinimumKeys", "class,expver", true); fdb5::MessageDecoder decoder; std::vector requests = decoder.messageToRequests(args(0)); @@ -339,7 +325,8 @@ void FDBHammer::executeList(const eckit::option::CmdArgs &args) { request.setValue("class", args.getString("class")); eckit::LocalConfiguration userConfig{}; - if (!args.has("disable-subtocs")) userConfig.set("useSubToc", true); + if (!args.has("disable-subtocs")) + userConfig.set("useSubToc", true); eckit::Timer timer; timer.start(); @@ -378,20 +365,17 @@ void FDBHammer::executeList(const eckit::option::CmdArgs &args) { while (listObject.next(info)) { count++; } - } timer.stop(); Log::info() << "fdb-hammer - Fields listed: " << count << std::endl; Log::info() << "fdb-hammer - List duration: " << timer.elapsed() << std::endl; - } //---------------------------------------------------------------------------------------------------------------------- -int main(int argc, char **argv) { +int main(int argc, char** argv) { FDBHammer app(argc, argv); return app.start(); } - diff --git a/src/fdb5/tools/fdb-hide.cc b/src/fdb5/tools/fdb-hide.cc index f9849ef54..61f3d93d9 100644 --- a/src/fdb5/tools/fdb-hide.cc +++ b/src/fdb5/tools/fdb-hide.cc @@ -11,10 +11,10 @@ #include "eckit/option/CmdArgs.h" #include "eckit/option/SimpleOption.h" +#include "fdb5/LibFdb5.h" #include "fdb5/api/helpers/FDBToolRequest.h" #include "fdb5/config/Config.h" #include "fdb5/database/Key.h" -#include "fdb5/LibFdb5.h" #include "fdb5/rules/Schema.h" #include "fdb5/toc/TocCatalogueWriter.h" #include "fdb5/toc/TocEngine.h" @@ -31,30 +31,22 @@ namespace tools { class FdbHide : public FDBTool { public: // methods - - FdbHide(int argc, char **argv) : - FDBTool(argc, argv), - doit_(false) { + FdbHide(int argc, char** argv) : FDBTool(argc, argv), doit_(false) { options_.push_back(new SimpleOption("doit", "Do the actual change")); } private: // methods - virtual void init(const option::CmdArgs& args); virtual void execute(const option::CmdArgs& args); - virtual void usage(const std::string &tool) const; + virtual void usage(const std::string& tool) const; private: // members - bool doit_; }; -void FdbHide::usage(const std::string &tool) const { +void FdbHide::usage(const std::string& tool) const { - Log::info() << std::endl - << "Usage: " << tool << " [options] [DB request]" << std::endl - << std::endl - << std::endl; + Log::info() << std::endl << "Usage: " << tool << " [options] [DB request]" << std::endl << std::endl << std::endl; FDBTool::usage(tool); } @@ -112,9 +104,9 @@ void FdbHide::execute(const option::CmdArgs& args) { //---------------------------------------------------------------------------------------------------------------------- } // namespace tools -} // namespace fbb5 +} // namespace fdb5 -int main(int argc, char **argv) { +int main(int argc, char** argv) { fdb5::tools::FdbHide app(argc, argv); return app.start(); } diff --git a/src/fdb5/tools/fdb-info.cc b/src/fdb5/tools/fdb-info.cc index e511bada4..3c589c8a4 100644 --- a/src/fdb5/tools/fdb-info.cc +++ b/src/fdb5/tools/fdb-info.cc @@ -12,10 +12,10 @@ #include "eckit/option/SimpleOption.h" #include "fdb5/LibFdb5.h" -#include "fdb5/tools/FDBTool.h" #include "fdb5/config/Config.h" -#include "fdb5/io/LustreSettings.h" #include "fdb5/fdb5_version.h" +#include "fdb5/io/LustreSettings.h" +#include "fdb5/tools/FDBTool.h" using eckit::Log; @@ -26,29 +26,30 @@ namespace tools { class FDBInfo : public FDBTool { - public: // methods - - FDBInfo(int argc, char **argv) : - FDBTool(argc, argv), - all_(false), - version_(false), - home_(false), - schema_(false), - config_(false) - { +public: // methods + FDBInfo(int argc, char** argv) + : FDBTool(argc, argv) + , all_(false) + , version_(false) + , home_(false) + , schema_(false) + , config_(false) { options_.push_back(new eckit::option::SimpleOption("all", "Print all information")); options_.push_back(new eckit::option::SimpleOption("version", "Print the version of the FDB being used")); - options_.push_back(new eckit::option::SimpleOption("home", "Print the location of the FDB configuration files")); - options_.push_back(new eckit::option::SimpleOption("schema", "Print the location of the FDB schema file")); - options_.push_back(new eckit::option::SimpleOption("config-file", "Print the location of the FDB configuration file if being used")); - options_.push_back(new eckit::option::SimpleOption("lustre-api", "Indicate if the Lustre API is supported or disabled in this build")); + options_.push_back( + new eckit::option::SimpleOption("home", "Print the location of the FDB configuration files")); + options_.push_back( + new eckit::option::SimpleOption("schema", "Print the location of the FDB schema file")); + options_.push_back(new eckit::option::SimpleOption( + "config-file", "Print the location of the FDB configuration file if being used")); + options_.push_back(new eckit::option::SimpleOption( + "lustre-api", "Indicate if the Lustre API is supported or disabled in this build")); } - private: // methods - - virtual void usage(const std::string &tool) const; +private: // methods + virtual void usage(const std::string& tool) const; virtual void execute(const eckit::option::CmdArgs& args); - virtual void init(const eckit::option::CmdArgs &args); + virtual void init(const eckit::option::CmdArgs& args); bool all_; bool version_; @@ -58,13 +59,14 @@ class FDBInfo : public FDBTool { bool lustreApi_; }; -void FDBInfo::usage(const std::string &tool) const { +void FDBInfo::usage(const std::string& tool) const { Log::info() << std::endl << "Usage: " << tool << " [options]" << std::endl << std::endl << std::endl << "Examples:" << std::endl - << "=========" << std::endl << std::endl + << "=========" << std::endl + << std::endl << tool << " --all" << std::endl << tool << " --version" << std::endl << tool << " --home" << std::endl @@ -75,7 +77,7 @@ void FDBInfo::usage(const std::string &tool) const { FDBTool::usage(tool); } -void FDBInfo::init(const eckit::option::CmdArgs &args) { +void FDBInfo::init(const eckit::option::CmdArgs& args) { all_ = args.getBool("all", false); version_ = args.getBool("version", false); home_ = args.getBool("home", false); @@ -86,30 +88,34 @@ void FDBInfo::init(const eckit::option::CmdArgs &args) { void FDBInfo::execute(const eckit::option::CmdArgs& args) { - if(all_ || version_) { + if (all_ || version_) { Log::info() << (all_ ? "Version: " : "") << fdb5_version_str() << std::endl; - if(!all_) return; + if (!all_) + return; } - if(all_ || home_) { + if (all_ || home_) { // print FDB_HOME and exit -- note that is used in the bin/fdb wrapper script Log::info() << (all_ ? "Home: " : "") << eckit::PathName("~fdb/") << std::endl; - if(!all_) return; + if (!all_) + return; } Config conf = config(args); - if(all_ || schema_) { + if (all_ || schema_) { Log::info() << (all_ ? "Schema: " : "") << conf.schemaPath() << std::endl; - if(!all_) return; + if (!all_) + return; } - if(all_ || config_) { + if (all_ || config_) { Log::info() << (all_ ? "Config: " : "") << conf.configPath() << std::endl; - if(!all_) return; + if (!all_) + return; } - if(all_ || lustreApi_) { + if (all_ || lustreApi_) { Log::info() << (all_ ? "Lustre-API: " : "") << (fdb5LustreapiSupported() ? "enabled" : "disabled") << std::endl; } } @@ -119,8 +125,7 @@ void FDBInfo::execute(const eckit::option::CmdArgs& args) { } // namespace tools } // namespace fdb5 -int main(int argc, char **argv) { +int main(int argc, char** argv) { fdb5::tools::FDBInfo app(argc, argv); return app.start(); } - diff --git a/src/fdb5/tools/fdb-list.cc b/src/fdb5/tools/fdb-list.cc index 907faa756..768792d26 100644 --- a/src/fdb5/tools/fdb-list.cc +++ b/src/fdb5/tools/fdb-list.cc @@ -10,10 +10,9 @@ #include +#include "eckit/log/JSON.h" #include "eckit/option/CmdArgs.h" #include "eckit/option/SimpleOption.h" -#include "eckit/option/CmdArgs.h" -#include "eckit/log/JSON.h" #include "metkit/hypercube/HyperCube.h" @@ -33,30 +32,29 @@ namespace tools { class FDBList : public FDBVisitTool { - public: // methods - - FDBList(int argc, char **argv) : - FDBVisitTool(argc, argv, "class,expver"), - location_(false), - timestamp_(false), - length_(false), - full_(false), - porcelain_(false), - json_(false) { +public: // methods + FDBList(int argc, char** argv) + : FDBVisitTool(argc, argv, "class,expver") + , location_(false) + , timestamp_(false) + , length_(false) + , full_(false) + , porcelain_(false) + , json_(false) { options_.push_back(new SimpleOption("location", "Also print the location of each field")); options_.push_back(new SimpleOption("timestamp", "Also print the timestamp when the field was indexed")); options_.push_back(new SimpleOption("length", "Also print the field size")); options_.push_back(new SimpleOption("full", "Include all entries (including masked duplicates)")); - options_.push_back(new SimpleOption("porcelain", "Streamlined and stable output for input into other tools")); + options_.push_back( + new SimpleOption("porcelain", "Streamlined and stable output for input into other tools")); options_.push_back(new SimpleOption("json", "Output available fields in JSON form")); options_.push_back(new SimpleOption("compact", "Aggregate available fields in MARS requests")); } - private: // methods - +private: // methods virtual void execute(const CmdArgs& args); - virtual void init(const CmdArgs &args); + virtual void init(const CmdArgs& args); bool location_; bool timestamp_; @@ -67,18 +65,16 @@ class FDBList : public FDBVisitTool { bool compact_; }; - std::string keySignature(const fdb5::Key& key) { std::string signature; std::string separator; for (auto k : key.keys()) { - signature += separator+k; - separator=":"; + signature += separator + k; + separator = ":"; } return signature; } - void FDBList::init(const CmdArgs& args) { FDBVisitTool::init(args); @@ -133,7 +129,8 @@ void FDBList::execute(const CmdArgs& args) { // If --full is supplied, then include all entries including duplicates. auto listObject = fdb.list(request, !full_ && !compact_); - std::map>>> requests; + std::map>>> + requests; ListElement elem; while (listObject.next(elem)) { @@ -146,7 +143,7 @@ void FDBList::execute(const CmdArgs& args) { treeAxes += ","; treeAxes += keys[1]; - std::string signature=keySignature(keys[2]); // i.e. step:levelist:param + std::string signature = keySignature(keys[2]); // i.e. step:levelist:param auto it = requests.find(treeAxes); if (it == requests.end()) { @@ -159,7 +156,8 @@ void FDBList::execute(const CmdArgs& args) { h->second.first.merge(keys[2].request()); h->second.second.insert(keys[2]); } else { - it->second.emplace(signature, std::make_pair(keys[2].request(), std::unordered_set{keys[2]})); + it->second.emplace(signature, + std::make_pair(keys[2].request(), std::unordered_set{keys[2]})); } } } else { @@ -176,18 +174,18 @@ void FDBList::execute(const CmdArgs& args) { } } if (compact_) { - for (const auto& tree: requests) { - for (const auto& leaf: tree.second) { + for (const auto& tree : requests) { + for (const auto& leaf : tree.second) { metkit::hypercube::HyperCube h{leaf.second.first}; if (h.size() == leaf.second.second.size()) { Log::info() << "retrieve," << tree.first << ","; leaf.second.first.dump(Log::info(), "", "", false); Log::info() << std::endl; } else { - for (const auto& k: leaf.second.second) { + for (const auto& k : leaf.second.second) { h.clear(k.request()); } - for (const auto& r: h.requests()) { + for (const auto& r : h.requests()) { Log::info() << "retrieve," << tree.first << ","; r.dump(Log::info(), "", "", false); Log::info() << std::endl; @@ -209,8 +207,7 @@ void FDBList::execute(const CmdArgs& args) { } // namespace tools } // namespace fdb5 -int main(int argc, char **argv) { +int main(int argc, char** argv) { fdb5::tools::FDBList app(argc, argv); return app.start(); } - diff --git a/src/fdb5/tools/fdb-lock.cc b/src/fdb5/tools/fdb-lock.cc index 069725ccd..0a464b799 100644 --- a/src/fdb5/tools/fdb-lock.cc +++ b/src/fdb5/tools/fdb-lock.cc @@ -13,7 +13,7 @@ using namespace eckit::option; using namespace eckit; -int main(int argc, char **argv) { +int main(int argc, char** argv) { fdb5::tools::FDBLock app(argc, argv); return app.start(); } diff --git a/src/fdb5/tools/fdb-move.cc b/src/fdb5/tools/fdb-move.cc index 8fbb407b3..a9f9916e2 100644 --- a/src/fdb5/tools/fdb-move.cc +++ b/src/fdb5/tools/fdb-move.cc @@ -18,10 +18,10 @@ #include "eckit/option/SimpleOption.h" #include "eckit/thread/ThreadPool.h" -#include "fdb5/tools/FDBVisitTool.h" +#include "fdb5/LibFdb5.h" #include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" -#include "fdb5/LibFdb5.h" +#include "fdb5/tools/FDBVisitTool.h" #define MAX_THREADS 256 @@ -32,27 +32,24 @@ namespace fdb5::tools { //---------------------------------------------------------------------------------------------------------------------- - struct fdb_moveiterator_t { public: fdb_moveiterator_t(fdb5::MoveIterator&& iter) : iter_(std::move(iter)) {} - bool next(fdb5::MoveElement& elem) { - return iter_.next(elem); } + bool next(fdb5::MoveElement& elem) { return iter_.next(elem); } private: fdb5::MoveIterator iter_; }; - class MoveProducer : public eckit::distributed::Producer { public: // methods - - MoveProducer(eckit::distributed::Transport &transport, - const Config& config, - const std::vector& requests, - const eckit::option::CmdArgs &args) : - eckit::distributed::Producer(transport), fdb_(config), keep_(false), removeDelay_(0) { + MoveProducer(eckit::distributed::Transport& transport, const Config& config, + const std::vector& requests, const eckit::option::CmdArgs& args) + : eckit::distributed::Producer(transport) + , fdb_(config) + , keep_(false) + , removeDelay_(0) { keep_ = args.getBool("keep", false); removeDelay_ = args.getInt("delay", 0); @@ -85,12 +82,13 @@ class MoveProducer : public eckit::distributed::Producer { // check that the request is only referring a single DB - no ranges of values const metkit::mars::MarsRequest& marsReq = toolReq.request(); std::vector params = marsReq.params(); - for (const std::string& param: params) { + for (const std::string& param : params) { const std::vector& values = marsReq.values(param); if (values.size() != 1) { std::stringstream ss; - ss << "Move requires a single value for each parameter in the request." << std::endl << "Parameter " << param << "=" << values << " not supported." << std::endl; + ss << "Move requires a single value for each parameter in the request." << std::endl + << "Parameter " << param << "=" << values << " not supported." << std::endl; throw eckit::UserError(ss.str(), Here()); } } @@ -100,12 +98,14 @@ class MoveProducer : public eckit::distributed::Producer { StatsElement se; if (!it.next(se)) { std::stringstream ss; - ss << "Request " << toolReq << " does not matches with an existing database. Please specify a single database." << std::endl; + ss << "Request " << toolReq + << " does not matches with an existing database. Please specify a single database." << std::endl; throw eckit::UserError(ss.str(), Here()); } if (it.next(se)) { std::stringstream ss; - ss << "Request " << toolReq << " matches with more than one existing database. Please specify a single database." << std::endl; + ss << "Request " << toolReq + << " matches with more than one existing database. Please specify a single database." << std::endl; throw eckit::UserError(ss.str(), Here()); } @@ -127,7 +127,6 @@ class MoveProducer : public eckit::distributed::Producer { ~MoveProducer() {} private: // methods - virtual bool produce(eckit::distributed::Message& message) { ASSERT(moveIterator_); @@ -163,10 +162,9 @@ class MoveProducer : public eckit::distributed::Producer { } } - void messageFromWorker( eckit::distributed::Message& message, int worker) const {} + void messageFromWorker(eckit::distributed::Message& message, int worker) const {} private: // attributes - fdb5::FDB fdb_; fdb5::MoveElement last_; std::vector list_; @@ -174,20 +172,17 @@ class MoveProducer : public eckit::distributed::Producer { bool keep_; int removeDelay_; - }; - class MoveWorker : public eckit::distributed::Consumer { public: // methods - - MoveWorker(eckit::distributed::Transport& transport, const eckit::option::CmdArgs &args) : - eckit::distributed::Consumer(transport), count_(0) {} + MoveWorker(eckit::distributed::Transport& transport, const eckit::option::CmdArgs& args) + : eckit::distributed::Consumer(transport) + , count_(0) {} ~MoveWorker() {} protected: // members - void consume(eckit::distributed::Message& message) override { fdb5::FileCopy fileCopy(message); LOG_DEBUG_LIB(LibFdb5) << "MoveWorker " << fileCopy << std::endl; @@ -195,23 +190,18 @@ class MoveWorker : public eckit::distributed::Consumer { count_++; fileCopy.execute(); } - void finalise() override { - transport_.synchronise(); - }; + void finalise() override { transport_.synchronise(); }; // virtual void getNextMessage(eckit::Message& message) const; // virtual void failure(eckit::Message &message); - void shutdown(eckit::distributed::Message & message) override { + void shutdown(eckit::distributed::Message& message) override { LOG_DEBUG_LIB(LibFdb5) << "Shuting down MoveWorker..." << std::endl; message << count_; } - void getNextMessage(eckit::distributed::Message & message) const override { - getNextWorkMessage(message); - } + void getNextMessage(eckit::distributed::Message& message) const override { getNextWorkMessage(message); } private: // attributes - size_t count_; }; @@ -219,21 +209,19 @@ class MoveWorker : public eckit::distributed::Consumer { class MoveLoner : public eckit::distributed::Actor { public: // methods - - MoveLoner(eckit::distributed::Transport &transport, - eckit::distributed::Producer* producer, - eckit::distributed::Consumer* consumer, - size_t numThreads=1) : - eckit::distributed::Actor(transport), - producer_(producer), consumer_(consumer), numThreads_(numThreads) {} + MoveLoner(eckit::distributed::Transport& transport, eckit::distributed::Producer* producer, + eckit::distributed::Consumer* consumer, size_t numThreads = 1) + : eckit::distributed::Actor(transport) + , producer_(producer) + , consumer_(consumer) + , numThreads_(numThreads) {} private: // methods - virtual void run() { eckit::distributed::Message message; - eckit::distributed::Producer &producer = *producer_; - eckit::distributed::Consumer &consumer = *consumer_; + eckit::distributed::Producer& producer = *producer_; + eckit::distributed::Consumer& consumer = *consumer_; eckit::ThreadPool pool("move", numThreads_); @@ -241,7 +229,7 @@ class MoveLoner : public eckit::distributed::Actor { message.rewind(); try { pool.push(new FileCopy(message)); - } catch (eckit::Exception &e) { + } catch (eckit::Exception& e) { eckit::Log::error() << e.what() << std::endl; consumer.failure(message); } @@ -256,15 +244,14 @@ class MoveLoner : public eckit::distributed::Actor { } virtual void finalise() { - eckit::distributed::Producer &producer = *producer_; - eckit::distributed::Consumer &consumer = *consumer_; + eckit::distributed::Producer& producer = *producer_; + eckit::distributed::Consumer& consumer = *consumer_; consumer.finalise(); producer.finalise(); } private: // members - std::unique_ptr producer_; std::unique_ptr consumer_; size_t numThreads_; @@ -272,26 +259,21 @@ class MoveLoner : public eckit::distributed::Actor { //---------------------------------------------------------------------------------------------------------------------- - class FDBMove : public FDBVisitTool { public: // methods - - FDBMove(int argc, char **argv); + FDBMove(int argc, char** argv); ~FDBMove() override; private: // methods - void execute(const CmdArgs& args) override; - void init(const CmdArgs &args) override; + void init(const CmdArgs& args) override; private: // members - std::unique_ptr transport_; int threads_; }; -FDBMove::FDBMove(int argc, char **argv) : - FDBVisitTool(argc, argv, "class,expver,stream,date,time") { +FDBMove::FDBMove(int argc, char** argv) : FDBVisitTool(argc, argv, "class,expver,stream,date,time") { options_.push_back(new SimpleOption("dest", "Destination root")); options_.push_back(new SimpleOption("keep", "Keep source DB")); @@ -307,7 +289,6 @@ void FDBMove::init(const CmdArgs& args) { FDBVisitTool::init(args); } - void FDBMove::execute(const CmdArgs& args) { std::unique_ptr transport(eckit::distributed::TransportFactory::build(args)); @@ -323,10 +304,9 @@ void FDBMove::execute(const CmdArgs& args) { throw UserError(ss.str(), Here()); } - actor.reset(new MoveLoner(*transport, - new MoveProducer(*transport, config(args), requests("read"), args), - new MoveWorker(*transport, args), - threads_)); // do it all actor :) + actor.reset(new MoveLoner(*transport, new MoveProducer(*transport, config(args), requests("read"), args), + new MoveWorker(*transport, args), + threads_)); // do it all actor :) } else { if (transport->producer()) { actor.reset(new MoveProducer(*transport, config(args), requests("read"), args)); // work dispatcher @@ -346,13 +326,12 @@ void FDBMove::execute(const CmdArgs& args) { // << eckit::system::ResourceUsage() // << std::endl; - } catch (std::exception &e) { + } catch (std::exception& e) { eckit::Log::info() - // << eckit::TimeStamp() - // << " " - // << title - << " EXCEPTION: " - << e.what() << std::endl; + // << eckit::TimeStamp() + // << " " + // << title + << " EXCEPTION: " << e.what() << std::endl; transport->abort(); throw; } @@ -362,7 +341,7 @@ void FDBMove::execute(const CmdArgs& args) { } // namespace fdb5::tools -int main(int argc, char **argv) { +int main(int argc, char** argv) { fdb5::tools::FDBMove app(argc, argv); return app.start(); } diff --git a/src/fdb5/tools/fdb-overlay.cc b/src/fdb5/tools/fdb-overlay.cc index 900364827..e61435f97 100644 --- a/src/fdb5/tools/fdb-overlay.cc +++ b/src/fdb5/tools/fdb-overlay.cc @@ -12,10 +12,10 @@ #include "eckit/option/SimpleOption.h" #include "eckit/option/VectorOption.h" +#include "fdb5/LibFdb5.h" #include "fdb5/api/helpers/FDBToolRequest.h" #include "fdb5/config/Config.h" #include "fdb5/database/Key.h" -#include "fdb5/LibFdb5.h" #include "fdb5/rules/Schema.h" #include "fdb5/toc/TocEngine.h" #include "fdb5/tools/FDBTool.h" @@ -31,33 +31,29 @@ namespace tools { class FdbOverlay : public FDBTool { public: // methods - - FdbOverlay(int argc, char **argv) : - FDBTool(argc, argv), - variableKeys_{"class", "expver"}, - remove_(false), - force_(false) { - options_.push_back(new VectorOption("variable-keys", - "The keys that may vary between mounted DBs", - 0, ",")); + FdbOverlay(int argc, char** argv) + : FDBTool(argc, argv) + , variableKeys_{"class", "expver"} + , remove_(false) + , force_(false) { + options_.push_back( + new VectorOption("variable-keys", "The keys that may vary between mounted DBs", 0, ",")); options_.push_back(new SimpleOption("remove", "Remove a previously FDB overlay")); options_.push_back(new SimpleOption("force", "Apply overlay even if target already exists")); } private: // methods - virtual void init(const option::CmdArgs& args); virtual void execute(const option::CmdArgs& args); - virtual void usage(const std::string &tool) const; + virtual void usage(const std::string& tool) const; private: // members - std::vector variableKeys_; bool remove_; bool force_; }; -void FdbOverlay::usage(const std::string &tool) const { +void FdbOverlay::usage(const std::string& tool) const { Log::info() << std::endl << "Usage: " << tool << " [options] [source DB request] [target DB request]" << std::endl @@ -155,7 +151,7 @@ void FdbOverlay::execute(const option::CmdArgs& args) { ASSERT(dbTarget->uri() != dbSource->uri()); std::unique_ptr newCatalogue = CatalogueWriterFactory::instance().build(target.canonical(), conf); - if (newCatalogue->type() == TocEngine::typeName() && dbSource->type() == TocEngine::typeName()) { + if (newCatalogue->type() == TocEngine::typeName() && dbSource->type() == TocEngine::typeName()) { newCatalogue->overlayDB(*dbSource, vkeys, remove_); } } @@ -163,9 +159,9 @@ void FdbOverlay::execute(const option::CmdArgs& args) { //---------------------------------------------------------------------------------------------------------------------- } // namespace tools -} // namespace fbb5 +} // namespace fdb5 -int main(int argc, char **argv) { +int main(int argc, char** argv) { fdb5::tools::FdbOverlay app(argc, argv); return app.start(); } diff --git a/src/fdb5/tools/fdb-patch.cc b/src/fdb5/tools/fdb-patch.cc index 16e91b3c3..8f80b2112 100644 --- a/src/fdb5/tools/fdb-patch.cc +++ b/src/fdb5/tools/fdb-patch.cc @@ -12,15 +12,15 @@ #include "eccodes.h" -#include "eckit/option/CmdArgs.h" -#include "eckit/option/SimpleOption.h" #include "eckit/log/Bytes.h" #include "eckit/log/Plural.h" #include "eckit/message/Message.h" +#include "eckit/option/CmdArgs.h" +#include "eckit/option/SimpleOption.h" #include "fdb5/api/helpers/ListIterator.h" -#include "fdb5/message/MessageArchiver.h" #include "fdb5/io/HandleGatherer.h" +#include "fdb5/message/MessageArchiver.h" #include "fdb5/tools/FDBVisitTool.h" #include "metkit/codes/CodesContent.h" @@ -36,15 +36,12 @@ namespace tools { class PatchArchiver : public MessageArchiver { public: // methods - explicit PatchArchiver(const Key& key) : key_(key) {} private: // methods - eckit::message::Message patch(const eckit::message::Message& msg) override; private: // members - const Key& key_; }; @@ -60,9 +57,7 @@ eckit::message::Message PatchArchiver::patch(const eckit::message::Message& msg) } return eckit::message::Message(new metkit::codes::CodesContent(h, true)); - } - catch (...) - { + } catch (...) { grib_handle_delete(h); throw; } @@ -73,26 +68,21 @@ eckit::message::Message PatchArchiver::patch(const eckit::message::Message& msg) class FDBPatch : public FDBVisitTool { public: // methods - - FDBPatch(int argc, char **argv) : - FDBVisitTool(argc, argv, "class,expver,stream,date,time") { + FDBPatch(int argc, char** argv) : FDBVisitTool(argc, argv, "class,expver,stream,date,time") { options_.push_back(new SimpleOption("expver", "Set the expver")); options_.push_back(new SimpleOption("class", "Set the class")); } private: // methods - virtual void execute(const CmdArgs& args); - virtual void init(const CmdArgs &args); + virtual void init(const CmdArgs& args); virtual int minimumPositionalArguments() const; private: // members - fdb5::Key key_; }; - int FDBPatch::minimumPositionalArguments() const { return 1; } @@ -121,10 +111,8 @@ void FDBPatch::init(const CmdArgs& args) { void FDBPatch::execute(const CmdArgs& args) { - FDB fdb(config(args)); - Timer timer(args.tool()); std::set uniqueKeys; @@ -178,21 +166,10 @@ void FDBPatch::execute(const CmdArgs& args) { // Status report - Log::info() << std::endl - << "Summary" << std::endl - << "=======" << std::endl << std::endl; - Log::info() << Plural(uniqueKeys.size(), "field") - << " (" - << Bytes(bytes) - << ") copied to " - << key_ - << std::endl; + Log::info() << std::endl << "Summary" << std::endl << "=======" << std::endl << std::endl; + Log::info() << Plural(uniqueKeys.size(), "field") << " (" << Bytes(bytes) << ") copied to " << key_ << std::endl; - Log::info() << "Rates: " - << Bytes(bytes, timer) - << ", " - << uniqueKeys.size() / timer.elapsed() - << " fields/s" + Log::info() << "Rates: " << Bytes(bytes, timer) << ", " << uniqueKeys.size() / timer.elapsed() << " fields/s" << std::endl; } @@ -201,7 +178,7 @@ void FDBPatch::execute(const CmdArgs& args) { } // namespace tools } // namespace fdb5 -int main(int argc, char **argv) { +int main(int argc, char** argv) { fdb5::tools::FDBPatch app(argc, argv); return app.start(); } diff --git a/src/fdb5/tools/fdb-purge.cc b/src/fdb5/tools/fdb-purge.cc index e6947277a..9bd0f76a8 100644 --- a/src/fdb5/tools/fdb-purge.cc +++ b/src/fdb5/tools/fdb-purge.cc @@ -8,14 +8,13 @@ * does it submit to any jurisdiction. */ -#include "fdb5/tools/FDBVisitTool.h" #include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/tools/FDBVisitTool.h" #include "eckit/option/CmdArgs.h" #include "eckit/option/SimpleOption.h" - using namespace eckit; using namespace eckit::option; @@ -29,12 +28,11 @@ namespace tools { class FDBPurge : public FDBVisitTool { public: // methods - - FDBPurge(int argc, char **argv) : - FDBVisitTool(argc, argv, "class,expver,stream,date,time"), - doit_(false), - porcelain_(false), - ignoreNoData_(false) { + FDBPurge(int argc, char** argv) + : FDBVisitTool(argc, argv, "class,expver,stream,date,time") + , doit_(false) + , porcelain_(false) + , ignoreNoData_(false) { options_.push_back(new SimpleOption("doit", "Delete the files (data and indexes)")); options_.push_back(new SimpleOption("ignore-no-data", "No data available to delete is not an error")); @@ -42,17 +40,15 @@ class FDBPurge : public FDBVisitTool { } private: // methods - - virtual void init(const CmdArgs &args); + virtual void init(const CmdArgs& args); virtual void execute(const CmdArgs& args); - virtual void finish(const CmdArgs &args); + virtual void finish(const CmdArgs& args); bool doit_; bool porcelain_; bool ignoreNoData_; }; - void FDBPurge::init(const CmdArgs& args) { FDBVisitTool::init(args); doit_ = args.getBool("doit", false); @@ -87,14 +83,10 @@ void FDBPurge::execute(const CmdArgs& args) { } } - void FDBPurge::finish(const CmdArgs&) { if (!doit_ && !porcelain_) { - Log::info() << std::endl - << "Rerun command with --doit flag to delete unused files" - << std::endl - << std::endl; + Log::info() << std::endl << "Rerun command with --doit flag to delete unused files" << std::endl << std::endl; } } @@ -103,8 +95,7 @@ void FDBPurge::finish(const CmdArgs&) { } // namespace tools } // namespace fdb5 - -int main(int argc, char **argv) { +int main(int argc, char** argv) { fdb5::tools::FDBPurge app(argc, argv); return app.start(); } diff --git a/src/fdb5/tools/fdb-read.cc b/src/fdb5/tools/fdb-read.cc index eb6aaca4f..fd2e4d4a8 100644 --- a/src/fdb5/tools/fdb-read.cc +++ b/src/fdb5/tools/fdb-read.cc @@ -15,31 +15,29 @@ #include "eckit/option/CmdArgs.h" #include "eckit/option/SimpleOption.h" -#include "metkit/mars/MarsRequest.h" -#include "metkit/mars/MarsParser.h" #include "metkit/mars/MarsExpension.h" +#include "metkit/mars/MarsParser.h" +#include "metkit/mars/MarsRequest.h" #include "fdb5/api/FDB.h" -#include "fdb5/message/MessageDecoder.h" #include "fdb5/io/HandleGatherer.h" +#include "fdb5/message/MessageDecoder.h" #include "fdb5/tools/FDBTool.h" - class FDBRead : public fdb5::FDBTool { - virtual void execute(const eckit::option::CmdArgs &args); - virtual void usage(const std::string &tool) const; + virtual void execute(const eckit::option::CmdArgs& args); + virtual void usage(const std::string& tool) const; virtual int numberOfPositionalArguments() const { return 2; } - public: - FDBRead(int argc, char **argv): fdb5::FDBTool(argc, argv) { + +public: + FDBRead(int argc, char** argv) : fdb5::FDBTool(argc, argv) { options_.push_back(new eckit::option::SimpleOption("extract", "Extract request from a GRIB file")); options_.push_back(new eckit::option::SimpleOption("raw", "Uses the raw request, without expansion")); - options_.push_back( - new eckit::option::SimpleOption("statistics", - "Report timing statistics")); + options_.push_back(new eckit::option::SimpleOption("statistics", "Report timing statistics")); } }; -void FDBRead::usage(const std::string &tool) const { +void FDBRead::usage(const std::string& tool) const { eckit::Log::info() << std::endl << "Usage: " << tool << " request.mars target.grib" << std::endl << " " << tool << " --raw request.mars target.grib" << std::endl @@ -48,7 +46,7 @@ void FDBRead::usage(const std::string &tool) const { fdb5::FDBTool::usage(tool); } -void FDBRead::execute(const eckit::option::CmdArgs &args) { +void FDBRead::execute(const eckit::option::CmdArgs& args) { bool extract = args.getBool("extract", false); bool raw = args.getBool("raw", false); @@ -101,9 +99,7 @@ void FDBRead::execute(const eckit::option::CmdArgs &args) { dh->copyTo(out); } - -int main(int argc, char **argv) { +int main(int argc, char** argv) { FDBRead app(argc, argv); return app.start(); } - diff --git a/src/fdb5/tools/fdb-reconsolidate-toc.cc b/src/fdb5/tools/fdb-reconsolidate-toc.cc index 5d5184dfe..bb7d00f39 100644 --- a/src/fdb5/tools/fdb-reconsolidate-toc.cc +++ b/src/fdb5/tools/fdb-reconsolidate-toc.cc @@ -8,12 +8,12 @@ * does it submit to any jurisdiction. */ -#include "fdb5/tools/FDBTool.h" #include "fdb5/database/Catalogue.h" +#include "fdb5/tools/FDBTool.h" -#include "eckit/option/CmdArgs.h" #include "eckit/config/LocalConfiguration.h" #include "eckit/filesystem/URI.h" +#include "eckit/option/CmdArgs.h" using namespace eckit; @@ -21,24 +21,19 @@ using namespace eckit; class FDBReconsolidateToc : public fdb5::FDBTool { - public: // methods +public: // methods + FDBReconsolidateToc(int argc, char** argv) : fdb5::FDBTool(argc, argv) {} - FDBReconsolidateToc(int argc, char **argv) : - fdb5::FDBTool(argc, argv) {} - - private: // methods - - virtual void usage(const std::string &tool) const; +private: // methods + virtual void usage(const std::string& tool) const; virtual void execute(const eckit::option::CmdArgs& args); }; -void FDBReconsolidateToc::usage(const std::string &tool) const { - Log::info() << std::endl - << "Usage: " << tool << " path" << std::endl; +void FDBReconsolidateToc::usage(const std::string& tool) const { + Log::info() << std::endl << "Usage: " << tool << " path" << std::endl; fdb5::FDBTool::usage(tool); } - void FDBReconsolidateToc::execute(const eckit::option::CmdArgs& args) { if (args.count() != 1) { @@ -54,15 +49,15 @@ void FDBReconsolidateToc::execute(const eckit::option::CmdArgs& args) { dbPath = dbPath.dirName(); } - std::unique_ptr catalogue = fdb5::CatalogueWriterFactory::instance().build(eckit::URI("toc", dbPath), config(args)); + std::unique_ptr catalogue = + fdb5::CatalogueWriterFactory::instance().build(eckit::URI("toc", dbPath), config(args)); catalogue->reconsolidate(); } //---------------------------------------------------------------------------------------------------------------------- -int main(int argc, char **argv) { +int main(int argc, char** argv) { FDBReconsolidateToc app(argc, argv); return app.start(); } - diff --git a/src/fdb5/tools/fdb-root.cc b/src/fdb5/tools/fdb-root.cc index 970858a0b..8a0e9d6db 100644 --- a/src/fdb5/tools/fdb-root.cc +++ b/src/fdb5/tools/fdb-root.cc @@ -11,10 +11,10 @@ #include "eckit/option/CmdArgs.h" #include "eckit/option/SimpleOption.h" +#include "fdb5/LibFdb5.h" #include "fdb5/api/helpers/FDBToolRequest.h" #include "fdb5/config/Config.h" #include "fdb5/database/Key.h" -#include "fdb5/LibFdb5.h" #include "fdb5/rules/Schema.h" #include "fdb5/tools/FDBTool.h" @@ -26,28 +26,26 @@ namespace tools { class FdbRoot : public FDBTool { public: // methods - - FdbRoot(int argc, char **argv) : - FDBTool(argc, argv) { - options_.push_back(new eckit::option::SimpleOption("create", "If a DB does not exist for the provided key, create it")); + FdbRoot(int argc, char** argv) : FDBTool(argc, argv) { + options_.push_back( + new eckit::option::SimpleOption("create", "If a DB does not exist for the provided key, create it")); } private: // methods - virtual void execute(const eckit::option::CmdArgs& args); - virtual void usage(const std::string &tool) const; + virtual void usage(const std::string& tool) const; }; -void FdbRoot::usage(const std::string &tool) const { +void FdbRoot::usage(const std::string& tool) const { eckit::Log::info() << std::endl << "Usage: " << tool << " [options] [request1] [request2] ..." << std::endl << std::endl << std::endl << "Examples:" << std::endl - << "=========" << std::endl << std::endl - << tool << " class=od,expver=0001,stream=oper,date=20160428,time=1200" + << "=========" << std::endl << std::endl + << tool << " class=od,expver=0001,stream=oper,date=20160428,time=1200" << std::endl << std::endl; FDBTool::usage(tool); } @@ -66,7 +64,7 @@ void FdbRoot::execute(const eckit::option::CmdArgs& args) { Config conf = config(args); const Schema& schema = conf.schema(); TypedKey result{conf.schema().registry()}; - ASSERT( schema.expandFirstLevel(request.request(), result) ); + ASSERT(schema.expandFirstLevel(request.request(), result)); const auto key = result.canonical(); eckit::Log::info() << result << std::endl; @@ -89,9 +87,9 @@ void FdbRoot::execute(const eckit::option::CmdArgs& args) { //---------------------------------------------------------------------------------------------------------------------- } // namespace tools -} // namespace fbb5 +} // namespace fdb5 -int main(int argc, char **argv) { +int main(int argc, char** argv) { fdb5::tools::FdbRoot app(argc, argv); return app.start(); } diff --git a/src/fdb5/tools/fdb-schema.cc b/src/fdb5/tools/fdb-schema.cc index 6892e361a..d9b03664f 100644 --- a/src/fdb5/tools/fdb-schema.cc +++ b/src/fdb5/tools/fdb-schema.cc @@ -10,8 +10,8 @@ #include -#include "eckit/option/CmdArgs.h" #include "eckit/log/Log.h" +#include "eckit/option/CmdArgs.h" #include "fdb5/LibFdb5.h" #include "fdb5/config/Config.h" @@ -26,20 +26,19 @@ namespace tools { //---------------------------------------------------------------------------------------------------------------------- class FdbSchema : public FDBTool { - virtual void execute(const eckit::option::CmdArgs &args); - virtual void usage(const std::string &tool) const; + virtual void execute(const eckit::option::CmdArgs& args); + virtual void usage(const std::string& tool) const; + public: - FdbSchema(int argc, char **argv): FDBTool(argc, argv) {} + FdbSchema(int argc, char** argv) : FDBTool(argc, argv) {} }; -void FdbSchema::usage(const std::string &tool) const { - eckit::Log::info() << std::endl - << "Usage: " << tool << " [shema] ..." << std::endl; +void FdbSchema::usage(const std::string& tool) const { + eckit::Log::info() << std::endl << "Usage: " << tool << " [shema] ..." << std::endl; FDBTool::usage(tool); } -void FdbSchema:: execute(const eckit::option::CmdArgs &args) { - +void FdbSchema::execute(const eckit::option::CmdArgs& args) { // With no arguments, provide the current master configuration schema (i.e. that selected by FDB_HOME) @@ -56,7 +55,8 @@ void FdbSchema:: execute(const eckit::option::CmdArgs &args) { eckit::PathName path(args(i)); if (path.isDir()) { - std::unique_ptr cat = CatalogueReaderFactory::instance().build(eckit::URI("toc", path), fdb5::Config()); + std::unique_ptr cat = + CatalogueReaderFactory::instance().build(eckit::URI("toc", path), fdb5::Config()); ASSERT(cat->open()); cat->schema().dump(Log::info()); } else { @@ -73,8 +73,7 @@ void FdbSchema:: execute(const eckit::option::CmdArgs &args) { } // namespace tools } // namespace fdb5 -int main(int argc, char **argv) { +int main(int argc, char** argv) { fdb5::tools::FdbSchema app(argc, argv); return app.start(); } - diff --git a/src/fdb5/tools/fdb-stats.cc b/src/fdb5/tools/fdb-stats.cc index 10abd8145..13358c83a 100644 --- a/src/fdb5/tools/fdb-stats.cc +++ b/src/fdb5/tools/fdb-stats.cc @@ -8,10 +8,10 @@ * does it submit to any jurisdiction. */ -#include "fdb5/tools/FDBVisitTool.h" +#include "fdb5/api/FDB.h" #include "fdb5/database/DbStats.h" #include "fdb5/database/IndexStats.h" -#include "fdb5/api/FDB.h" +#include "fdb5/tools/FDBVisitTool.h" #include "eckit/option/CmdArgs.h" #include "eckit/option/SimpleOption.h" @@ -19,7 +19,6 @@ using namespace eckit; using namespace eckit::option; - namespace fdb5 { namespace tools { @@ -28,10 +27,7 @@ namespace tools { class FDBStats : public FDBVisitTool { public: // methods - - FDBStats(int argc, char **argv) : - FDBVisitTool(argc, argv, "class,expver"), - details_(false) { + FDBStats(int argc, char** argv) : FDBVisitTool(argc, argv, "class,expver"), details_(false) { options_.push_back(new SimpleOption("details", "Print report for each database visited")); } @@ -39,17 +35,14 @@ class FDBStats : public FDBVisitTool { ~FDBStats() override {} private: // methods - void execute(const CmdArgs& args) override; - void init(const CmdArgs &args) override; + void init(const CmdArgs& args) override; private: // members - bool details_; - }; -void FDBStats::init(const eckit::option::CmdArgs &args) { +void FDBStats::init(const eckit::option::CmdArgs& args) { FDBVisitTool::init(args); details_ = args.getBool("details", false); } @@ -111,7 +104,7 @@ void FDBStats::execute(const CmdArgs& args) { } // namespace tools } // namespace fdb5 -int main(int argc, char **argv) { +int main(int argc, char** argv) { fdb5::tools::FDBStats app(argc, argv); return app.start(); } diff --git a/src/fdb5/tools/fdb-status.cc b/src/fdb5/tools/fdb-status.cc index 14f635799..6fe3c9c7e 100644 --- a/src/fdb5/tools/fdb-status.cc +++ b/src/fdb5/tools/fdb-status.cc @@ -18,7 +18,6 @@ using namespace eckit::option; using namespace eckit; - namespace fdb5 { namespace tools { @@ -26,17 +25,12 @@ namespace tools { class FDBStatus : public FDBVisitTool { public: // methods - - FDBStatus(int argc, char **argv) : - FDBVisitTool(argc, argv, "class,expver") {} + FDBStatus(int argc, char** argv) : FDBVisitTool(argc, argv, "class,expver") {} private: // methods - void execute(const CmdArgs& args) override; - }; - void FDBStatus::execute(const CmdArgs& args) { FDB fdb(config(args)); @@ -52,11 +46,16 @@ void FDBStatus::execute(const CmdArgs& args) { Log::info() << "Database: " << elem.key << std::endl << " location: " << elem.location.asString() << std::endl; - if (!elem.controlIdentifiers.enabled(ControlIdentifier::Retrieve)) Log::info() << " retrieve: LOCKED" << std::endl; - if (!elem.controlIdentifiers.enabled(ControlIdentifier::Archive)) Log::info() << " archive: LOCKED" << std::endl; - if (!elem.controlIdentifiers.enabled(ControlIdentifier::List)) Log::info() << " list: LOCKED" << std::endl; - if (!elem.controlIdentifiers.enabled(ControlIdentifier::Wipe)) Log::info() << " wipe: LOCKED" << std::endl; - if (!elem.controlIdentifiers.enabled(ControlIdentifier::UniqueRoot)) Log::info() << " multi-root: PERMITTED" << std::endl; + if (!elem.controlIdentifiers.enabled(ControlIdentifier::Retrieve)) + Log::info() << " retrieve: LOCKED" << std::endl; + if (!elem.controlIdentifiers.enabled(ControlIdentifier::Archive)) + Log::info() << " archive: LOCKED" << std::endl; + if (!elem.controlIdentifiers.enabled(ControlIdentifier::List)) + Log::info() << " list: LOCKED" << std::endl; + if (!elem.controlIdentifiers.enabled(ControlIdentifier::Wipe)) + Log::info() << " wipe: LOCKED" << std::endl; + if (!elem.controlIdentifiers.enabled(ControlIdentifier::UniqueRoot)) + Log::info() << " multi-root: PERMITTED" << std::endl; count++; } @@ -74,8 +73,7 @@ void FDBStatus::execute(const CmdArgs& args) { } // namespace tools } // namespace fdb5 - -int main(int argc, char **argv) { +int main(int argc, char** argv) { fdb5::tools::FDBStatus app(argc, argv); return app.start(); } diff --git a/src/fdb5/tools/fdb-unlock.cc b/src/fdb5/tools/fdb-unlock.cc index dcf35f88c..513ebf94b 100644 --- a/src/fdb5/tools/fdb-unlock.cc +++ b/src/fdb5/tools/fdb-unlock.cc @@ -13,7 +13,7 @@ using namespace eckit::option; using namespace eckit; -int main(int argc, char **argv) { +int main(int argc, char** argv) { bool unlock = true; fdb5::tools::FDBLock app(argc, argv, unlock); return app.start(); diff --git a/src/fdb5/tools/fdb-where.cc b/src/fdb5/tools/fdb-where.cc index de13dedf3..8ccdd982d 100644 --- a/src/fdb5/tools/fdb-where.cc +++ b/src/fdb5/tools/fdb-where.cc @@ -19,29 +19,23 @@ using namespace eckit::option; using namespace eckit; - namespace fdb5 { namespace tools { //---------------------------------------------------------------------------------------------------------------------- class FDBWhere : public FDBVisitTool { - public: // methods - - FDBWhere(int argc, char **argv) : - FDBVisitTool(argc, argv, "class,expver"), - porcelain_(false) { +public: // methods + FDBWhere(int argc, char** argv) : FDBVisitTool(argc, argv, "class,expver"), porcelain_(false) { options_.push_back(new SimpleOption("porcelain", "Streamlined output for input into other tools")); } - private: // methods - +private: // methods void execute(const CmdArgs& args) override; - void init(const CmdArgs &args) override; - - private: // members + void init(const CmdArgs& args) override; +private: // members bool porcelain_; }; @@ -51,7 +45,6 @@ void FDBWhere::init(const CmdArgs& args) { porcelain_ = args.getBool("porcelain", false); } - void FDBWhere::execute(const CmdArgs& args) { FDB fdb(config(args)); @@ -80,8 +73,7 @@ void FDBWhere::execute(const CmdArgs& args) { } // namespace tools } // namespace fdb5 - -int main(int argc, char **argv) { +int main(int argc, char** argv) { fdb5::tools::FDBWhere app(argc, argv); return app.start(); } diff --git a/src/fdb5/tools/fdb-wipe.cc b/src/fdb5/tools/fdb-wipe.cc index d76aec07d..427ad7dd8 100644 --- a/src/fdb5/tools/fdb-wipe.cc +++ b/src/fdb5/tools/fdb-wipe.cc @@ -11,9 +11,9 @@ #include "eckit/option/CmdArgs.h" #include "eckit/option/SimpleOption.h" -#include "fdb5/tools/FDBVisitTool.h" #include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/tools/FDBVisitTool.h" using namespace eckit; using namespace eckit::option; @@ -26,26 +26,25 @@ namespace tools { class FDBWipe : public FDBVisitTool { public: // methods - - FDBWipe(int argc, char **argv) : - FDBVisitTool(argc, argv, "class,expver,stream,date,time"), - doit_(false), - ignoreNoData_(false), - porcelain_(false), - unsafeWipeAll_(false) { + FDBWipe(int argc, char** argv) + : FDBVisitTool(argc, argv, "class,expver,stream,date,time") + , doit_(false) + , ignoreNoData_(false) + , porcelain_(false) + , unsafeWipeAll_(false) { options_.push_back(new SimpleOption("doit", "Delete the files (data and indexes)")); options_.push_back(new SimpleOption("ignore-no-data", "No data available to delete is not an error")); options_.push_back(new SimpleOption("porcelain", "List only the deleted files")); - options_.push_back(new SimpleOption("unsafe-wipe-all", "Wipe all (unowned) contents of an unclean database")); + options_.push_back( + new SimpleOption("unsafe-wipe-all", "Wipe all (unowned) contents of an unclean database")); } private: // methods - - virtual void usage(const std::string &tool) const; - virtual void init(const CmdArgs &args); + virtual void usage(const std::string& tool) const; + virtual void init(const CmdArgs& args); virtual void execute(const CmdArgs& args); - virtual void finish(const CmdArgs &args); + virtual void finish(const CmdArgs& args); private: // members bool doit_; @@ -54,20 +53,20 @@ class FDBWipe : public FDBVisitTool { bool unsafeWipeAll_; }; -void FDBWipe::usage(const std::string &tool) const { +void FDBWipe::usage(const std::string& tool) const { Log::info() << std::endl << "Usage: " << tool << " [options] [DB request]" << std::endl << std::endl << std::endl << "Examples:" << std::endl - << "=========" << std::endl << std::endl - << tool << " class=rd,expver=xywz,stream=oper,date=20190603,time=00" + << "=========" << std::endl << std::endl + << tool << " class=rd,expver=xywz,stream=oper,date=20190603,time=00" << std::endl << std::endl; } -void FDBWipe::init(const CmdArgs &args) { +void FDBWipe::init(const CmdArgs& args) { FDBVisitTool::init(args); @@ -106,14 +105,10 @@ void FDBWipe::execute(const CmdArgs& args) { } } - void FDBWipe::finish(const CmdArgs&) { if (!doit_ && !porcelain_) { - Log::info() << std::endl - << "Rerun command with --doit flag to delete unused files" - << std::endl - << std::endl; + Log::info() << std::endl << "Rerun command with --doit flag to delete unused files" << std::endl << std::endl; } } @@ -122,7 +117,7 @@ void FDBWipe::finish(const CmdArgs&) { } // namespace tools } // namespace fdb5 -int main(int argc, char **argv) { +int main(int argc, char** argv) { fdb5::tools::FDBWipe app(argc, argv); return app.start(); } diff --git a/src/fdb5/tools/fdb-write.cc b/src/fdb5/tools/fdb-write.cc index fdc77641e..76c343267 100644 --- a/src/fdb5/tools/fdb-write.cc +++ b/src/fdb5/tools/fdb-write.cc @@ -12,44 +12,45 @@ #include "eckit/config/LocalConfiguration.h" #include "eckit/io/DataHandle.h" +#include "eckit/log/Log.h" #include "eckit/option/CmdArgs.h" #include "eckit/option/SimpleOption.h" -#include "eckit/log/Log.h" #include "fdb5/message/MessageArchiver.h" #include "fdb5/tools/FDBTool.h" - class FDBWrite : public fdb5::FDBTool { - virtual void usage(const std::string &tool) const; + virtual void usage(const std::string& tool) const; - virtual void init(const eckit::option::CmdArgs &args); + virtual void init(const eckit::option::CmdArgs& args); virtual int minimumPositionalArguments() const { return 1; } - virtual void execute(const eckit::option::CmdArgs &args); + virtual void execute(const eckit::option::CmdArgs& args); public: + FDBWrite(int argc, char** argv) : fdb5::FDBTool(argc, argv), archivers_(1), verbose_(false) { - FDBWrite(int argc, char **argv) : - fdb5::FDBTool(argc, argv), - archivers_(1), verbose_(false) { + options_.push_back(new eckit::option::SimpleOption( + "include-filter", + "List of comma separated key-values of what to include from the input data, e.g " + "--include-filter=stream=enfo,date=10102017")); options_.push_back( - new eckit::option::SimpleOption("include-filter", - "List of comma separated key-values of what to include from the input data, e.g --include-filter=stream=enfo,date=10102017")); + new eckit::option::SimpleOption("exclude-filter", + "List of comma separated key-values of what to exclude from " + "the input data, e.g --exclude-filter=time=0000")); - options_.push_back( - new eckit::option::SimpleOption("exclude-filter", - "List of comma separated key-values of what to exclude from the input data, e.g --exclude-filter=time=0000")); + options_.push_back(new eckit::option::SimpleOption( + "modifiers", + "List of comma separated key-values of modifiers to each message " + "int input data, e.g --modifiers=packingType=grib_ccsds,expver=0042")); options_.push_back( - new eckit::option::SimpleOption("modifiers", - "List of comma separated key-values of modifiers to each message " - "int input data, e.g --modifiers=packingType=grib_ccsds,expver=0042")); - - options_.push_back(new eckit::option::SimpleOption("archivers", "Number of archivers (default is 1). Input files are distributed among the user-specified archivers. For testing purposes only!")); + new eckit::option::SimpleOption("archivers", + "Number of archivers (default is 1). Input files are distributed " + "among the user-specified archivers. For testing purposes only!")); options_.push_back(new eckit::option::SimpleOption("statistics", "Report timing statistics")); @@ -63,13 +64,14 @@ class FDBWrite : public fdb5::FDBTool { bool verbose_; }; -void FDBWrite::usage(const std::string &tool) const { - eckit::Log::info() << std::endl << "Usage: " << tool << " [--filter-include=...] [--filter-exclude=...] [path2] ..." << std::endl; +void FDBWrite::usage(const std::string& tool) const { + eckit::Log::info() << std::endl + << "Usage: " << tool << " [--filter-include=...] [--filter-exclude=...] [path2] ..." + << std::endl; fdb5::FDBTool::usage(tool); } -void FDBWrite::init(const eckit::option::CmdArgs& args) -{ +void FDBWrite::init(const eckit::option::CmdArgs& args) { FDBTool::init(args); args.get("include-filter", filterInclude_); args.get("exclude-filter", filterExclude_); @@ -78,10 +80,10 @@ void FDBWrite::init(const eckit::option::CmdArgs& args) verbose_ = args.getBool("verbose", false); } -void FDBWrite::execute(const eckit::option::CmdArgs &args) { +void FDBWrite::execute(const eckit::option::CmdArgs& args) { std::vector> archivers; - for (int i=0; i a(new fdb5::MessageArchiver(fdb5::Key(), false, verbose_, config(args))); a->filters(filterInclude_, filterExclude_); a->modifiers(modifiers_); @@ -94,9 +96,9 @@ void FDBWrite::execute(const eckit::option::CmdArgs &args) { eckit::Log::info() << "Processing " << path << std::endl; - std::unique_ptr dh ( path.fileHandle() ); + std::unique_ptr dh(path.fileHandle()); - archivers.at(i%archivers_)->archive( *dh ); + archivers.at(i % archivers_)->archive(*dh); } for (auto& a : archivers) { @@ -106,8 +108,7 @@ void FDBWrite::execute(const eckit::option::CmdArgs &args) { //---------------------------------------------------------------------------------------------------------------------- -int main(int argc, char **argv) { +int main(int argc, char** argv) { FDBWrite app(argc, argv); return app.start(); } - diff --git a/src/fdb5/types/Type.cc b/src/fdb5/types/Type.cc index 99c4cddf5..c412a1724 100644 --- a/src/fdb5/types/Type.cc +++ b/src/fdb5/types/Type.cc @@ -16,41 +16,33 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -Type::Type(const std::string &name, const std::string &type) : - name_(name), - type_(type) { -} +Type::Type(const std::string& name, const std::string& type) : name_(name), type_(type) {} -Type::~Type() { -} +Type::~Type() {} -void Type::getValues(const metkit::mars::MarsRequest &request, - const std::string &keyword, - eckit::StringList &values, - const Notifier&, - const CatalogueReader*) const { +void Type::getValues(const metkit::mars::MarsRequest& request, const std::string& keyword, eckit::StringList& values, + const Notifier&, const CatalogueReader*) const { request.getValues(keyword, values, true); } -const std::string &Type::type() const { +const std::string& Type::type() const { return type_; } -std::string Type::tidy(const std::string &value) const { +std::string Type::tidy(const std::string& value) const { return eckit::StringTools::lower(value); } -std::string Type::toKey(const std::string &value) const { +std::string Type::toKey(const std::string& value) const { return eckit::StringTools::lower(value); } -std::ostream &operator<<(std::ostream &s, const Type &x) { +std::ostream& operator<<(std::ostream& s, const Type& x) { x.print(s); return s; } -bool Type::match(const std::string&, const std::string& value1, const std::string& value2) const -{ +bool Type::match(const std::string&, const std::string& value1, const std::string& value2) const { return (value1 == value2); } diff --git a/src/fdb5/types/Type.h b/src/fdb5/types/Type.h index cfb3c109e..497d6bc11 100644 --- a/src/fdb5/types/Type.h +++ b/src/fdb5/types/Type.h @@ -25,9 +25,9 @@ class Notifier; namespace metkit { namespace mars { - class MarsRequest; -} +class MarsRequest; } +} // namespace metkit namespace fdb5 { @@ -39,40 +39,32 @@ class Notifier; class Type : private eckit::NonCopyable { public: // methods - - Type(const std::string &name, const std::string &type); + Type(const std::string& name, const std::string& type); virtual ~Type(); - virtual std::string tidy(const std::string &value) const ; + virtual std::string tidy(const std::string& value) const; - virtual std::string toKey(const std::string &value) const ; + virtual std::string toKey(const std::string& value) const; - virtual void getValues(const metkit::mars::MarsRequest &request, - const std::string &keyword, - eckit::StringList &values, - const Notifier &wind, - const CatalogueReader* cat) const; + virtual void getValues(const metkit::mars::MarsRequest& request, const std::string& keyword, + eckit::StringList& values, const Notifier& wind, const CatalogueReader* cat) const; virtual bool match(const std::string& keyword, const std::string& value1, const std::string& value2) const; - friend std::ostream &operator<<(std::ostream &s, const Type &x); + friend std::ostream& operator<<(std::ostream& s, const Type& x); public: // class methods + static const Type& lookup(const std::string& keyword); - static const Type &lookup(const std::string &keyword); - - const std::string &type() const; + const std::string& type() const; private: // methods - - virtual void print( std::ostream &out ) const = 0; + virtual void print(std::ostream& out) const = 0; protected: // members - std::string name_; std::string type_; - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/types/TypeAbbreviation.cc b/src/fdb5/types/TypeAbbreviation.cc index ebc8218de..2925dc0c4 100644 --- a/src/fdb5/types/TypeAbbreviation.cc +++ b/src/fdb5/types/TypeAbbreviation.cc @@ -18,24 +18,19 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -TypeAbbreviation::TypeAbbreviation(const std::string &name, const std::string &type) : - Type(name, type) { +TypeAbbreviation::TypeAbbreviation(const std::string& name, const std::string& type) : Type(name, type) { count_ = char(type[type.length() - 1]) - '0'; } -TypeAbbreviation::~TypeAbbreviation() { -} +TypeAbbreviation::~TypeAbbreviation() {} -std::string TypeAbbreviation::toKey(const std::string &value) const { +std::string TypeAbbreviation::toKey(const std::string& value) const { return value.substr(0, count_); } -void TypeAbbreviation::getValues(const metkit::mars::MarsRequest &request, - const std::string &keyword, - eckit::StringList &values, - const Notifier&, - const CatalogueReader*) const { +void TypeAbbreviation::getValues(const metkit::mars::MarsRequest& request, const std::string& keyword, + eckit::StringList& values, const Notifier&, const CatalogueReader*) const { std::vector vals; request.getValues(keyword, vals, true); @@ -47,7 +42,7 @@ void TypeAbbreviation::getValues(const metkit::mars::MarsRequest &request, } } -void TypeAbbreviation::print(std::ostream &out) const { +void TypeAbbreviation::print(std::ostream& out) const { out << "TypeAbbreviation[name=" << name_ << ",count=" << count_ << "]"; } diff --git a/src/fdb5/types/TypeAbbreviation.h b/src/fdb5/types/TypeAbbreviation.h index 6da0be41e..2f9aeee9c 100644 --- a/src/fdb5/types/TypeAbbreviation.h +++ b/src/fdb5/types/TypeAbbreviation.h @@ -25,24 +25,18 @@ namespace fdb5 { class TypeAbbreviation : public Type { public: // methods - - TypeAbbreviation(const std::string &name, const std::string &type); + TypeAbbreviation(const std::string& name, const std::string& type); ~TypeAbbreviation() override; std::string toKey(const std::string& value) const override; - virtual void getValues(const metkit::mars::MarsRequest &request, - const std::string &keyword, - eckit::StringList &values, - const Notifier &wind, - const CatalogueReader* cat) const override; + virtual void getValues(const metkit::mars::MarsRequest& request, const std::string& keyword, + eckit::StringList& values, const Notifier& wind, const CatalogueReader* cat) const override; private: // methods - - void print( std::ostream &out ) const override; + void print(std::ostream& out) const override; size_t count_; - }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/types/TypeClimateDaily.cc b/src/fdb5/types/TypeClimateDaily.cc index 22faf5d59..86ce4c06d 100644 --- a/src/fdb5/types/TypeClimateDaily.cc +++ b/src/fdb5/types/TypeClimateDaily.cc @@ -9,92 +9,81 @@ */ #include "eckit/exception/Exceptions.h" -#include "eckit/utils/Translator.h" -#include "eckit/utils/Tokenizer.h" #include "eckit/types/Date.h" +#include "eckit/utils/Tokenizer.h" +#include "eckit/utils/Translator.h" #include "metkit/mars/MarsRequest.h" -#include "fdb5/types/TypesFactory.h" #include "fdb5/types/TypeClimateDaily.h" +#include "fdb5/types/TypesFactory.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -static const char *months[] = { - "jan", "feb", "mar", "apr", "may", "jun", - "jul", "aug", "sep", "oct", "nov", "dec", +static const char* months[] = { + "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec", }; -TypeClimateDaily::TypeClimateDaily(const std::string &name, const std::string &type) : - Type(name, type) { -} - -TypeClimateDaily::~TypeClimateDaily() { -} +TypeClimateDaily::TypeClimateDaily(const std::string& name, const std::string& type) : Type(name, type) {} -static int month(const std::string &value) { - if (isdigit(value[0])) { - eckit::Date date(value); - return date.month() * 100 + date.day(); - } else { +TypeClimateDaily::~TypeClimateDaily() {} - eckit::Translator t; - eckit::Tokenizer parse("-"); - eckit::StringList v; +static int month(const std::string& value) { + if (isdigit(value[0])) { + eckit::Date date(value); + return date.month() * 100 + date.day(); + } else { - parse(value, v); - ASSERT(v.size() == 2); + eckit::Translator t; + eckit::Tokenizer parse("-"); + eckit::StringList v; + parse(value, v); + ASSERT(v.size() == 2); + for (int i = 0; i < 12; i++) { + if (v[0] == months[i]) { + return (i + 1) * 100 + t(v[1]); + } + } - for (int i = 0; i < 12 ; i++ ) { - if (v[0] == months[i]) { - return (i + 1) * 100 + t(v[1]); - } + throw eckit::SeriousBug("TypeClimateDaily: invalid date: " + value); } - - throw eckit::SeriousBug("TypeClimateDaily: invalid date: " + value); - } } -std::string TypeClimateDaily::toKey(const std::string &value) const { - std::ostringstream out; - char prev = out.fill ('0'); - out.width(4); - out << month(value); - out.fill(prev); - return out.str(); +std::string TypeClimateDaily::toKey(const std::string& value) const { + std::ostringstream out; + char prev = out.fill('0'); + out.width(4); + out << month(value); + out.fill(prev); + return out.str(); } -void TypeClimateDaily::getValues(const metkit::mars::MarsRequest &request, - const std::string &keyword, - eckit::StringList &values, - const Notifier&, - const CatalogueReader*) const { - std::vector dates; +void TypeClimateDaily::getValues(const metkit::mars::MarsRequest& request, const std::string& keyword, + eckit::StringList& values, const Notifier&, const CatalogueReader*) const { + std::vector dates; - request.getValues(keyword, dates, true); + request.getValues(keyword, dates, true); - values.reserve(dates.size()); + values.reserve(dates.size()); - eckit::Translator t; + eckit::Translator t; - for (std::vector::const_iterator i = dates.begin(); i != dates.end(); ++i) { - values.push_back(t(*i)); - } + for (std::vector::const_iterator i = dates.begin(); i != dates.end(); ++i) { + values.push_back(t(*i)); + } } -bool TypeClimateDaily::match(const std::string&, - const std::string& value1, - const std::string& value2) const { +bool TypeClimateDaily::match(const std::string&, const std::string& value1, const std::string& value2) const { - return month(value1) == month(value2); + return month(value1) == month(value2); } -void TypeClimateDaily::print(std::ostream &out) const { - out << "TypeClimateDaily[name=" << name_ << "]"; +void TypeClimateDaily::print(std::ostream& out) const { + out << "TypeClimateDaily[name=" << name_ << "]"; } static TypeBuilder type("ClimateDaily"); diff --git a/src/fdb5/types/TypeClimateDaily.h b/src/fdb5/types/TypeClimateDaily.h index c04783ea7..dc5e4209d 100644 --- a/src/fdb5/types/TypeClimateDaily.h +++ b/src/fdb5/types/TypeClimateDaily.h @@ -25,27 +25,19 @@ namespace fdb5 { class TypeClimateDaily : public Type { public: // methods - - TypeClimateDaily(const std::string &name, const std::string &type); + TypeClimateDaily(const std::string& name, const std::string& type); ~TypeClimateDaily() override; std::string toKey(const std::string& value) const override; - virtual void getValues(const metkit::mars::MarsRequest &request, - const std::string &keyword, - eckit::StringList &values, - const Notifier &wind, - const CatalogueReader* cat) const override; + virtual void getValues(const metkit::mars::MarsRequest& request, const std::string& keyword, + eckit::StringList& values, const Notifier& wind, const CatalogueReader* cat) const override; - virtual bool match(const std::string&, - const std::string& value1, - const std::string& value2) const override; + virtual bool match(const std::string&, const std::string& value1, const std::string& value2) const override; private: // methods - - void print( std::ostream &out ) const override; - + void print(std::ostream& out) const override; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/types/TypeClimateMonthly.cc b/src/fdb5/types/TypeClimateMonthly.cc index 34adca901..4cab8f063 100644 --- a/src/fdb5/types/TypeClimateMonthly.cc +++ b/src/fdb5/types/TypeClimateMonthly.cc @@ -8,41 +8,35 @@ * does it submit to any jurisdiction. */ - #include "eckit/types/Date.h" #include "metkit/mars/MarsRequest.h" -#include "fdb5/types/TypesFactory.h" #include "fdb5/types/TypeClimateMonthly.h" - +#include "fdb5/types/TypesFactory.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -static const char *months[] = { - "jan", "feb", "mar", "apr", "may", "jun", - "jul", "aug", "sep", "oct", "nov", "dec", +static const char* months[] = { + "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec", }; -TypeClimateMonthly::TypeClimateMonthly(const std::string &name, const std::string &type) : - Type(name, type) { -} +TypeClimateMonthly::TypeClimateMonthly(const std::string& name, const std::string& type) : Type(name, type) {} -TypeClimateMonthly::~TypeClimateMonthly() { -} +TypeClimateMonthly::~TypeClimateMonthly() {} -static int month(const std::string &value) { +static int month(const std::string& value) { if (isdigit(value[0])) { int n = stoi(value); - if (n<=12) { + if (n <= 12) { return n; } eckit::Date date(n); return date.month(); } else { - for (int i = 0; i < 12 ; i++ ) { + for (int i = 0; i < 12; i++) { if (value == months[i]) { return i + 1; } @@ -52,16 +46,13 @@ static int month(const std::string &value) { } } -std::string TypeClimateMonthly::toKey( const std::string &value) const { +std::string TypeClimateMonthly::toKey(const std::string& value) const { return std::to_string(month(value)); } -void TypeClimateMonthly::getValues(const metkit::mars::MarsRequest &request, - const std::string &keyword, - eckit::StringList &values, - const Notifier&, - const CatalogueReader*) const { +void TypeClimateMonthly::getValues(const metkit::mars::MarsRequest& request, const std::string& keyword, + eckit::StringList& values, const Notifier&, const CatalogueReader*) const { std::vector dates; request.getValues(keyword, dates, true); @@ -73,7 +64,7 @@ void TypeClimateMonthly::getValues(const metkit::mars::MarsRequest &request, } } -void TypeClimateMonthly::print(std::ostream &out) const { +void TypeClimateMonthly::print(std::ostream& out) const { out << "TypeClimateMonthly[name=" << name_ << "]"; } diff --git a/src/fdb5/types/TypeClimateMonthly.h b/src/fdb5/types/TypeClimateMonthly.h index 4358ed9b5..2c028f947 100644 --- a/src/fdb5/types/TypeClimateMonthly.h +++ b/src/fdb5/types/TypeClimateMonthly.h @@ -25,23 +25,17 @@ namespace fdb5 { class TypeClimateMonthly : public Type { public: // methods - - TypeClimateMonthly(const std::string &name, const std::string &type); + TypeClimateMonthly(const std::string& name, const std::string& type); ~TypeClimateMonthly() override; std::string toKey(const std::string& value) const override; - virtual void getValues(const metkit::mars::MarsRequest &request, - const std::string &keyword, - eckit::StringList &values, - const Notifier &wind, - const CatalogueReader* cat) const override; + virtual void getValues(const metkit::mars::MarsRequest& request, const std::string& keyword, + eckit::StringList& values, const Notifier& wind, const CatalogueReader* cat) const override; private: // methods - - void print( std::ostream &out ) const override; - + void print(std::ostream& out) const override; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/types/TypeDate.cc b/src/fdb5/types/TypeDate.cc index 0889f5c6a..881bfab36 100644 --- a/src/fdb5/types/TypeDate.cc +++ b/src/fdb5/types/TypeDate.cc @@ -13,22 +13,18 @@ #include "eckit/types/Date.h" #include "metkit/mars/MarsRequest.h" -#include "fdb5/types/TypesFactory.h" #include "fdb5/types/TypeDate.h" +#include "fdb5/types/TypesFactory.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -TypeDate::TypeDate(const std::string &name, const std::string &type) : - Type(name, type) { -} - -TypeDate::~TypeDate() { -} +TypeDate::TypeDate(const std::string& name, const std::string& type) : Type(name, type) {} +TypeDate::~TypeDate() {} -std::string TypeDate::tidy(const std::string &value) const { +std::string TypeDate::tidy(const std::string& value) const { if (!value.empty() && (value[0] == '0' || value[0] == '-')) { eckit::Translator t; @@ -40,14 +36,10 @@ std::string TypeDate::tidy(const std::string &value) const { } } return value; - } -void TypeDate::getValues(const metkit::mars::MarsRequest & request, - const std::string & keyword, - eckit::StringList & values, - const Notifier&, - const CatalogueReader*) const { +void TypeDate::getValues(const metkit::mars::MarsRequest& request, const std::string& keyword, + eckit::StringList& values, const Notifier&, const CatalogueReader*) const { std::vector dates; request.getValues(keyword, dates, true); @@ -61,7 +53,7 @@ void TypeDate::getValues(const metkit::mars::MarsRequest & request, } } -void TypeDate::print(std::ostream & out) const { +void TypeDate::print(std::ostream& out) const { out << "TypeDate[name=" << name_ << "]"; } diff --git a/src/fdb5/types/TypeDate.h b/src/fdb5/types/TypeDate.h index ef031fdde..67e0f131c 100644 --- a/src/fdb5/types/TypeDate.h +++ b/src/fdb5/types/TypeDate.h @@ -25,23 +25,17 @@ namespace fdb5 { class TypeDate : public Type { public: // methods - - TypeDate(const std::string &name, const std::string &type); + TypeDate(const std::string& name, const std::string& type); ~TypeDate() override; - std::string tidy(const std::string &value) const override; + std::string tidy(const std::string& value) const override; - virtual void getValues(const metkit::mars::MarsRequest &request, - const std::string &keyword, - eckit::StringList &values, - const Notifier &wind, - const CatalogueReader* cat) const override; + virtual void getValues(const metkit::mars::MarsRequest& request, const std::string& keyword, + eckit::StringList& values, const Notifier& wind, const CatalogueReader* cat) const override; private: // methods - - void print( std::ostream &out ) const override; - + void print(std::ostream& out) const override; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/types/TypeDefault.cc b/src/fdb5/types/TypeDefault.cc index 6e09b5d4a..d8308692d 100644 --- a/src/fdb5/types/TypeDefault.cc +++ b/src/fdb5/types/TypeDefault.cc @@ -8,21 +8,18 @@ * does it submit to any jurisdiction. */ -#include "fdb5/types/TypesFactory.h" #include "fdb5/types/TypeDefault.h" +#include "fdb5/types/TypesFactory.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -TypeDefault::TypeDefault(const std::string &name, const std::string &type) : - Type(name, type) { -} +TypeDefault::TypeDefault(const std::string& name, const std::string& type) : Type(name, type) {} -TypeDefault::~TypeDefault() { -} +TypeDefault::~TypeDefault() {} -void TypeDefault::print(std::ostream &out) const { +void TypeDefault::print(std::ostream& out) const { out << "TypeDefault[name=" << name_ << "]"; } diff --git a/src/fdb5/types/TypeDefault.h b/src/fdb5/types/TypeDefault.h index d3f19043e..784e862a5 100644 --- a/src/fdb5/types/TypeDefault.h +++ b/src/fdb5/types/TypeDefault.h @@ -25,15 +25,12 @@ namespace fdb5 { class TypeDefault : public Type { public: // methods - - TypeDefault(const std::string &name, const std::string &type); + TypeDefault(const std::string& name, const std::string& type); ~TypeDefault() override; private: // methods - - void print( std::ostream &out ) const override; - + void print(std::ostream& out) const override; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/types/TypeDouble.cc b/src/fdb5/types/TypeDouble.cc index ab97fbcb2..540f6fba4 100644 --- a/src/fdb5/types/TypeDouble.cc +++ b/src/fdb5/types/TypeDouble.cc @@ -12,51 +12,45 @@ #include "metkit/mars/MarsRequest.h" -#include "fdb5/types/TypesFactory.h" #include "fdb5/types/TypeDouble.h" +#include "fdb5/types/TypesFactory.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -TypeDouble::TypeDouble(const std::string &name, const std::string &type) : - Type(name, type) { -} +TypeDouble::TypeDouble(const std::string& name, const std::string& type) : Type(name, type) {} -TypeDouble::~TypeDouble() { -} +TypeDouble::~TypeDouble() {} std::string TypeDouble::toKey(const std::string& value) const { - double v = eckit::Translator()(value); - long long ll = static_cast(v); - - if (ll == v) { - return eckit::Translator()(ll); - } else { - return eckit::Translator()(v); - } + double v = eckit::Translator()(value); + long long ll = static_cast(v); + + if (ll == v) { + return eckit::Translator()(ll); + } else { + return eckit::Translator()(v); + } } -void TypeDouble::getValues(const metkit::mars::MarsRequest& request, - const std::string& keyword, - eckit::StringList& values, - const Notifier&, - const CatalogueReader*) const { - std::vector dblValues; +void TypeDouble::getValues(const metkit::mars::MarsRequest& request, const std::string& keyword, + eckit::StringList& values, const Notifier&, const CatalogueReader*) const { + std::vector dblValues; - request.getValues(keyword, dblValues, true); + request.getValues(keyword, dblValues, true); - eckit::Translator t; + eckit::Translator t; - values.reserve(dblValues.size()); + values.reserve(dblValues.size()); - for (std::vector::const_iterator i = dblValues.begin(); i != dblValues.end(); ++i) { - values.push_back(t(*i)); - } + for (std::vector::const_iterator i = dblValues.begin(); i != dblValues.end(); ++i) { + values.push_back(t(*i)); + } } -void TypeDouble::print(std::ostream &out) const { - out << "TypeDouble[name=" << name_ << "]"; +void TypeDouble::print(std::ostream& out) const { + out << "TypeDouble[name=" << name_ << "]"; } static TypeBuilder type("Double"); diff --git a/src/fdb5/types/TypeDouble.h b/src/fdb5/types/TypeDouble.h index 8135eca24..9c27a72e2 100644 --- a/src/fdb5/types/TypeDouble.h +++ b/src/fdb5/types/TypeDouble.h @@ -25,23 +25,17 @@ namespace fdb5 { class TypeDouble : public Type { public: // methods - - TypeDouble(const std::string &name, const std::string &type); + TypeDouble(const std::string& name, const std::string& type); ~TypeDouble() override; std::string toKey(const std::string& value) const override; - virtual void getValues(const metkit::mars::MarsRequest &request, - const std::string &keyword, - eckit::StringList &values, - const Notifier &wind, - const CatalogueReader* cat) const override; + virtual void getValues(const metkit::mars::MarsRequest& request, const std::string& keyword, + eckit::StringList& values, const Notifier& wind, const CatalogueReader* cat) const override; private: // methods - - void print( std::ostream &out ) const override; - + void print(std::ostream& out) const override; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/types/TypeExpver.cc b/src/fdb5/types/TypeExpver.cc index b2747f82b..02d6f59dc 100644 --- a/src/fdb5/types/TypeExpver.cc +++ b/src/fdb5/types/TypeExpver.cc @@ -11,25 +11,21 @@ #include #include -#include "eckit/utils/Translator.h" #include "eckit/utils/StringTools.h" +#include "eckit/utils/Translator.h" #include "eckit/types/Date.h" -#include "fdb5/types/TypesFactory.h" #include "fdb5/types/TypeExpver.h" +#include "fdb5/types/TypesFactory.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -TypeExpver::TypeExpver(const std::string &name, const std::string &type) : - Type(name, type) { -} - -TypeExpver::~TypeExpver() { -} +TypeExpver::TypeExpver(const std::string& name, const std::string& type) : Type(name, type) {} +TypeExpver::~TypeExpver() {} std::string TypeExpver::tidy(const std::string& value) const { @@ -38,8 +34,7 @@ std::string TypeExpver::tidy(const std::string& value) const { return oss.str(); } - -void TypeExpver::print(std::ostream &out) const { +void TypeExpver::print(std::ostream& out) const { out << "TypeExpver[name=" << name_ << "]"; } diff --git a/src/fdb5/types/TypeExpver.h b/src/fdb5/types/TypeExpver.h index f50df3da7..127a0f510 100644 --- a/src/fdb5/types/TypeExpver.h +++ b/src/fdb5/types/TypeExpver.h @@ -25,17 +25,14 @@ namespace fdb5 { class TypeExpver : public Type { public: // methods - - TypeExpver(const std::string &name, const std::string &type); + TypeExpver(const std::string& name, const std::string& type); ~TypeExpver() override; - std::string tidy(const std::string &value) const override; + std::string tidy(const std::string& value) const override; private: // methods - - void print( std::ostream &out ) const override; - + void print(std::ostream& out) const override; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/types/TypeGrid.cc b/src/fdb5/types/TypeGrid.cc index 595edf9ea..37981e873 100644 --- a/src/fdb5/types/TypeGrid.cc +++ b/src/fdb5/types/TypeGrid.cc @@ -13,39 +13,32 @@ #include "eckit/utils/StringTools.h" #include "metkit/mars/MarsRequest.h" -#include "fdb5/types/TypesFactory.h" #include "fdb5/types/TypeGrid.h" +#include "fdb5/types/TypesFactory.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -TypeGrid::TypeGrid(const std::string &name, const std::string &type) : - Type(name, type) { -} +TypeGrid::TypeGrid(const std::string& name, const std::string& type) : Type(name, type) {} -TypeGrid::~TypeGrid() { -} +TypeGrid::~TypeGrid() {} std::string TypeGrid::toKey(const std::string& value) const { std::string s(value); - std::replace( s.begin(), s.end(), '/', '+'); + std::replace(s.begin(), s.end(), '/', '+'); return s; } -void TypeGrid::getValues(const metkit::mars::MarsRequest& request, - const std::string& keyword, - eckit::StringList& values, - const Notifier&, - const CatalogueReader*) const { +void TypeGrid::getValues(const metkit::mars::MarsRequest& request, const std::string& keyword, + eckit::StringList& values, const Notifier&, const CatalogueReader*) const { std::vector v; request.getValues(keyword, v, true); values.push_back(eckit::StringTools::join("+", v)); } - -void TypeGrid::print(std::ostream &out) const { +void TypeGrid::print(std::ostream& out) const { out << "TypeGrid[name=" << name_ << "]"; } diff --git a/src/fdb5/types/TypeGrid.h b/src/fdb5/types/TypeGrid.h index fe0e659ae..0798d5973 100644 --- a/src/fdb5/types/TypeGrid.h +++ b/src/fdb5/types/TypeGrid.h @@ -25,23 +25,17 @@ namespace fdb5 { class TypeGrid : public Type { public: // methods - - TypeGrid(const std::string &name, const std::string &type); + TypeGrid(const std::string& name, const std::string& type); ~TypeGrid() override; private: // methods - std::string toKey(const std::string& value) const override; - virtual void getValues(const metkit::mars::MarsRequest &request, - const std::string &keyword, - eckit::StringList &values, - const Notifier &wind, - const CatalogueReader* cat) const override; - - void print( std::ostream &out ) const override; + virtual void getValues(const metkit::mars::MarsRequest& request, const std::string& keyword, + eckit::StringList& values, const Notifier& wind, const CatalogueReader* cat) const override; + void print(std::ostream& out) const override; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/types/TypeIgnore.cc b/src/fdb5/types/TypeIgnore.cc index 8b52a9ca3..fa9b1b357 100644 --- a/src/fdb5/types/TypeIgnore.cc +++ b/src/fdb5/types/TypeIgnore.cc @@ -8,35 +8,25 @@ * does it submit to any jurisdiction. */ - -#include "fdb5/types/TypesFactory.h" #include "fdb5/types/TypeIgnore.h" - +#include "fdb5/types/TypesFactory.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -TypeIgnore::TypeIgnore(const std::string &name, const std::string &type) : - Type(name, type) { -} - -TypeIgnore::~TypeIgnore() { -} +TypeIgnore::TypeIgnore(const std::string& name, const std::string& type) : Type(name, type) {} +TypeIgnore::~TypeIgnore() {} std::string TypeIgnore::toKey(const std::string&) const { return ""; } -void TypeIgnore::getValues(const metkit::mars::MarsRequest&, - const std::string&, - eckit::StringList&, - const Notifier&, - const CatalogueReader*) const { -} +void TypeIgnore::getValues(const metkit::mars::MarsRequest&, const std::string&, eckit::StringList&, const Notifier&, + const CatalogueReader*) const {} -void TypeIgnore::print(std::ostream &out) const { +void TypeIgnore::print(std::ostream& out) const { out << "TypeIgnore[name=" << name_ << "]"; } diff --git a/src/fdb5/types/TypeIgnore.h b/src/fdb5/types/TypeIgnore.h index 4825263f4..a01df6b24 100644 --- a/src/fdb5/types/TypeIgnore.h +++ b/src/fdb5/types/TypeIgnore.h @@ -25,23 +25,17 @@ namespace fdb5 { class TypeIgnore : public Type { public: // methods - - TypeIgnore(const std::string &name, const std::string &type); + TypeIgnore(const std::string& name, const std::string& type); ~TypeIgnore() override; - virtual void getValues(const metkit::mars::MarsRequest &request, - const std::string &keyword, - eckit::StringList &values, - const Notifier &wind, - const CatalogueReader* cat) const override; + virtual void getValues(const metkit::mars::MarsRequest& request, const std::string& keyword, + eckit::StringList& values, const Notifier& wind, const CatalogueReader* cat) const override; std::string toKey(const std::string& value) const override; private: // methods - - void print( std::ostream &out ) const override; - + void print(std::ostream& out) const override; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/types/TypeInteger.cc b/src/fdb5/types/TypeInteger.cc index 17801f217..8f2cc32a5 100644 --- a/src/fdb5/types/TypeInteger.cc +++ b/src/fdb5/types/TypeInteger.cc @@ -12,25 +12,19 @@ #include "metkit/mars/MarsRequest.h" -#include "fdb5/types/TypesFactory.h" #include "fdb5/types/TypeInteger.h" +#include "fdb5/types/TypesFactory.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -TypeInteger::TypeInteger(const std::string &name, const std::string &type) : - Type(name, type) { -} +TypeInteger::TypeInteger(const std::string& name, const std::string& type) : Type(name, type) {} -TypeInteger::~TypeInteger() { -} +TypeInteger::~TypeInteger() {} -void TypeInteger::getValues(const metkit::mars::MarsRequest& request, - const std::string& keyword, - eckit::StringList& values, - const Notifier&, - const CatalogueReader*) const { +void TypeInteger::getValues(const metkit::mars::MarsRequest& request, const std::string& keyword, + eckit::StringList& values, const Notifier&, const CatalogueReader*) const { std::vector intValues; request.getValues(keyword, intValues, true); @@ -44,7 +38,7 @@ void TypeInteger::getValues(const metkit::mars::MarsRequest& request, } } -void TypeInteger::print(std::ostream &out) const { +void TypeInteger::print(std::ostream& out) const { out << "TypeInteger[name=" << name_ << "]"; } diff --git a/src/fdb5/types/TypeInteger.h b/src/fdb5/types/TypeInteger.h index e9040c01a..33bc64e2c 100644 --- a/src/fdb5/types/TypeInteger.h +++ b/src/fdb5/types/TypeInteger.h @@ -25,21 +25,15 @@ namespace fdb5 { class TypeInteger : public Type { public: // methods - - TypeInteger(const std::string &name, const std::string &type); + TypeInteger(const std::string& name, const std::string& type); ~TypeInteger() override; - virtual void getValues(const metkit::mars::MarsRequest &request, - const std::string &keyword, - eckit::StringList &values, - const Notifier &wind, - const CatalogueReader* cat) const override; + virtual void getValues(const metkit::mars::MarsRequest& request, const std::string& keyword, + eckit::StringList& values, const Notifier& wind, const CatalogueReader* cat) const override; private: // methods - - void print( std::ostream &out ) const override; - + void print(std::ostream& out) const override; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/types/TypeMonth.cc b/src/fdb5/types/TypeMonth.cc index 980c715b5..a1be16209 100644 --- a/src/fdb5/types/TypeMonth.cc +++ b/src/fdb5/types/TypeMonth.cc @@ -9,25 +9,21 @@ */ // #include "eckit/exception/Exceptions.h" -#include "eckit/utils/Translator.h" #include "eckit/types/Date.h" +#include "eckit/utils/Translator.h" #include "metkit/mars/MarsRequest.h" -#include "fdb5/types/TypesFactory.h" #include "fdb5/types/TypeMonth.h" - +#include "fdb5/types/TypesFactory.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -TypeMonth::TypeMonth(const std::string &name, const std::string &type) : - Type(name, type) { -} +TypeMonth::TypeMonth(const std::string& name, const std::string& type) : Type(name, type) {} -TypeMonth::~TypeMonth() { -} +TypeMonth::~TypeMonth() {} std::string TypeMonth::toKey(const std::string& value) const { @@ -35,11 +31,8 @@ std::string TypeMonth::toKey(const std::string& value) const { return std::to_string(date.year() * 100 + date.month()); } -void TypeMonth::getValues(const metkit::mars::MarsRequest& request, - const std::string& keyword, - eckit::StringList& values, - const Notifier&, - const CatalogueReader*) const { +void TypeMonth::getValues(const metkit::mars::MarsRequest& request, const std::string& keyword, + eckit::StringList& values, const Notifier&, const CatalogueReader*) const { std::vector dates; request.getValues(keyword, dates, true); @@ -49,12 +42,12 @@ void TypeMonth::getValues(const metkit::mars::MarsRequest& request, eckit::Translator t; for (std::vector::const_iterator i = dates.begin(); i != dates.end(); ++i) { - const eckit::Date &date = *i; + const eckit::Date& date = *i; values.push_back(t(date)); } } -void TypeMonth::print(std::ostream &out) const { +void TypeMonth::print(std::ostream& out) const { out << "TypeMonth[name=" << name_ << "]"; } diff --git a/src/fdb5/types/TypeMonth.h b/src/fdb5/types/TypeMonth.h index 0f5114d0c..c5e86441b 100644 --- a/src/fdb5/types/TypeMonth.h +++ b/src/fdb5/types/TypeMonth.h @@ -25,23 +25,17 @@ namespace fdb5 { class TypeMonth : public Type { public: // methods - - TypeMonth(const std::string &name, const std::string &type); + TypeMonth(const std::string& name, const std::string& type); ~TypeMonth() override; std::string toKey(const std::string& value) const override; - virtual void getValues(const metkit::mars::MarsRequest &request, - const std::string &keyword, - eckit::StringList &values, - const Notifier &wind, - const CatalogueReader* cat) const override; + virtual void getValues(const metkit::mars::MarsRequest& request, const std::string& keyword, + eckit::StringList& values, const Notifier& wind, const CatalogueReader* cat) const override; private: // methods - - void print( std::ostream &out ) const override; - + void print(std::ostream& out) const override; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/types/TypeParam.cc b/src/fdb5/types/TypeParam.cc index ea87bae71..d29ab82c0 100644 --- a/src/fdb5/types/TypeParam.cc +++ b/src/fdb5/types/TypeParam.cc @@ -10,8 +10,8 @@ #include -#include "metkit/mars/ParamID.h" #include "metkit/mars/Param.h" +#include "metkit/mars/ParamID.h" #include "fdb5/database/Catalogue.h" #include "fdb5/database/Notifier.h" @@ -24,18 +24,12 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -TypeParam::TypeParam(const std::string &name, const std::string &type) : - Type(name, type) { -} +TypeParam::TypeParam(const std::string& name, const std::string& type) : Type(name, type) {} -TypeParam::~TypeParam() { -} +TypeParam::~TypeParam() {} -void TypeParam::getValues(const metkit::mars::MarsRequest &request, - const std::string &keyword, - eckit::StringList &values, - const Notifier &wind, - const CatalogueReader* cat) const { +void TypeParam::getValues(const metkit::mars::MarsRequest& request, const std::string& keyword, + eckit::StringList& values, const Notifier& wind, const CatalogueReader* cat) const { ASSERT(cat); eckit::StringSet ax; @@ -78,34 +72,30 @@ void TypeParam::getValues(const metkit::mars::MarsRequest &request, } } -bool TypeParam::match(const std::string&, - const std::string& value1, - const std::string& value2) const { - if(value1 == value2) { +bool TypeParam::match(const std::string&, const std::string& value1, const std::string& value2) const { + if (value1 == value2) { return true; } Param p1(value1); Param p2(value2); - if((p1.value() == p2.value()) && (p1.table() == 0 || p2.table() == 0)) { + if ((p1.value() == p2.value()) && (p1.table() == 0 || p2.table() == 0)) { return true; } - if(p1.table() * 1000 + p1.value() == p2.value()) - { + if (p1.table() * 1000 + p1.value() == p2.value()) { return true; } - if(p2.table() * 1000 + p2.value() == p1.value()) - { + if (p2.table() * 1000 + p2.value() == p1.value()) { return true; } return false; } -void TypeParam::print(std::ostream &out) const { +void TypeParam::print(std::ostream& out) const { out << "TypeParam[name=" << name_ << "]"; } diff --git a/src/fdb5/types/TypeParam.h b/src/fdb5/types/TypeParam.h index f946ba9f5..13434e709 100644 --- a/src/fdb5/types/TypeParam.h +++ b/src/fdb5/types/TypeParam.h @@ -25,25 +25,17 @@ namespace fdb5 { class TypeParam : public Type { public: // methods - - TypeParam(const std::string &name, const std::string &type); + TypeParam(const std::string& name, const std::string& type); ~TypeParam() override; - virtual void getValues(const metkit::mars::MarsRequest &request, - const std::string &keyword, - eckit::StringList &values, - const Notifier &wind, - const CatalogueReader* cat) const override; + virtual void getValues(const metkit::mars::MarsRequest& request, const std::string& keyword, + eckit::StringList& values, const Notifier& wind, const CatalogueReader* cat) const override; - virtual bool match(const std::string& keyword, - const std::string& value1, - const std::string& value2) const override; + virtual bool match(const std::string& keyword, const std::string& value1, const std::string& value2) const override; private: // methods - - void print( std::ostream &out ) const override; - + void print(std::ostream& out) const override; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/types/TypeStep.cc b/src/fdb5/types/TypeStep.cc index 1b6a0c017..a39abd394 100644 --- a/src/fdb5/types/TypeStep.cc +++ b/src/fdb5/types/TypeStep.cc @@ -14,48 +14,45 @@ #include "metkit/mars/StepRange.h" #include "metkit/mars/StepRangeNormalise.h" -#include "fdb5/types/TypesFactory.h" -#include "fdb5/types/TypeStep.h" #include "fdb5/database/Catalogue.h" +#include "fdb5/types/TypeStep.h" +#include "fdb5/types/TypesFactory.h" using metkit::mars::StepRange; using metkit::mars::StepRangeNormalise; - namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -TypeStep::TypeStep(const std::string &name, const std::string &type) : - Type(name, type) { -} - -TypeStep::~TypeStep() { -} +TypeStep::TypeStep(const std::string& name, const std::string& type) : Type(name, type) {} +TypeStep::~TypeStep() {} std::string TypeStep::toKey(const std::string& value) const { return StepRange(value); } -bool TypeStep::match(const std::string&, const std::string& value1, const std::string& value2) const -{ - if(value1 == value2) { return true; } +bool TypeStep::match(const std::string&, const std::string& value1, const std::string& value2) const { + if (value1 == value2) { + return true; + } std::string z1 = "0-" + value1; - if(z1 == value2) { return true; } + if (z1 == value2) { + return true; + } std::string z2 = "0-" + value2; - if(z2 == value1) { return true; } + if (z2 == value1) { + return true; + } return false; } -void TypeStep::getValues(const metkit::mars::MarsRequest& request, - const std::string& keyword, - eckit::StringList& values, - const Notifier&, - const CatalogueReader* cat) const { +void TypeStep::getValues(const metkit::mars::MarsRequest& request, const std::string& keyword, + eckit::StringList& values, const Notifier&, const CatalogueReader* cat) const { // Get the steps / step ranges from the request @@ -75,7 +72,7 @@ void TypeStep::getValues(const metkit::mars::MarsRequest& request, cat->axis("step", ax); std::vector axis; - for (auto step: ax) { + for (auto step : ax) { if (!step.empty()) { axis.push_back(StepRange(step)); } @@ -92,11 +89,10 @@ void TypeStep::getValues(const metkit::mars::MarsRequest& request, eckit::Translator t; values.reserve(ranges.size()); - std::transform(ranges.begin(), ranges.end(), std::back_inserter(values), - [&](const StepRange& r) { return t(r); }); + std::transform(ranges.begin(), ranges.end(), std::back_inserter(values), [&](const StepRange& r) { return t(r); }); } -void TypeStep::print(std::ostream &out) const { +void TypeStep::print(std::ostream& out) const { out << "TypeStep[name=" << name_ << "]"; } diff --git a/src/fdb5/types/TypeStep.h b/src/fdb5/types/TypeStep.h index 91fde3607..d306c691a 100644 --- a/src/fdb5/types/TypeStep.h +++ b/src/fdb5/types/TypeStep.h @@ -25,25 +25,19 @@ namespace fdb5 { class TypeStep : public Type { public: // methods - - TypeStep(const std::string &name, const std::string &type); + TypeStep(const std::string& name, const std::string& type); ~TypeStep() override; - virtual void getValues(const metkit::mars::MarsRequest &request, - const std::string &keyword, - eckit::StringList &values, - const Notifier &wind, - const CatalogueReader* cat) const override; + virtual void getValues(const metkit::mars::MarsRequest& request, const std::string& keyword, + eckit::StringList& values, const Notifier& wind, const CatalogueReader* cat) const override; std::string toKey(const std::string& value) const override; bool match(const std::string& keyword, const std::string& value1, const std::string& value2) const override; private: // methods - - void print( std::ostream &out ) const override; - + void print(std::ostream& out) const override; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/types/TypeTime.cc b/src/fdb5/types/TypeTime.cc index d3dc7dc34..fb507f271 100644 --- a/src/fdb5/types/TypeTime.cc +++ b/src/fdb5/types/TypeTime.cc @@ -15,19 +15,16 @@ #include "eckit/types/Date.h" #include "metkit/mars/MarsRequest.h" -#include "fdb5/types/TypesFactory.h" #include "fdb5/types/TypeTime.h" +#include "fdb5/types/TypesFactory.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -TypeTime::TypeTime(const std::string &name, const std::string &type) : - Type(name, type) { -} +TypeTime::TypeTime(const std::string& name, const std::string& type) : Type(name, type) {} -TypeTime::~TypeTime() { -} +TypeTime::~TypeTime() {} std::string TypeTime::tidy(const std::string& value) const { @@ -37,7 +34,7 @@ std::string TypeTime::tidy(const std::string& value) const { std::string TypeTime::toKey(const std::string& value) const { // if value just contains a digit, add a leading zero to be compliant with eckit::Time - std::string t = value.size() < 2 ? "0"+value : value; + std::string t = value.size() < 2 ? "0" + value : value; eckit::Time time(t); std::ostringstream oss; @@ -45,7 +42,7 @@ std::string TypeTime::toKey(const std::string& value) const { return oss.str(); } -void TypeTime::print(std::ostream &out) const { +void TypeTime::print(std::ostream& out) const { out << "TypeTime[name=" << name_ << "]"; } diff --git a/src/fdb5/types/TypeTime.h b/src/fdb5/types/TypeTime.h index 3c541151b..b0b4e7dd2 100644 --- a/src/fdb5/types/TypeTime.h +++ b/src/fdb5/types/TypeTime.h @@ -25,19 +25,16 @@ namespace fdb5 { class TypeTime : public Type { public: // methods - - TypeTime(const std::string &name, const std::string &type); + TypeTime(const std::string& name, const std::string& type); ~TypeTime() override; - std::string tidy(const std::string &value) const override; - - std::string toKey(const std::string &value) const override; - -private: // methods + std::string tidy(const std::string& value) const override; - void print( std::ostream &out ) const override; + std::string toKey(const std::string& value) const override; +private: // methods + void print(std::ostream& out) const override; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/types/TypesFactory.cc b/src/fdb5/types/TypesFactory.cc index 8fd56a578..fa9ae5029 100644 --- a/src/fdb5/types/TypesFactory.cc +++ b/src/fdb5/types/TypesFactory.cc @@ -8,8 +8,8 @@ * does it submit to any jurisdiction. */ -#include "eckit/thread/AutoLock.h" #include "eckit/exception/Exceptions.h" +#include "eckit/thread/AutoLock.h" #include "fdb5/LibFdb5.h" #include "fdb5/types/TypesFactory.h" @@ -18,18 +18,17 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -static eckit::Mutex *local_mutex = 0; -static std::map *m = 0; +static eckit::Mutex* local_mutex = 0; +static std::map* m = 0; static pthread_once_t once = PTHREAD_ONCE_INIT; static void init() { local_mutex = new eckit::Mutex(); - m = new std::map(); + m = new std::map(); } //---------------------------------------------------------------------------------------------------------------------- -TypesFactory::TypesFactory(const std::string &name) : - name_(name) { +TypesFactory::TypesFactory(const std::string& name) : name_(name) { pthread_once(&once, init); eckit::AutoLock lock(local_mutex); @@ -38,21 +37,22 @@ TypesFactory::TypesFactory(const std::string &name) : } TypesFactory::~TypesFactory() { - if(LibFdb5::instance().dontDeregisterFactories()) return; + if (LibFdb5::instance().dontDeregisterFactories()) + return; eckit::AutoLock lock(local_mutex); m->erase(name_); } -Type *TypesFactory::build(const std::string &name, const std::string &keyword) { +Type* TypesFactory::build(const std::string& name, const std::string& keyword) { pthread_once(&once, init); eckit::AutoLock lock(local_mutex); - std::map::const_iterator j = m->find(name); + std::map::const_iterator j = m->find(name); if (j == m->end()) { eckit::Log::error() << "No TypesFactory for [" << name << "]" << std::endl; eckit::Log::error() << "KeywordTypes are:" << std::endl; - for (j = m->begin() ; j != m->end() ; ++j) + for (j = m->begin(); j != m->end(); ++j) eckit::Log::error() << " " << (*j).first << std::endl; throw eckit::SeriousBug(std::string("No TypesFactory called ") + name); } diff --git a/src/fdb5/types/TypesFactory.h b/src/fdb5/types/TypesFactory.h index 44f27341c..644e141ed 100644 --- a/src/fdb5/types/TypesFactory.h +++ b/src/fdb5/types/TypesFactory.h @@ -31,34 +31,29 @@ class Type; class TypesFactory { - virtual Type *make(const std::string &keyword) const = 0 ; + virtual Type* make(const std::string& keyword) const = 0; protected: - - TypesFactory(const std::string &); + TypesFactory(const std::string&); virtual ~TypesFactory(); std::string name_; public: - - static void list(std::ostream &); - static Type *build(const std::string &name, const std::string &keyword); - + static void list(std::ostream&); + static Type* build(const std::string& name, const std::string& keyword); }; /// Templated specialisation of the self-registering factory, /// that does the self-registration, and the construction of each object. -template< class T> +template class TypeBuilder : public TypesFactory { - Type *make(const std::string &keyword) const override { - return new T(keyword, name_); - } + Type* make(const std::string& keyword) const override { return new T(keyword, name_); } public: - TypeBuilder(const std::string &name) : TypesFactory(name) {} + TypeBuilder(const std::string& name) : TypesFactory(name) {} }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/types/TypesRegistry.cc b/src/fdb5/types/TypesRegistry.cc index 30670bbba..69c69c6e8 100644 --- a/src/fdb5/types/TypesRegistry.cc +++ b/src/fdb5/types/TypesRegistry.cc @@ -1,4 +1,4 @@ - /* +/* * (C) Copyright 1996- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 @@ -20,11 +20,14 @@ #include "metkit/mars/MarsRequest.h" #include "metkit/mars/Parameter.h" - namespace fdb5 { +namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -eckit::ClassSpec TypesRegistry::classSpec_ = { &eckit::Streamable::classSpec(), "TypesRegistry", }; +eckit::ClassSpec TypesRegistry::classSpec_ = { + &eckit::Streamable::classSpec(), + "TypesRegistry", +}; eckit::Reanimator TypesRegistry::reanimator_; @@ -37,7 +40,7 @@ TypesRegistry::TypesRegistry(eckit::Stream& s) { std::string type; s >> numTypes; - for (size_t i=0; i> name; s >> type; types_[name] = type; @@ -47,29 +50,30 @@ TypesRegistry::TypesRegistry(eckit::Stream& s) { void TypesRegistry::encode(eckit::Stream& s) const { s << types_.size(); - for (auto t: types_) { + for (auto t : types_) { s << t.first; s << t.second; } } - TypesRegistry::~TypesRegistry() { - for (auto& item : cache_) { delete item.second; } + for (auto& item : cache_) { + delete item.second; + } } void TypesRegistry::updateParent(const TypesRegistry& parent) { parent_ = std::cref(parent); } -void TypesRegistry::addType(const std::string &keyword, const std::string &type) { +void TypesRegistry::addType(const std::string& keyword, const std::string& type) { ASSERT(types_.find(keyword) == types_.end()); types_[keyword] = type; } -const Type &TypesRegistry::lookupType(const std::string &keyword) const { +const Type& TypesRegistry::lookupType(const std::string& keyword) const { - std::map::const_iterator j = cache_.find(keyword); + std::map::const_iterator j = cache_.find(keyword); if (j != cache_.end()) { return *(*j).second; @@ -106,23 +110,22 @@ metkit::mars::MarsRequest TypesRegistry::canonicalise(const metkit::mars::MarsRe return result; } -std::ostream &operator<<(std::ostream &s, const TypesRegistry &x) { +std::ostream& operator<<(std::ostream& s, const TypesRegistry& x) { x.print(s); return s; } -void TypesRegistry::print( std::ostream &out ) const { +void TypesRegistry::print(std::ostream& out) const { out << this << "(" << types_ << ")"; } -void TypesRegistry::dump( std::ostream &out ) const { +void TypesRegistry::dump(std::ostream& out) const { for (std::map::const_iterator i = types_.begin(); i != types_.end(); ++i) { out << i->first << ":" << i->second << ";" << std::endl; } } - -void TypesRegistry::dump( std::ostream &out, const std::string &keyword ) const { +void TypesRegistry::dump(std::ostream& out, const std::string& keyword) const { std::map::const_iterator i = types_.find(keyword); out << keyword; @@ -133,4 +136,4 @@ void TypesRegistry::dump( std::ostream &out, const std::string &keyword ) const //---------------------------------------------------------------------------------------------------------------------- -} // namespace fdb5 +} // namespace fdb5 diff --git a/src/fdb5/types/TypesRegistry.h b/src/fdb5/types/TypesRegistry.h index 057b9f7e9..abe49fb79 100644 --- a/src/fdb5/types/TypesRegistry.h +++ b/src/fdb5/types/TypesRegistry.h @@ -16,11 +16,11 @@ #ifndef fdb5_TypesRegistry_H #define fdb5_TypesRegistry_H -#include +#include #include #include #include -#include +#include #include "eckit/serialisation/Streamable.h" @@ -34,45 +34,41 @@ class Type; //---------------------------------------------------------------------------------------------------------------------- -class TypesRegistry : public eckit::Streamable { +class TypesRegistry : public eckit::Streamable { public: // methods - TypesRegistry(); explicit TypesRegistry(eckit::Stream& s); ~TypesRegistry() override; - const Type &lookupType(const std::string &keyword) const; + const Type& lookupType(const std::string& keyword) const; - void addType(const std::string &, const std::string &); + void addType(const std::string&, const std::string&); void updateParent(const TypesRegistry& parent); - void dump( std::ostream &out ) const; - void dump( std::ostream &out, const std::string &keyword ) const; + void dump(std::ostream& out) const; + void dump(std::ostream& out, const std::string& keyword) const; metkit::mars::MarsRequest canonicalise(const metkit::mars::MarsRequest& request) const; - const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } - static const eckit::ClassSpec& classSpec() { return classSpec_; } + const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } + static const eckit::ClassSpec& classSpec() { return classSpec_; } void encode(eckit::Stream& s) const override; private: // members - static eckit::ClassSpec classSpec_; static eckit::Reanimator reanimator_; - typedef std::map TypeMap; + typedef std::map TypeMap; mutable TypeMap cache_; std::map types_; std::optional> parent_; - friend std::ostream &operator<<(std::ostream &s, const TypesRegistry &x); - - - void print( std::ostream &out ) const; + friend std::ostream& operator<<(std::ostream& s, const TypesRegistry& x); + void print(std::ostream& out) const; }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/tests/fdb/api/ApiSpy.h b/tests/fdb/api/ApiSpy.h index 40a160b62..0e4f16c7c 100644 --- a/tests/fdb/api/ApiSpy.h +++ b/tests/fdb/api/ApiSpy.h @@ -18,13 +18,13 @@ #ifndef fdb_testing_ApiSpy_H #define fdb_testing_ApiSpy_H -#include #include +#include #include "eckit/message/Message.h" -#include "fdb5/api/FDBFactory.h" #include "fdb5/api/FDB.h" +#include "fdb5/api/FDBFactory.h" #include "metkit/mars/MarsRequest.h" @@ -36,11 +36,20 @@ namespace test { class ApiSpy : public fdb5::FDBBase { private: // types - struct Counts { - Counts() : - archive(0), inspect(0), list(0), axes(0), dump(0), status(0), wipe(0), - purge(0), stats(0), flush(0), control(0), move(0) {} + Counts() + : archive(0) + , inspect(0) + , list(0) + , axes(0) + , dump(0) + , status(0) + , wipe(0) + , purge(0) + , stats(0) + , flush(0) + , control(0) + , move(0) {} size_t archive; size_t inspect; size_t list; @@ -64,21 +73,18 @@ class ApiSpy : public fdb5::FDBBase { void openForWrite(const eckit::Length&) override { NOTIMP; } void openForAppend(const eckit::Length&) override { NOTIMP; } eckit::Length estimate() override { return 999; } - long read(void*,long) override { NOTIMP; } - long write(const void*,long) override { NOTIMP; } + long read(void*, long) override { NOTIMP; } + long write(const void*, long) override { NOTIMP; } void close() override { NOTIMP; } }; public: // methods - using FDBBase::stats; ApiSpy(const fdb5::Config& config, const std::string& name) : FDBBase(config, name) { knownSpies().push_back(this); } - ~ApiSpy() override { - knownSpies().erase(std::find(knownSpies().begin(), knownSpies().end(), this)); - } + ~ApiSpy() override { knownSpies().erase(std::find(knownSpies().begin(), knownSpies().end(), this)); } void archive(const fdb5::Key& key, const void* data, size_t length) override { counts_.archive += 1; @@ -96,7 +102,7 @@ class ApiSpy : public fdb5::FDBBase { return fdb5::ListIterator(0); } - fdb5::AxesIterator axesIterator(const fdb5::FDBToolRequest& request, int level=3) override { + fdb5::AxesIterator axesIterator(const fdb5::FDBToolRequest& request, int level = 3) override { counts_.axes += 1; return fdb5::AxesIterator(0); } @@ -131,16 +137,13 @@ class ApiSpy : public fdb5::FDBBase { return fdb5::MoveIterator(0); } - fdb5::StatusIterator control(const fdb5::FDBToolRequest& request, - fdb5::ControlAction action, - fdb5::ControlIdentifiers identifiers) override { + fdb5::StatusIterator control(const fdb5::FDBToolRequest& request, fdb5::ControlAction action, + fdb5::ControlIdentifiers identifiers) override { counts_.control += 1; return fdb5::StatusIterator(0); } - void flush() override { - counts_.flush += 1; - } + void flush() override { counts_.flush += 1; } // For diagnostics @@ -155,18 +158,15 @@ class ApiSpy : public fdb5::FDBBase { } private: // methods - void print(std::ostream& s) const override { s << "ApiSpy()"; } private: // members - Counts counts_; Archives archives_; Retrieves retrieves_; }; - static fdb5::FDBBuilder selectFdbBuilder("spy"); //---------------------------------------------------------------------------------------------------------------------- diff --git a/tests/fdb/api/test_auxiliary.cc b/tests/fdb/api/test_auxiliary.cc index 1f88983d6..f13917173 100644 --- a/tests/fdb/api/test_auxiliary.cc +++ b/tests/fdb/api/test_auxiliary.cc @@ -1,9 +1,9 @@ -#include "eckit/testing/Test.h" -#include "eckit/filesystem/TmpDir.h" #include "eckit/filesystem/LocalPathName.h" -#include "metkit/mars/MarsRequest.h" +#include "eckit/filesystem/TmpDir.h" +#include "eckit/testing/Test.h" #include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" +#include "metkit/mars/MarsRequest.h" namespace fdb5::test { @@ -14,7 +14,7 @@ std::set extensions = {"foo", "bar"}; eckit::PathName writeAuxiliaryData(const eckit::PathName datapath, const std::string ext) { eckit::PathName auxpath(datapath + "." + ext); std::string data_str = "Some extra data"; - const void* data = static_cast(data_str.c_str()); + const void* data = static_cast(data_str.c_str()); size_t length = data_str.size(); eckit::FileHandle file(auxpath); file.openForWrite(0); @@ -26,7 +26,8 @@ eckit::PathName writeAuxiliaryData(const eckit::PathName datapath, const std::st std::set setup(FDB& fdb) { // Setup: Write data, generating auxiliary files using the archive callback std::set auxPaths; - fdb.registerArchiveCallback([&auxPaths] (const Key& key, const void* data, size_t length, std::future> future) { + fdb.registerArchiveCallback([&auxPaths](const Key& key, const void* data, size_t length, + std::future> future) { std::shared_ptr location = future.get(); for (const auto& ext : extensions) { auxPaths.insert(writeAuxiliaryData(location->uri().path(), ext)); @@ -34,29 +35,29 @@ std::set setup(FDB& fdb) { }); std::string data_str = "Raining cats and dogs"; - const void* data = static_cast(data_str.c_str()); + const void* data = static_cast(data_str.c_str()); size_t length = data_str.size(); Key key; - key.set("class","od"); - key.set("expver","xxxx"); - key.set("type","fc"); - key.set("stream","oper"); - key.set("date","20101010"); - key.set("time","0000"); - key.set("domain","g"); - key.set("levtype","sfc"); - key.set("param","130"); - - key.set("step","1"); + key.set("class", "od"); + key.set("expver", "xxxx"); + key.set("type", "fc"); + key.set("stream", "oper"); + key.set("date", "20101010"); + key.set("time", "0000"); + key.set("domain", "g"); + key.set("levtype", "sfc"); + key.set("param", "130"); + + key.set("step", "1"); fdb.archive(key, data, length); - key.set("date","20111213"); + key.set("date", "20111213"); fdb.archive(key, data, length); - key.set("type","pf"); + key.set("type", "pf"); fdb.archive(key, data, length); - + fdb.flush(); return auxPaths; @@ -75,7 +76,7 @@ CASE("Wipe with extensions") { for (const auto& auxPath : auxPaths) { EXPECT(auxPath.exists()); } - + // call wipe FDBToolRequest request = FDBToolRequest::requestsFromString("class=od,expver=xxxx")[0]; bool doit = true; @@ -125,7 +126,7 @@ CASE("Purge with extensions") { PurgeElement elem; while (listObject.next(elem)) { - eckit::Log::info () << elem << std::endl; + eckit::Log::info() << elem << std::endl; } // Check that the masked auxiliary files have been removed diff --git a/tests/fdb/api/test_callback.cc b/tests/fdb/api/test_callback.cc index 2be8f9f0a..7af85d8ff 100644 --- a/tests/fdb/api/test_callback.cc +++ b/tests/fdb/api/test_callback.cc @@ -1,6 +1,6 @@ -#include "eckit/testing/Test.h" -#include "eckit/filesystem/TmpDir.h" #include "eckit/filesystem/LocalPathName.h" +#include "eckit/filesystem/TmpDir.h" +#include "eckit/testing/Test.h" #include "fdb5/api/FDB.h" namespace fdb5::test { @@ -14,45 +14,44 @@ CASE("Archive and flush callback") { FDB fdb; std::string data_str = "Raining cats and dogs"; - const void* data = static_cast(data_str.c_str()); + const void* data = static_cast(data_str.c_str()); size_t length = data_str.size(); Key key; - key.set("class","od"); - key.set("expver","xxxx"); - key.set("type","fc"); - key.set("stream","oper"); - key.set("date","20101010"); - key.set("time","0000"); - key.set("domain","g"); - key.set("levtype","sfc"); - key.set("param","130"); + key.set("class", "od"); + key.set("expver", "xxxx"); + key.set("type", "fc"); + key.set("stream", "oper"); + key.set("date", "20101010"); + key.set("time", "0000"); + key.set("domain", "g"); + key.set("levtype", "sfc"); + key.set("param", "130"); std::map map; std::vector keys; bool flushCalled = false; - fdb.registerArchiveCallback([&map] (const Key& key, const void* data, size_t length, std::future> future) { + fdb.registerArchiveCallback([&map](const Key& key, const void* data, size_t length, + std::future> future) { std::shared_ptr location = future.get(); map[key] = location->fullUri(); }); - fdb.registerFlushCallback([&flushCalled] () { - flushCalled = true; - }); + fdb.registerFlushCallback([&flushCalled]() { flushCalled = true; }); - key.set("step","1"); + key.set("step", "1"); keys.push_back(key); fdb.archive(key, data, length); - key.set("date","20111213"); + key.set("date", "20111213"); keys.push_back(key); fdb.archive(key, data, length); - key.set("type","pf"); + key.set("type", "pf"); keys.push_back(key); fdb.archive(key, data, length); - + fdb.flush(); EXPECT(flushCalled); @@ -62,14 +61,13 @@ CASE("Archive and flush callback") { for (const auto& [key, uri] : map) { bool found = false; for (const auto& originalKey : keys) { - if (key == originalKey){ + if (key == originalKey) { found = true; break; } } EXPECT(found); } - } //---------------------------------------------------------------------------------------------------------------------- diff --git a/tests/fdb/api/test_config.cc b/tests/fdb/api/test_config.cc index 5c9e5eb3e..d789eb8dd 100644 --- a/tests/fdb/api/test_config.cc +++ b/tests/fdb/api/test_config.cc @@ -17,8 +17,8 @@ #include "eckit/config/Resource.h" #include "eckit/filesystem/PathName.h" -#include "eckit/filesystem/TmpFile.h" #include "eckit/filesystem/TmpDir.h" +#include "eckit/filesystem/TmpFile.h" #include "eckit/io/DataHandle.h" #include "eckit/testing/Test.h" @@ -27,13 +27,12 @@ using namespace eckit::testing; using namespace eckit; - namespace fdb { namespace test { //---------------------------------------------------------------------------------------------------------------------- -CASE( "config_expands_from_environment_variable_json" ) { +CASE("config_expands_from_environment_variable_json") { const std::string config_str(R"XX( { @@ -55,11 +54,11 @@ CASE( "config_expands_from_environment_variable_json" ) { EXPECT(expanded.getString("engine") == "toc"); EXPECT(expanded.getSubConfigurations("groups").size() == 1); EXPECT(expanded.getSubConfigurations("groups")[0].getSubConfigurations("pools").size() == 1); - EXPECT(expanded.getSubConfigurations("groups")[0].getSubConfigurations("pools")[0].getString("path") == "/a/path/is/something"); + EXPECT(expanded.getSubConfigurations("groups")[0].getSubConfigurations("pools")[0].getString("path") == + "/a/path/is/something"); } - -CASE( "config_expands_from_environment_variable_yaml" ) { +CASE("config_expands_from_environment_variable_yaml") { const std::string config_str(R"XX( --- @@ -78,10 +77,11 @@ CASE( "config_expands_from_environment_variable_yaml" ) { EXPECT(expanded.getString("engine") == "toc"); EXPECT(expanded.getSubConfigurations("spaces").size() == 1); EXPECT(expanded.getSubConfigurations("spaces")[0].getSubConfigurations("roots").size() == 1); - EXPECT(expanded.getSubConfigurations("spaces")[0].getSubConfigurations("roots")[0].getString("path") == "/a/path/is/something"); + EXPECT(expanded.getSubConfigurations("spaces")[0].getSubConfigurations("roots")[0].getString("path") == + "/a/path/is/something"); } -CASE( "config_expands_explicit_path" ) { +CASE("config_expands_explicit_path") { const std::string config_str(R"XX( --- @@ -110,10 +110,11 @@ CASE( "config_expands_explicit_path" ) { EXPECT(expanded.getString("engine") == "toc"); EXPECT(expanded.getSubConfigurations("spaces").size() == 1); EXPECT(expanded.getSubConfigurations("spaces")[0].getSubConfigurations("roots").size() == 1); - EXPECT(expanded.getSubConfigurations("spaces")[0].getSubConfigurations("roots")[0].getString("path") == "/a/path/is/different"); + EXPECT(expanded.getSubConfigurations("spaces")[0].getSubConfigurations("roots")[0].getString("path") == + "/a/path/is/different"); } -CASE( "config_expands_override_fdb_home" ) { +CASE("config_expands_override_fdb_home") { const std::string config_str(R"XX( --- @@ -146,11 +147,13 @@ CASE( "config_expands_override_fdb_home" ) { EXPECT(expanded.getString("engine") == "toc"); EXPECT(expanded.getSubConfigurations("spaces").size() == 1); EXPECT(expanded.getSubConfigurations("spaces")[0].getSubConfigurations("roots").size() == 2); - EXPECT(expanded.getSubConfigurations("spaces")[0].getSubConfigurations("roots")[0].getString("path") == "/a/path/is/first"); - EXPECT(expanded.getSubConfigurations("spaces")[0].getSubConfigurations("roots")[1].getString("path") == "/a/path/is/second"); + EXPECT(expanded.getSubConfigurations("spaces")[0].getSubConfigurations("roots")[0].getString("path") == + "/a/path/is/first"); + EXPECT(expanded.getSubConfigurations("spaces")[0].getSubConfigurations("roots")[1].getString("path") == + "/a/path/is/second"); } -CASE( "userConfig" ) { +CASE("userConfig") { const std::string config_str(R"XX( --- @@ -198,26 +201,26 @@ CASE( "userConfig" ) { { fdb5::Config cfg; cfg.set("type", "dist"); - cfg.set("lanes", { cfg_od, cfg_rd1, cfg_rd2 }); + cfg.set("lanes", {cfg_od, cfg_rd1, cfg_rd2}); EXPECT(cfg.userConfig().getBool("useSubToc", false) == false); std::vector configs = cfg.getSubConfigs("lanes"); ASSERT(configs.size() == 3); - for (const auto& c: configs) { + for (const auto& c : configs) { EXPECT(c.userConfig().getBool("useSubToc", false) == false); } } { fdb5::Config cfg(eckit::LocalConfiguration(), userConf); cfg.set("type", "dist"); - cfg.set("lanes", { cfg_od, cfg_rd1, cfg_rd2 }); + cfg.set("lanes", {cfg_od, cfg_rd1, cfg_rd2}); EXPECT(cfg.userConfig().getBool("useSubToc", false) == true); std::vector configs = cfg.getSubConfigs("lanes"); ASSERT(configs.size() == 3); - for (const auto& c: configs) { + for (const auto& c : configs) { EXPECT(c.userConfig().getBool("useSubToc", false) == true); } } @@ -225,10 +228,9 @@ CASE( "userConfig" ) { //---------------------------------------------------------------------------------------------------------------------- -} // namespace test -} // namespace fdb +} // namespace test +} // namespace fdb -int main(int argc, char **argv) -{ - return run_tests ( argc, argv ); +int main(int argc, char** argv) { + return run_tests(argc, argv); } diff --git a/tests/fdb/api/test_dist.cc b/tests/fdb/api/test_dist.cc index dd86905f7..7636d1073 100644 --- a/tests/fdb/api/test_dist.cc +++ b/tests/fdb/api/test_dist.cc @@ -20,21 +20,19 @@ #include "metkit/mars/TypeAny.h" -#include "fdb5/config/Config.h" #include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/config/Config.h" #include "ApiSpy.h" using namespace eckit::testing; using namespace eckit; - namespace fdb { namespace test { //---------------------------------------------------------------------------------------------------------------------- - fdb5::Config defaultConfig() { // Build a standard configuration to demonstrate features of DistFDB @@ -53,13 +51,12 @@ fdb5::Config defaultConfig() { fdb5::Config cfg; cfg.set("type", "dist"); - cfg.set("lanes", { cfg_od, cfg_rd1, cfg_rd2 }); + cfg.set("lanes", {cfg_od, cfg_rd1, cfg_rd2}); return cfg; } - -CASE( "archives_distributed_according_to_dist" ) { +CASE("archives_distributed_according_to_dist") { // Build FDB from default config @@ -96,9 +93,9 @@ CASE( "archives_distributed_according_to_dist" ) { k.set("f", eckit::Translator()(f)); k.set("a", eckit::Translator()(a)); - data.assign(data.size(), 100*f + a); + data.assign(data.size(), 100 * f + a); - size_t len = data.size()*sizeof(int); + size_t len = data.size() * sizeof(int); fdb.archive(k, data.data(), len); @@ -108,7 +105,7 @@ CASE( "archives_distributed_according_to_dist" ) { fdb.flush(); - EXPECT(spy1.counts().flush + spy2.counts().flush + spy3.counts().flush <= flush_count+3); + EXPECT(spy1.counts().flush + spy2.counts().flush + spy3.counts().flush <= flush_count + 3); flush_count = (spy1.counts().flush + spy2.counts().flush + spy3.counts().flush); } @@ -129,8 +126,7 @@ CASE( "archives_distributed_according_to_dist" ) { } } - -CASE( "retrieves_distributed_according_to_dist" ) { +CASE("retrieves_distributed_according_to_dist") { // Build FDB from default config @@ -201,7 +197,7 @@ CASE( "retrieves_distributed_according_to_dist" ) { } } -CASE( "lists_distributed_according_to_dist" ) { +CASE("lists_distributed_according_to_dist") { // Build FDB from default config @@ -267,8 +263,7 @@ CASE( "lists_distributed_according_to_dist" ) { } } - -CASE( "dump_distributed_according_to_dist" ) { +CASE("dump_distributed_according_to_dist") { // Build FDB from default config @@ -334,7 +329,7 @@ CASE( "dump_distributed_according_to_dist" ) { } } -CASE( "status_distributed_according_to_dist" ) { +CASE("status_distributed_according_to_dist") { // Build FDB from default config @@ -400,8 +395,7 @@ CASE( "status_distributed_according_to_dist" ) { } } - -CASE( "wipe_distributed_according_to_dist" ) { +CASE("wipe_distributed_according_to_dist") { // Build FDB from default config @@ -467,8 +461,7 @@ CASE( "wipe_distributed_according_to_dist" ) { } } - -CASE( "purge_distributed_according_to_dist" ) { +CASE("purge_distributed_according_to_dist") { // Build FDB from default config @@ -534,8 +527,7 @@ CASE( "purge_distributed_according_to_dist" ) { } } - -CASE( "stats_distributed_according_to_dist" ) { +CASE("stats_distributed_according_to_dist") { // Build FDB from default config @@ -601,8 +593,7 @@ CASE( "stats_distributed_according_to_dist" ) { } } - -CASE( "control_distributed_according_to_dist" ) { +CASE("control_distributed_according_to_dist") { // Build FDB from default config @@ -614,15 +605,15 @@ CASE( "control_distributed_according_to_dist" ) { // Do some archiving - fdb.control(fdb5::FDBToolRequest::requestsFromString("class=od,expver=xxxx")[0], - fdb5::ControlAction::Disable, fdb5::ControlIdentifiers(fdb5::ControlIdentifier::List)); + fdb.control(fdb5::FDBToolRequest::requestsFromString("class=od,expver=xxxx")[0], fdb5::ControlAction::Disable, + fdb5::ControlIdentifiers(fdb5::ControlIdentifier::List)); EXPECT(spy1.counts().control == 1); EXPECT(spy2.counts().control == 1); EXPECT(spy3.counts().control == 1); - fdb.control(fdb5::FDBToolRequest::requestsFromString("class=rd,expver=xxxx")[0], - fdb5::ControlAction::Disable, fdb5::ControlIdentifiers(fdb5::ControlIdentifier::Wipe)); + fdb.control(fdb5::FDBToolRequest::requestsFromString("class=rd,expver=xxxx")[0], fdb5::ControlAction::Disable, + fdb5::ControlIdentifiers(fdb5::ControlIdentifier::Wipe)); EXPECT(spy1.counts().control == 2); EXPECT(spy2.counts().control == 2); @@ -631,8 +622,8 @@ CASE( "control_distributed_according_to_dist" ) { // Under specified - matches nothing. Requests halted at this point, as FDB retrieves need // to be fully specified - fdb.control(fdb5::FDBToolRequest::requestsFromString("class=rd,expver=zzzz")[0], - fdb5::ControlAction::Enable, fdb5::ControlIdentifiers(fdb5::ControlIdentifier::Retrieve)); + fdb.control(fdb5::FDBToolRequest::requestsFromString("class=rd,expver=zzzz")[0], fdb5::ControlAction::Enable, + fdb5::ControlIdentifiers(fdb5::ControlIdentifier::Retrieve)); EXPECT(spy1.counts().control == 3); EXPECT(spy2.counts().control == 3); @@ -640,8 +631,8 @@ CASE( "control_distributed_according_to_dist" ) { //// Now match all the rd lanes - fdb.control(fdb5::FDBToolRequest::requestsFromString("class=rd")[0], - fdb5::ControlAction::Enable, fdb5::ControlIdentifiers(fdb5::ControlIdentifier::Archive)); + fdb.control(fdb5::FDBToolRequest::requestsFromString("class=rd")[0], fdb5::ControlAction::Enable, + fdb5::ControlIdentifiers(fdb5::ControlIdentifier::Archive)); EXPECT(spy1.counts().control == 4); EXPECT(spy2.counts().control == 4); @@ -649,8 +640,8 @@ CASE( "control_distributed_according_to_dist" ) { // Explicitly match everything - fdb.control(fdb5::FDBToolRequest({}, true), - fdb5::ControlAction::Disable, fdb5::ControlIdentifiers(fdb5::ControlIdentifier::List)); + fdb.control(fdb5::FDBToolRequest({}, true), fdb5::ControlAction::Disable, + fdb5::ControlIdentifiers(fdb5::ControlIdentifier::List)); EXPECT(spy1.counts().control == 5); EXPECT(spy2.counts().control == 5); @@ -673,10 +664,9 @@ CASE( "control_distributed_according_to_dist" ) { //---------------------------------------------------------------------------------------------------------------------- -} // namespace test -} // namespace fdb +} // namespace test +} // namespace fdb -int main(int argc, char **argv) -{ - return run_tests ( argc, argv ); +int main(int argc, char** argv) { + return run_tests(argc, argv); } diff --git a/tests/fdb/api/test_fdb_c.cc b/tests/fdb/api/test_fdb_c.cc index 7d8d34c5f..cdc1c91aa 100644 --- a/tests/fdb/api/test_fdb_c.cc +++ b/tests/fdb/api/test_fdb_c.cc @@ -23,7 +23,6 @@ using namespace eckit::testing; using namespace eckit; - namespace fdb { namespace test { @@ -32,9 +31,9 @@ int fdb_request_add1(fdb_request_t* req, const char* param, const char* value) { return fdb_request_add(req, param, &value, 1); } -void key_compare(const std::vector& keys, fdb_listiterator_t *it, bool checkLevel = true) { - const char *k; - const char *v; +void key_compare(const std::vector& keys, fdb_listiterator_t* it, bool checkLevel = true) { + const char* k; + const char* v; size_t l; int err; @@ -44,8 +43,8 @@ void key_compare(const std::vector& keys, fdb_listiterator_t *it, boo EXPECT(err == FDB_SUCCESS); size_t level = 0; - for (auto key: keys) { - for (auto k1: key) { + for (auto key : keys) { + for (auto k1 : key) { int err = fdb_splitkey_next_metadata(sk, &k, &v, checkLevel ? &l : nullptr); EXPECT(err == FDB_SUCCESS); EXPECT(k1.first == k); @@ -58,13 +57,13 @@ void key_compare(const std::vector& keys, fdb_listiterator_t *it, boo } err = fdb_splitkey_next_metadata(sk, &k, &v, &l); EXPECT(err == FDB_ITERATION_COMPLETE); - + err = fdb_delete_splitkey(sk); } -CASE( "fdb_c - archive & list" ) { +CASE("fdb_c - archive & list") { size_t length; - DataHandle *dh; + DataHandle* dh; fdb_handle_t* fdb; fdb_new_handle(&fdb); @@ -108,32 +107,36 @@ CASE( "fdb_c - archive & list" ) { fdb_request_add1(request, "type", "an"); fdb_request_add1(request, "expver", "xxxx"); - const char **item= new const char*; + const char** item = new const char*; fdb_listiterator_t* it; fdb_list(fdb, request, &it, true); int err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); - - const char *uri; + + const char* uri; size_t off, attr_len; fdb_listiterator_attrs(it, &uri, &off, &attr_len); EXPECT(attr_len == 3280398); - std::vector k1test { - {{"class", "rd"}, {"expver", "xxxx"}, {"stream", "oper"}, {"date", "20191110"}, {"time", "0000"}, {"domain", "g"}}, + std::vector k1test{ + {{"class", "rd"}, + {"expver", "xxxx"}, + {"stream", "oper"}, + {"date", "20191110"}, + {"time", "0000"}, + {"domain", "g"}}, {{"type", "an"}, {"levtype", "pl"}}, {{"step", "0"}, {"levelist", "300"}, {"param", "138"}}, }; -// std::vector k1test{fdb5::Key{"class=rd,expver=xxxx,stream=oper,date=20191110,time=0000,domain=g"}, -// fdb5::Key{"type=an,levtype=pl"},fdb5::Key{"step=0,levelist=300,param=138"}}; + // std::vector k1test{fdb5::Key{"class=rd,expver=xxxx,stream=oper,date=20191110,time=0000,domain=g"}, + // fdb5::Key{"type=an,levtype=pl"},fdb5::Key{"step=0,levelist=300,param=138"}}; key_compare(k1test, it); err = fdb_listiterator_next(it); ASSERT(err == FDB_ITERATION_COMPLETE); fdb_delete_listiterator(it); - fdb_request_add1(request, "param", "139"); fdb_list(fdb, request, &it, true); err = fdb_listiterator_next(it); @@ -159,22 +162,26 @@ CASE( "fdb_c - archive & list" ) { ASSERT(err == FDB_ITERATION_COMPLETE); fdb_delete_listiterator(it); - fdb_request_add1(request, "param", "138"); fdb_list(fdb, request, &it, true); err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); - + fdb_listiterator_attrs(it, &uri, &off, &attr_len); EXPECT(attr_len == 3280398); - std::vector k2test { - {{"class", "rd"}, {"expver", "xxxx"}, {"stream", "oper"}, {"date", "20191110"}, {"time", "0000"}, {"domain", "g"}}, + std::vector k2test{ + {{"class", "rd"}, + {"expver", "xxxx"}, + {"stream", "oper"}, + {"date", "20191110"}, + {"time", "0000"}, + {"domain", "g"}}, {{"type", "an"}, {"levtype", "pl"}}, {{"step", "0"}, {"levelist", "400"}, {"param", "138"}}, }; -// std::vector k2test{fdb5::Key{"class=rd,expver=xxxx,stream=oper,date=20191110,time=0000,domain=g"}, -// fdb5::Key{"type=an,levtype=pl"},fdb5::Key{"step=0,levelist=400,param=138"}}; + // std::vector k2test{fdb5::Key{"class=rd,expver=xxxx,stream=oper,date=20191110,time=0000,domain=g"}, + // fdb5::Key{"type=an,levtype=pl"},fdb5::Key{"step=0,levelist=400,param=138"}}; key_compare(k2test, it, false); key_compare(k2test, it); @@ -214,25 +221,33 @@ CASE( "fdb_c - archive & list" ) { dh->openForRead(); dh->read(buf3, length); dh->close(); - } - #if fdb5_HAVE_GRIB -CASE( "fdb_c - multiple archive & list" ) { +CASE("fdb_c - multiple archive & list") { size_t length1, length2, length3; - DataHandle *dh; + DataHandle* dh; fdb_handle_t* fdb; fdb_new_handle(&fdb); - std::vector k1 { - {{"class", "rd"}, {"expver", "xxxx"}, {"stream", "oper"}, {"date", "20191110"}, {"time", "0000"}, {"domain", "g"}}, + std::vector k1{ + {{"class", "rd"}, + {"expver", "xxxx"}, + {"stream", "oper"}, + {"date", "20191110"}, + {"time", "0000"}, + {"domain", "g"}}, {{"type", "an"}, {"levtype", "pl"}}, {{"step", "0"}, {"levelist", "300"}, {"param", "138"}}, }; - std::vector k2 { - {{"class", "rd"}, {"expver", "xxxx"}, {"stream", "oper"}, {"date", "20191110"}, {"time", "0000"}, {"domain", "g"}}, + std::vector k2{ + {{"class", "rd"}, + {"expver", "xxxx"}, + {"stream", "oper"}, + {"date", "20191110"}, + {"time", "0000"}, + {"domain", "g"}}, {{"type", "an"}, {"levtype", "pl"}}, {{"step", "0"}, {"levelist", "400"}, {"param", "138"}}, }; @@ -244,7 +259,7 @@ CASE( "fdb_c - multiple archive & list" ) { eckit::PathName grib3("y138-400.grib"); length3 = grib3.size(); - eckit::Buffer buf(length1+length2+length3); + eckit::Buffer buf(length1 + length2 + length3); dh = grib1.fileHandle(); dh->openForRead(); dh->read(buf, length1); @@ -252,7 +267,7 @@ CASE( "fdb_c - multiple archive & list" ) { dh = grib2.fileHandle(); dh->openForRead(); - dh->read(buf+length1, length2); + dh->read(buf + length1, length2); dh->close(); fdb_request_t* req; @@ -273,22 +288,22 @@ CASE( "fdb_c - multiple archive & list" ) { EXPECT(FDB_ERROR_GENERAL_EXCEPTION == fdb_archive_multiple(fdb, req, buf, length1)); EXPECT(FDB_SUCCESS == fdb_flush(fdb)); - EXPECT(FDB_SUCCESS == fdb_archive_multiple(fdb, req, buf, length1+length2)); + EXPECT(FDB_SUCCESS == fdb_archive_multiple(fdb, req, buf, length1 + length2)); EXPECT(FDB_SUCCESS == fdb_flush(fdb)); fdb_request_add1(req, "levelist", "300"); - EXPECT(FDB_ERROR_GENERAL_EXCEPTION == fdb_archive_multiple(fdb, req, buf, length1+length2)); + EXPECT(FDB_ERROR_GENERAL_EXCEPTION == fdb_archive_multiple(fdb, req, buf, length1 + length2)); EXPECT(FDB_SUCCESS == fdb_flush(fdb)); fdb_request_add(req, "levelist", levels, 2); - EXPECT(FDB_SUCCESS == fdb_archive_multiple(fdb, req, buf, length1+length2)); + EXPECT(FDB_SUCCESS == fdb_archive_multiple(fdb, req, buf, length1 + length2)); EXPECT(FDB_SUCCESS == fdb_flush(fdb)); dh = grib3.fileHandle(); dh->openForRead(); - dh->read(buf+length1+length2, length3); + dh->read(buf + length1 + length2, length3); dh->close(); const char* expvers[] = {"xxxx", "xxxy"}; @@ -297,10 +312,10 @@ CASE( "fdb_c - multiple archive & list" ) { const char* levels3[] = {"300", "400", "500"}; fdb_request_add(req, "levelist", levels3, 3); - EXPECT(FDB_ERROR_GENERAL_EXCEPTION == fdb_archive_multiple(fdb, req, buf, length1+length2+length3)); + EXPECT(FDB_ERROR_GENERAL_EXCEPTION == fdb_archive_multiple(fdb, req, buf, length1 + length2 + length3)); EXPECT(FDB_SUCCESS == fdb_flush(fdb)); - EXPECT(FDB_SUCCESS == fdb_archive_multiple(fdb, nullptr, buf, length1+length2+length3)); + EXPECT(FDB_SUCCESS == fdb_archive_multiple(fdb, nullptr, buf, length1 + length2 + length3)); EXPECT(FDB_SUCCESS == fdb_flush(fdb)); fdb_request_t* request; @@ -317,7 +332,7 @@ CASE( "fdb_c - multiple archive & list" ) { fdb_request_add1(request, "type", "an"); fdb_request_add1(request, "expver", "xxxx"); - const char **item= new const char*; + const char** item = new const char*; bool exist; fdb_listiterator_t* it; fdb_list(fdb, request, &it, true); @@ -342,12 +357,12 @@ CASE( "fdb_c - multiple archive & list" ) { fdb_list(fdb, request, &it, true); err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); - + key_compare(k1, it); err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); - + key_compare(k2, it); err = fdb_listiterator_next(it); @@ -356,7 +371,7 @@ CASE( "fdb_c - multiple archive & list" ) { } #endif -CASE( "fdb_c - retrieve bad request" ) { +CASE("fdb_c - retrieve bad request") { fdb_handle_t* fdb; fdb_new_handle(&fdb); @@ -380,11 +395,11 @@ CASE( "fdb_c - retrieve bad request" ) { long size; fdb_datareader_t* dr; fdb_new_datareader(&dr); -// thrown by deduplication (now deactivted) -// EXPECT(fdb_retrieve(fdb, request, dr) == FDB_ERROR_GENERAL_EXCEPTION); + // thrown by deduplication (now deactivted) + // EXPECT(fdb_retrieve(fdb, request, dr) == FDB_ERROR_GENERAL_EXCEPTION); } -CASE( "fdb_c - retrieve" ) { +CASE("fdb_c - retrieve") { fdb_handle_t* fdb; fdb_new_handle(&fdb); @@ -434,22 +449,20 @@ CASE( "fdb_c - retrieve" ) { fdb_new_datareader(&dr); EXPECT(fdb_retrieve(fdb, request, dr) == FDB_SUCCESS); fdb_datareader_open(dr, &size2); - EXPECT_EQUAL(2*size, size2); + EXPECT_EQUAL(2 * size, size2); fdb_datareader_seek(dr, size); fdb_datareader_read(dr, grib, 4, &read); EXPECT_EQUAL(4, read); EXPECT_EQUAL(0, strncmp(grib, "GRIB", 4)); fdb_datareader_tell(dr, &read); - EXPECT_EQUAL(4+size, read); - fdb_datareader_seek(dr, size2-4); + EXPECT_EQUAL(4 + size, read); + fdb_datareader_seek(dr, size2 - 4); fdb_datareader_read(dr, grib, 6, &read); EXPECT_EQUAL(4, read); fdb_delete_datareader(dr); - } - -CASE( "fdb_c - expand" ) { +CASE("fdb_c - expand") { fdb_handle_t* fdb; fdb_new_handle(&fdb); @@ -480,7 +493,7 @@ CASE( "fdb_c - expand" ) { size_t numValues; char** values; - + fdb_request_get(request, "date", &values, &numValues); EXPECT_EQUAL(numValues, 2); EXPECT_EQUAL(0, strncmp(values[0], "20191110", 8)); @@ -518,7 +531,7 @@ CASE( "fdb_c - expand" ) { EXPECT_EQUAL(0, strncmp(values[2], "20191115", 8)); EXPECT_EQUAL(0, strncmp(values[3], "by", 2)); EXPECT_EQUAL(0, strncmp(values[4], "2", 1)); - for (size_t i = 0; icounts().inspect == 0); @@ -148,8 +145,7 @@ CASE( "archives_distributed_according_to_select" ) { } } - -CASE( "retrieves_distributed_according_to_select" ) { +CASE("retrieves_distributed_according_to_select") { // Build FDB from default config @@ -205,7 +201,7 @@ CASE( "retrieves_distributed_according_to_select" ) { // And unused functions - ApiSpy* spies[] ={&spy_od, &spy_rd1, &spy_rd2}; + ApiSpy* spies[] = {&spy_od, &spy_rd1, &spy_rd2}; for (int i = 0; i < 3; i++) { ApiSpy* spy = spies[i]; EXPECT(spy->counts().archive == 0); @@ -220,7 +216,7 @@ CASE( "retrieves_distributed_according_to_select" ) { } } -CASE( "lists_distributed_according_to_select" ) { +CASE("lists_distributed_according_to_select") { // Build FDB from default config @@ -271,7 +267,7 @@ CASE( "lists_distributed_according_to_select" ) { // And unused functions - ApiSpy* spies[] ={&spy_od, &spy_rd1, &spy_rd2}; + ApiSpy* spies[] = {&spy_od, &spy_rd1, &spy_rd2}; for (int i = 0; i < 3; i++) { ApiSpy* spy = spies[i]; EXPECT(spy->counts().archive == 0); @@ -286,8 +282,7 @@ CASE( "lists_distributed_according_to_select" ) { } } - -CASE( "dump_distributed_according_to_select" ) { +CASE("dump_distributed_according_to_select") { // Build FDB from default config @@ -338,7 +333,7 @@ CASE( "dump_distributed_according_to_select" ) { // And unused functions - ApiSpy* spies[] ={&spy_od, &spy_rd1, &spy_rd2}; + ApiSpy* spies[] = {&spy_od, &spy_rd1, &spy_rd2}; for (int i = 0; i < 3; i++) { ApiSpy* spy = spies[i]; EXPECT(spy->counts().archive == 0); @@ -353,7 +348,7 @@ CASE( "dump_distributed_according_to_select" ) { } } -CASE( "status_distributed_according_to_select" ) { +CASE("status_distributed_according_to_select") { // Build FDB from default config @@ -404,7 +399,7 @@ CASE( "status_distributed_according_to_select" ) { // And unused functions - ApiSpy* spies[] ={&spy_od, &spy_rd1, &spy_rd2}; + ApiSpy* spies[] = {&spy_od, &spy_rd1, &spy_rd2}; for (int i = 0; i < 3; i++) { ApiSpy* spy = spies[i]; EXPECT(spy->counts().archive == 0); @@ -419,8 +414,7 @@ CASE( "status_distributed_according_to_select" ) { } } - -CASE( "wipe_distributed_according_to_select" ) { +CASE("wipe_distributed_according_to_select") { // Build FDB from default config @@ -471,7 +465,7 @@ CASE( "wipe_distributed_according_to_select" ) { // And unused functions - ApiSpy* spies[] ={&spy_od, &spy_rd1, &spy_rd2}; + ApiSpy* spies[] = {&spy_od, &spy_rd1, &spy_rd2}; for (int i = 0; i < 3; i++) { ApiSpy* spy = spies[i]; EXPECT(spy->counts().archive == 0); @@ -486,8 +480,7 @@ CASE( "wipe_distributed_according_to_select" ) { } } - -CASE( "purge_distributed_according_to_select" ) { +CASE("purge_distributed_according_to_select") { // Build FDB from default config @@ -538,7 +531,7 @@ CASE( "purge_distributed_according_to_select" ) { // And unused functions - ApiSpy* spies[] ={&spy_od, &spy_rd1, &spy_rd2}; + ApiSpy* spies[] = {&spy_od, &spy_rd1, &spy_rd2}; for (int i = 0; i < 3; i++) { ApiSpy* spy = spies[i]; EXPECT(spy->counts().archive == 0); @@ -553,8 +546,7 @@ CASE( "purge_distributed_according_to_select" ) { } } - -CASE( "stats_distributed_according_to_select" ) { +CASE("stats_distributed_according_to_select") { // Build FDB from default config @@ -605,7 +597,7 @@ CASE( "stats_distributed_according_to_select" ) { // And unused functions - ApiSpy* spies[] ={&spy_od, &spy_rd1, &spy_rd2}; + ApiSpy* spies[] = {&spy_od, &spy_rd1, &spy_rd2}; for (int i = 0; i < 3; i++) { ApiSpy* spy = spies[i]; EXPECT(spy->counts().archive == 0); @@ -620,8 +612,7 @@ CASE( "stats_distributed_according_to_select" ) { } } - -CASE( "control_distributed_according_to_select" ) { +CASE("control_distributed_according_to_select") { // Build FDB from default config @@ -633,15 +624,15 @@ CASE( "control_distributed_according_to_select" ) { // Do some archiving - fdb.control(fdb5::FDBToolRequest::requestsFromString("class=od,expver=xxxx")[0], - fdb5::ControlAction::Disable, fdb5::ControlIdentifiers(fdb5::ControlIdentifier::List)); + fdb.control(fdb5::FDBToolRequest::requestsFromString("class=od,expver=xxxx")[0], fdb5::ControlAction::Disable, + fdb5::ControlIdentifiers(fdb5::ControlIdentifier::List)); EXPECT(spy_od.counts().control == 1); EXPECT(spy_rd1.counts().control == 0); EXPECT(spy_rd2.counts().control == 0); - fdb.control(fdb5::FDBToolRequest::requestsFromString("class=rd,expver=xxxx")[0], - fdb5::ControlAction::Disable, fdb5::ControlIdentifiers(fdb5::ControlIdentifier::Wipe)); + fdb.control(fdb5::FDBToolRequest::requestsFromString("class=rd,expver=xxxx")[0], fdb5::ControlAction::Disable, + fdb5::ControlIdentifiers(fdb5::ControlIdentifier::Wipe)); EXPECT(spy_od.counts().control == 1); EXPECT(spy_rd1.counts().control == 1); @@ -650,8 +641,8 @@ CASE( "control_distributed_according_to_select" ) { // Under specified - matches nothing. Requests halted at this point, as FDB retrieves need // to be fully specified - fdb.control(fdb5::FDBToolRequest::requestsFromString("class=rd,expver=zzzz")[0], - fdb5::ControlAction::Enable, fdb5::ControlIdentifiers(fdb5::ControlIdentifier::Retrieve)); + fdb.control(fdb5::FDBToolRequest::requestsFromString("class=rd,expver=zzzz")[0], fdb5::ControlAction::Enable, + fdb5::ControlIdentifiers(fdb5::ControlIdentifier::Retrieve)); EXPECT(spy_od.counts().control == 1); EXPECT(spy_rd1.counts().control == 1); @@ -659,8 +650,8 @@ CASE( "control_distributed_according_to_select" ) { //// Now match all the rd lanes - fdb.control(fdb5::FDBToolRequest::requestsFromString("class=rd")[0], - fdb5::ControlAction::Enable, fdb5::ControlIdentifiers(fdb5::ControlIdentifier::Archive)); + fdb.control(fdb5::FDBToolRequest::requestsFromString("class=rd")[0], fdb5::ControlAction::Enable, + fdb5::ControlIdentifiers(fdb5::ControlIdentifier::Archive)); EXPECT(spy_od.counts().control == 1); EXPECT(spy_rd1.counts().control == 2); @@ -668,8 +659,8 @@ CASE( "control_distributed_according_to_select" ) { // Explicitly match everything - fdb.control(fdb5::FDBToolRequest({}, true), - fdb5::ControlAction::Disable, fdb5::ControlIdentifiers(fdb5::ControlIdentifier::List)); + fdb.control(fdb5::FDBToolRequest({}, true), fdb5::ControlAction::Disable, + fdb5::ControlIdentifiers(fdb5::ControlIdentifier::List)); EXPECT(spy_od.counts().control == 2); EXPECT(spy_rd1.counts().control == 3); @@ -692,10 +683,9 @@ CASE( "control_distributed_according_to_select" ) { //---------------------------------------------------------------------------------------------------------------------- -} // namespace test -} // namespace fdb +} // namespace test +} // namespace fdb -int main(int argc, char **argv) -{ - return run_tests ( argc, argv ); +int main(int argc, char** argv) { + return run_tests(argc, argv); } diff --git a/tests/fdb/daos/test_daos_catalogue.cc b/tests/fdb/daos/test_daos_catalogue.cc index 0ee22ca88..c4a30706d 100644 --- a/tests/fdb/daos/test_daos_catalogue.cc +++ b/tests/fdb/daos/test_daos_catalogue.cc @@ -12,56 +12,56 @@ #include #include "eckit/config/Resource.h" -#include "eckit/testing/Test.h" -#include "eckit/filesystem/URI.h" +#include "eckit/config/YAMLConfiguration.h" #include "eckit/filesystem/PathName.h" -#include "eckit/filesystem/TmpFile.h" #include "eckit/filesystem/TmpDir.h" +#include "eckit/filesystem/TmpFile.h" +#include "eckit/filesystem/URI.h" #include "eckit/io/FileHandle.h" #include "eckit/io/MemoryHandle.h" -#include "eckit/config/YAMLConfiguration.h" +#include "eckit/testing/Test.h" #include "metkit/mars/MarsRequest.h" -#include "fdb5/fdb5_config.h" -#include "fdb5/config/Config.h" #include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/config/Config.h" +#include "fdb5/fdb5_config.h" #include "fdb5/toc/TocStore.h" -#include "fdb5/daos/DaosSession.h" -#include "fdb5/daos/DaosPool.h" #include "fdb5/daos/DaosArrayPartHandle.h" +#include "fdb5/daos/DaosPool.h" +#include "fdb5/daos/DaosSession.h" -#include "fdb5/daos/DaosStore.h" -#include "fdb5/daos/DaosFieldLocation.h" -#include "fdb5/daos/DaosCatalogueWriter.h" #include "fdb5/daos/DaosCatalogueReader.h" +#include "fdb5/daos/DaosCatalogueWriter.h" +#include "fdb5/daos/DaosFieldLocation.h" +#include "fdb5/daos/DaosStore.h" using namespace eckit::testing; using namespace eckit; namespace { - void deldir(eckit::PathName& p) { - if (!p.exists()) { - return; - } +void deldir(eckit::PathName& p) { + if (!p.exists()) { + return; + } - std::vector files; - std::vector dirs; - p.children(files, dirs); + std::vector files; + std::vector dirs; + p.children(files, dirs); - for (auto& f : files) { - f.unlink(); - } - for (auto& d : dirs) { - deldir(d); - } + for (auto& f : files) { + f.unlink(); + } + for (auto& d : dirs) { + deldir(d); + } - p.rmdir(); - }; -} + p.rmdir(); +}; +} // namespace #ifdef fdb5_HAVE_DUMMY_DAOS eckit::TmpDir& tmp_dummy_daos_root() { @@ -90,16 +90,17 @@ eckit::PathName& catalogue_tests_tmp_root() { namespace fdb { namespace test { -CASE( "Setup" ) { +CASE("Setup") { #ifdef fdb5_HAVE_DUMMY_DAOS tmp_dummy_daos_root().mkdir(); ::setenv("DUMMY_DAOS_DATA_ROOT", tmp_dummy_daos_root().path().c_str(), 1); #endif - // ensure fdb root directory exists. If not, then that root is + // ensure fdb root directory exists. If not, then that root is // registered as non existing and Catalogue/Store tests fail. - if (catalogue_tests_tmp_root().exists()) deldir(catalogue_tests_tmp_root()); + if (catalogue_tests_tmp_root().exists()) + deldir(catalogue_tests_tmp_root()); catalogue_tests_tmp_root().mkdir(); ::setenv("FDB_ROOT_DIRECTORY", catalogue_tests_tmp_root().path().c_str(), 1); @@ -127,7 +128,6 @@ CASE( "Setup" ) { // LibFdb5::instance().defaultConfig().schema() is called // due to no specified schema file (e.g. in Key::registry()) ::setenv("FDB_SCHEMA_FILE", schema_file().path().c_str(), 1); - } CASE("DaosCatalogue tests") { @@ -140,9 +140,7 @@ CASE("DaosCatalogue tests") { std::string pool_name{"fdb_pool2"}; #else std::string pool_name; - pool_name = eckit::Resource( - "fdbDaosTestPool;$FDB_DAOS_TEST_POOL", pool_name - ); + pool_name = eckit::Resource("fdbDaosTestPool;$FDB_DAOS_TEST_POOL", pool_name); EXPECT(pool_name.length() > 0); #endif @@ -151,26 +149,21 @@ CASE("DaosCatalogue tests") { fdb5::UUID pool_uuid; { /// @btodo: should DaosManager really be configured here? May be better not to configure it - /// here, only specify client config in FDB config file, and let the constructor of + /// here, only specify client config in FDB config file, and let the constructor of /// DaosStore or DaosCatalogue configure DaosManager. - fdb5::DaosManager::instance().configure( - eckit::LocalConfiguration(YAMLConfiguration( - "container_oids_per_alloc: " + std::to_string(container_oids_per_alloc) - )) - ); + fdb5::DaosManager::instance().configure(eckit::LocalConfiguration( + YAMLConfiguration("container_oids_per_alloc: " + std::to_string(container_oids_per_alloc)))); fdb5::DaosSession s{}; #ifdef fdb5_HAVE_DAOS_ADMIN fdb5::DaosPool& pool = s.createPool(pool_name); #else - #ifdef fdb5_HAVE_DUMMY_DAOS +#ifdef fdb5_HAVE_DUMMY_DAOS std::string pool_uuid_str{"00000000-0000-0000-0000-000000000004"}; (tmp_dummy_daos_root() / pool_uuid_str).mkdir(); - ::symlink( - (tmp_dummy_daos_root() / pool_uuid_str).path().c_str(), - (tmp_dummy_daos_root() / pool_name).path().c_str() - ); - #endif + ::symlink((tmp_dummy_daos_root() / pool_uuid_str).path().c_str(), + (tmp_dummy_daos_root() / pool_name).path().c_str()); +#endif fdb5::DaosPool& pool = s.getPool(pool_name); #endif @@ -182,20 +175,29 @@ CASE("DaosCatalogue tests") { std::string config_str{ "spaces:\n" "- roots:\n" - " - path: " + catalogue_tests_tmp_root().asString() + "\n" - "schema : " + schema_file().path() + "\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" + "schema : " + + schema_file().path() + + "\n" "daos:\n" " catalogue:\n" - " pool: " + pool_name + "\n" - " root_cont: " + root_cont_name + "\n" + " pool: " + + pool_name + + "\n" + " root_cont: " + + root_cont_name + + "\n" " client:\n" - " container_oids_per_alloc: " + std::to_string(container_oids_per_alloc) - }; + " container_oids_per_alloc: " + + std::to_string(container_oids_per_alloc)}; fdb5::Config config{YAMLConfiguration(config_str)}; fdb5::Schema schema{schema_file()}; - /// @note: a=11,b=22 instead of a=1,b=2 to avoid collision with potential parallel runs of store tests using a=1,b=2 + /// @note: a=11,b=22 instead of a=1,b=2 to avoid collision with potential parallel runs of store tests using + /// a=1,b=2 fdb5::Key request_key({{"a", "11"}, {"b", "22"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); fdb5::Key db_key({{"a", "11"}, {"b", "22"}}); fdb5::Key index_key({{"c", "3"}, {"d", "4"}}); @@ -205,21 +207,20 @@ CASE("DaosCatalogue tests") { /// DaosManager is configured with client config from the file std::unique_ptr loc(new fdb5::DaosFieldLocation( - eckit::URI{"daos", "test_uri"}, eckit::Offset(0), eckit::Length(1), fdb5::Key() - )); + eckit::URI{"daos", "test_uri"}, eckit::Offset(0), eckit::Length(1), fdb5::Key())); { fdb5::DaosCatalogueWriter dcatw{db_key, config}; fdb5::DaosName db_cont{pool_name, db_key.valuesToString()}; - fdb5::DaosKeyValueOID cat_kv_oid{0, 0, OC_S1}; /// @todo: take oclass from config + fdb5::DaosKeyValueOID cat_kv_oid{0, 0, OC_S1}; /// @todo: take oclass from config fdb5::DaosKeyValueName cat_kv{pool_name, db_key.valuesToString(), cat_kv_oid}; EXPECT(db_cont.exists()); EXPECT(cat_kv.exists()); fdb5::Catalogue& cat = dcatw; cat.selectIndex(index_key); - fdb5::DaosKeyValueOID index_kv_oid{index_key.valuesToString(), OC_S1}; /// @todo: take oclass from config + fdb5::DaosKeyValueOID index_kv_oid{index_key.valuesToString(), OC_S1}; /// @todo: take oclass from config fdb5::DaosKeyValueName index_kv{pool_name, db_key.valuesToString(), index_kv_oid}; EXPECT(index_kv.exists()); EXPECT(cat_kv.has(index_key.valuesToString())); @@ -262,12 +263,11 @@ CASE("DaosCatalogue tests") { fdb5::DaosCatalogue::remove(db_cont, out, out, true); - fdb5::DaosKeyValueOID cat_kv_oid{0, 0, OC_S1}; /// @todo: take oclass from config + fdb5::DaosKeyValueOID cat_kv_oid{0, 0, OC_S1}; /// @todo: take oclass from config fdb5::DaosKeyValueName cat_kv{pool_name, db_key.valuesToString(), cat_kv_oid}; EXPECT_NOT(cat_kv.exists()); EXPECT_NOT(db_cont.exists()); } - } SECTION("DaosCatalogue archive (index) and retrieve with a DaosStore") { @@ -277,17 +277,27 @@ CASE("DaosCatalogue tests") { std::string config_str{ "spaces:\n" "- roots:\n" - " - path: " + catalogue_tests_tmp_root().asString() + "\n" - "schema : " + schema_file().path() + "\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" + "schema : " + + schema_file().path() + + "\n" "daos:\n" " store:\n" - " pool: " + pool_name + "\n" + " pool: " + + pool_name + + "\n" " catalogue:\n" - " pool: " + pool_name + "\n" - " root_cont: " + root_cont_name + "\n" + " pool: " + + pool_name + + "\n" + " root_cont: " + + root_cont_name + + "\n" " client:\n" - " container_oids_per_alloc: " + std::to_string(container_oids_per_alloc) - }; + " container_oids_per_alloc: " + + std::to_string(container_oids_per_alloc)}; fdb5::Config config{YAMLConfiguration(config_str)}; @@ -323,7 +333,7 @@ CASE("DaosCatalogue tests") { catw.archive(index_key, field_key, std::move(loc)); /// flush store before flushing catalogue - dstore.flush(); // not necessary if using a DAOS store + dstore.flush(); // not necessary if using a DAOS store } // find data @@ -342,7 +352,7 @@ CASE("DaosCatalogue tests") { std::unique_ptr dh(store.retrieve(field)); EXPECT(dynamic_cast(dh.get())); - + eckit::MemoryHandle mh; dh->copyTo(mh); EXPECT(mh.size() == eckit::Length(sizeof(data))); @@ -360,7 +370,6 @@ CASE("DaosCatalogue tests") { } /// @todo: again, daos_fini happening before - } SECTION("DaosCatalogue archive (index) and retrieve with a TocStore") { @@ -370,15 +379,23 @@ CASE("DaosCatalogue tests") { std::string config_str{ "spaces:\n" "- roots:\n" - " - path: " + catalogue_tests_tmp_root().asString() + "\n" - "schema : " + schema_file().path() + "\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" + "schema : " + + schema_file().path() + + "\n" "daos:\n" " catalogue:\n" - " pool: " + pool_name + "\n" - " root_cont: " + root_cont_name + "\n" + " pool: " + + pool_name + + "\n" + " root_cont: " + + root_cont_name + + "\n" " client:\n" - " container_oids_per_alloc: " + std::to_string(container_oids_per_alloc) - }; + " container_oids_per_alloc: " + + std::to_string(container_oids_per_alloc)}; fdb5::Config config{YAMLConfiguration(config_str)}; @@ -432,7 +449,7 @@ CASE("DaosCatalogue tests") { // retrieve data std::unique_ptr dh(store.retrieve(field)); - + std::vector test(dh->size()); dh->openForRead(); { @@ -466,7 +483,6 @@ CASE("DaosCatalogue tests") { } /// @todo: again, daos_fini happening before - } SECTION("Via FDB API with a DAOS catalogue and store") { @@ -478,20 +494,30 @@ CASE("DaosCatalogue tests") { std::string config_str{ "spaces:\n" "- roots:\n" - " - path: " + catalogue_tests_tmp_root().asString() + "\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" "type: local\n" - "schema : " + schema_file().path() + "\n" + "schema : " + + schema_file().path() + + "\n" "engine: daos\n" "store: daos\n" "daos:\n" " catalogue:\n" - " pool: " + pool_name + "\n" - " root_cont: " + root_cont_name + "\n" + " pool: " + + pool_name + + "\n" + " root_cont: " + + root_cont_name + + "\n" " store:\n" - " pool: " + pool_name + "\n" + " pool: " + + pool_name + + "\n" " client:\n" - " container_oids_per_alloc: " + std::to_string(container_oids_per_alloc_small) - }; + " container_oids_per_alloc: " + + std::to_string(container_oids_per_alloc_small)}; fdb5::Config config{YAMLConfiguration(config_str)}; @@ -501,26 +527,10 @@ CASE("DaosCatalogue tests") { fdb5::Key db_key({{"a", "11"}, {"b", "22"}}); fdb5::Key index_key({{"a", "11"}, {"b", "22"}, {"c", "3"}, {"d", "4"}}); - fdb5::FDBToolRequest full_req{ - request_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; - fdb5::FDBToolRequest index_req{ - index_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; - fdb5::FDBToolRequest db_req{ - db_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; - fdb5::FDBToolRequest all_req{ - metkit::mars::MarsRequest{}, - true, - std::vector{} - }; + fdb5::FDBToolRequest full_req{request_key.request("retrieve"), false, std::vector{"a", "b"}}; + fdb5::FDBToolRequest index_req{index_key.request("retrieve"), false, std::vector{"a", "b"}}; + fdb5::FDBToolRequest db_req{db_key.request("retrieve"), false, std::vector{"a", "b"}}; + fdb5::FDBToolRequest all_req{metkit::mars::MarsRequest{}, true, std::vector{}}; // initialise FDB @@ -549,7 +559,8 @@ CASE("DaosCatalogue tests") { char data[] = "test"; - /// @todo: here, DaosManager is being reconfigured with identical config, and it happens again multiple times below. + /// @todo: here, DaosManager is being reconfigured with identical config, and it happens again multiple times + /// below. // Should this be avoided? fdb.archive(request_key, data, sizeof(data)); @@ -559,7 +570,7 @@ CASE("DaosCatalogue tests") { metkit::mars::MarsRequest r = request_key.request("retrieve"); std::unique_ptr dh(fdb.retrieve(r)); - + eckit::MemoryHandle mh; dh->copyTo(mh); EXPECT(mh.size() == eckit::Length(sizeof(data))); @@ -584,19 +595,22 @@ CASE("DaosCatalogue tests") { auto wipeObject = fdb.wipe(full_req); count = 0; - while (wipeObject.next(elem)) count++; + while (wipeObject.next(elem)) + count++; EXPECT(count == 0); // dry run wipe index and store unit wipeObject = fdb.wipe(index_req); count = 0; - while (wipeObject.next(elem)) count++; + while (wipeObject.next(elem)) + count++; EXPECT(count > 0); // dry run wipe database wipeObject = fdb.wipe(db_req); count = 0; - while (wipeObject.next(elem)) count++; + while (wipeObject.next(elem)) + count++; EXPECT(count > 0); // ensure field still exists @@ -612,7 +626,8 @@ CASE("DaosCatalogue tests") { // attempt to wipe with too specific request wipeObject = fdb.wipe(full_req, true); count = 0; - while (wipeObject.next(elem)) count++; + while (wipeObject.next(elem)) + count++; EXPECT(count == 0); /// @todo: really needed? fdb.flush(); @@ -620,7 +635,8 @@ CASE("DaosCatalogue tests") { // wipe index and store unit wipeObject = fdb.wipe(index_req, true); count = 0; - while (wipeObject.next(elem)) count++; + while (wipeObject.next(elem)) + count++; EXPECT(count > 0); /// @todo: really needed? fdb.flush(); @@ -628,7 +644,8 @@ CASE("DaosCatalogue tests") { // ensure field does not exist listObject = fdb.list(full_req); count = 0; - while (listObject.next(info)) count++; + while (listObject.next(info)) + count++; EXPECT(count == 0); /// @todo: ensure index and corresponding container do not exist @@ -639,7 +656,7 @@ CASE("DaosCatalogue tests") { /// @note: FDB holds a LocalFDB which holds an Archiver which holds open DBs (DaosCatalogueWriters). /// If a whole DB is wiped, the top-level structures for that DB (main and catalogue KVs in this case) - /// are deleted. If willing to archive again into that DB, the DB needs to be constructed again as the + /// are deleted. If willing to archive again into that DB, the DB needs to be constructed again as the /// top-level structures are only generated as part of the DaosCatalogueWriter constructor. There is /// no way currently to destroy the open DBs held by FDB other than entirely destroying FDB. /// Alternatively, a separate FDB instance can be created. @@ -657,12 +674,13 @@ CASE("DaosCatalogue tests") { count++; } EXPECT(count == 1); - + // wipe full database wipeObject = fdb2.wipe(db_req, true); count = 0; - while (wipeObject.next(elem)) count++; + while (wipeObject.next(elem)) + count++; EXPECT(count > 0); /// @todo: really needed? fdb2.flush(); @@ -681,7 +699,6 @@ CASE("DaosCatalogue tests") { /// @todo: ensure DB and corresponding pool do not exist /// @todo: ensure new DaosSession has updated daos client config - } SECTION("OPTIONAL SCHEMA KEYS") { @@ -693,20 +710,30 @@ CASE("DaosCatalogue tests") { std::string config_str{ "spaces:\n" "- roots:\n" - " - path: " + catalogue_tests_tmp_root().asString() + "\n" + " - path: " + + catalogue_tests_tmp_root().asString() + + "\n" "type: local\n" - "schema : " + opt_schema_file().path() + "\n" + "schema : " + + opt_schema_file().path() + + "\n" "engine: daos\n" "store: daos\n" "daos:\n" " catalogue:\n" - " pool: " + pool_name + "\n" - " root_cont: " + root_cont_name + "\n" + " pool: " + + pool_name + + "\n" + " root_cont: " + + root_cont_name + + "\n" " store:\n" - " pool: " + pool_name + "\n" + " pool: " + + pool_name + + "\n" " client:\n" - " container_oids_per_alloc: " + std::to_string(container_oids_per_alloc) - }; + " container_oids_per_alloc: " + + std::to_string(container_oids_per_alloc)}; fdb5::Config config{YAMLConfiguration(config_str)}; @@ -717,31 +744,11 @@ CASE("DaosCatalogue tests") { fdb5::Key db_key({{"a", "11"}, {"b", "22"}}); fdb5::Key index_key({{"a", "11"}, {"b", "22"}, {"d", "4"}}); - fdb5::FDBToolRequest full_req{ - request_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; - fdb5::FDBToolRequest full_req2{ - request_key2.request("retrieve"), - false, - std::vector{"a", "b"} - }; - fdb5::FDBToolRequest index_req{ - index_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; - fdb5::FDBToolRequest db_req{ - db_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; - fdb5::FDBToolRequest all_req{ - metkit::mars::MarsRequest{}, - true, - std::vector{} - }; + fdb5::FDBToolRequest full_req{request_key.request("retrieve"), false, std::vector{"a", "b"}}; + fdb5::FDBToolRequest full_req2{request_key2.request("retrieve"), false, std::vector{"a", "b"}}; + fdb5::FDBToolRequest index_req{index_key.request("retrieve"), false, std::vector{"a", "b"}}; + fdb5::FDBToolRequest db_req{db_key.request("retrieve"), false, std::vector{"a", "b"}}; + fdb5::FDBToolRequest all_req{metkit::mars::MarsRequest{}, true, std::vector{}}; // initialise FDB @@ -835,13 +842,12 @@ CASE("DaosCatalogue tests") { EXPECT(mh.size() == eckit::Length(sizeof(data2))); EXPECT(::memcmp(mh.data(), data2, sizeof(data2)) == 0); } - } // teardown daos #ifdef fdb5_HAVE_DAOS_ADMIN - /// AutoPoolDestroy is not possible here because the pool is + /// AutoPoolDestroy is not possible here because the pool is /// created above with an ephemeral session fdb5::DaosSession().destroyPool(pool_uuid); #else @@ -849,13 +855,11 @@ CASE("DaosCatalogue tests") { if (c == root_cont_name || c == "11:22") fdb5::DaosSession().getPool(pool_uuid).destroyContainer(c); #endif - } -} // namespace test -} // namespace fdb +} // namespace test +} // namespace fdb -int main(int argc, char **argv) -{ - return run_tests ( argc, argv ); +int main(int argc, char** argv) { + return run_tests(argc, argv); } diff --git a/tests/fdb/daos/test_daos_handle.cc b/tests/fdb/daos/test_daos_handle.cc index 15c5af84b..d54595f2f 100644 --- a/tests/fdb/daos/test_daos_handle.cc +++ b/tests/fdb/daos/test_daos_handle.cc @@ -12,24 +12,24 @@ #include #include "eckit/config/Resource.h" -#include "eckit/testing/Test.h" -#include "eckit/filesystem/URI.h" +#include "eckit/config/YAMLConfiguration.h" #include "eckit/filesystem/PathName.h" -#include "eckit/filesystem/TmpFile.h" #include "eckit/filesystem/TmpDir.h" +#include "eckit/filesystem/TmpFile.h" +#include "eckit/filesystem/URI.h" #include "eckit/io/FileHandle.h" #include "eckit/io/MemoryHandle.h" -#include "eckit/config/YAMLConfiguration.h" +#include "eckit/testing/Test.h" #include "fdb5/fdb5_config.h" -#include "fdb5/daos/DaosSession.h" -#include "fdb5/daos/DaosPool.h" -#include "fdb5/daos/DaosContainer.h" -#include "fdb5/daos/DaosObject.h" -#include "fdb5/daos/DaosName.h" #include "fdb5/daos/DaosArrayHandle.h" +#include "fdb5/daos/DaosContainer.h" #include "fdb5/daos/DaosException.h" +#include "fdb5/daos/DaosName.h" +#include "fdb5/daos/DaosObject.h" +#include "fdb5/daos/DaosPool.h" +#include "fdb5/daos/DaosSession.h" using namespace eckit::testing; using namespace eckit; @@ -49,31 +49,32 @@ namespace fdb { namespace test { #ifdef fdb5_HAVE_DUMMY_DAOS -CASE( "Setup" ) { +CASE("Setup") { tmp_dummy_daos_root().mkdir(); ::setenv("DUMMY_DAOS_DATA_ROOT", tmp_dummy_daos_root().path().c_str(), 1); - } #endif #ifdef fdb5_HAVE_DAOS_ADMIN -CASE( "DaosPool" ) { +CASE("DaosPool") { - /// @todo: currently, all pool and container connections are cached and kept open for the duration of the process. Would - /// be nice to close container connections as they become unused. However the DaosContainer instances are managed by the - /// DaosPools/DaosSession, so we never know when the user has finished using a certain container. My current thought is - /// we don't need to fix this, as each process will only use a single pool and 2 * (indices involved) containers. - /// However in a large parallel application, while all client processes are running, there may be a large number of - /// open containers in the DAOS system. One idea would be to use shared pointers to count number of uses. + /// @todo: currently, all pool and container connections are cached and kept open for the duration of the process. + /// Would + /// be nice to close container connections as they become unused. However the DaosContainer instances are managed + /// by the DaosPools/DaosSession, so we never know when the user has finished using a certain container. My + /// current thought is we don't need to fix this, as each process will only use a single pool and 2 * (indices + /// involved) containers. However in a large parallel application, while all client processes are running, there + /// may be a large number of open containers in the DAOS system. One idea would be to use shared pointers to count + /// number of uses. /// @todo: A declarative approach would be better in my opinion. - /// The current approach is an imperative one, where DaosObject and DaosContainer instances always represent existing - /// entities in DAOS from the instant they are created. In highly parallel workflows, validity of such instances will be - /// ephemeral, and by the time we perform an action on them, the DAOS entity they represent may no longer exist. In the - /// declarative approach, the containers and objects would be opened right before the action and fail if they don't exist. - /// In the imperative approach they would fail as well, but the initial checks performed to ensure existence of the DAOS - /// entities would be useless and degrade performance. + /// The current approach is an imperative one, where DaosObject and DaosContainer instances always represent + /// existing entities in DAOS from the instant they are created. In highly parallel workflows, validity of such + /// instances will be ephemeral, and by the time we perform an action on them, the DAOS entity they represent may + /// no longer exist. In the declarative approach, the containers and objects would be opened right before the + /// action and fail if they don't exist. In the imperative approach they would fail as well, but the initial + /// checks performed to ensure existence of the DAOS entities would be useless and degrade performance. /// @todo: cpp uuid wrapper, to avoid weird headers @@ -86,7 +87,7 @@ CASE( "DaosPool" ) { SECTION("unnamed pool") { - fdb5::DaosPool& pool = s.createPool(); /// admin function, not usually called in the client code + fdb5::DaosPool& pool = s.createPool(); /// admin function, not usually called in the client code fdb5::AutoPoolDestroy destroyer(pool); std::cout << pool.name() << std::endl; @@ -94,7 +95,6 @@ CASE( "DaosPool" ) { EXPECT(pool.name().size() == 36); /// @todo: there's an attempt to close unopened pool here due to the pool destruction mechanism - } SECTION("named pool") { @@ -111,7 +111,6 @@ CASE( "DaosPool" ) { fdb5::DaosPool& pool_h = s.getPool(pool_name); EXPECT(&pool == &pool_h); - } SECTION("pool uuid actions") { @@ -124,17 +123,15 @@ CASE( "DaosPool" ) { char uuid_cstr[37]; uuid_unparse(pool_uuid.internal, uuid_cstr); std::cout << uuid_cstr << std::endl; - } /// @todo: test passing some session ad-hoc config for DAOS client /// @todo: there's an extra pair of daos_init and daos_fini happening here - } #endif -CASE( "DaosContainer, DaosArray and DaosKeyValue" ) { +CASE("DaosContainer, DaosArray and DaosKeyValue") { std::string cont_name{"test_cont_2"}; @@ -147,22 +144,17 @@ CASE( "DaosContainer, DaosArray and DaosKeyValue" ) { fdb5::AutoPoolDestroy destroyer(pool); fdb5::DaosContainer& cont = pool.createContainer(cont_name); #else - #ifdef fdb5_HAVE_DUMMY_DAOS +#ifdef fdb5_HAVE_DUMMY_DAOS std::string pool_uuid{"00000000-0000-0000-0000-000000000001"}; std::string pool_name{"test_pool_2"}; (tmp_dummy_daos_root() / pool_uuid).mkdir(); - ::symlink( - (tmp_dummy_daos_root() / pool_uuid).path().c_str(), - (tmp_dummy_daos_root() / pool_name).path().c_str() - ); - #else + ::symlink((tmp_dummy_daos_root() / pool_uuid).path().c_str(), (tmp_dummy_daos_root() / pool_name).path().c_str()); +#else std::string pool_name; - pool_name = eckit::Resource( - "fdbDaosTestPool;$FDB_DAOS_TEST_POOL", pool_name - ); + pool_name = eckit::Resource("fdbDaosTestPool;$FDB_DAOS_TEST_POOL", pool_name); EXPECT(pool_name.length() > 0); - #endif - fdb5::DaosPool& pool = s.getPool(pool_name); +#endif + fdb5::DaosPool& pool = s.getPool(pool_name); fdb5::DaosContainer& cont = pool.createContainer(cont_name); fdb5::AutoContainerDestroy destroyer(cont); #endif @@ -187,23 +179,21 @@ CASE( "DaosContainer, DaosArray and DaosKeyValue" ) { /// @todo: two attempts to close unopened containers here. This is due to mechanism triggered upon /// pool destroy to ensure all matching container handles in the session cache are closed. /// One way to address this would be to have a flag expectOpen = true in DaosContainer::close(). - /// Whenever close() is called as part of pool destroy or batch container close, a false value + /// Whenever close() is called as part of pool destroy or batch container close, a false value /// could be passed to the close() method to avoid logging a warning message. - } SECTION("unnamed object") { // create new object with new automatically allocated oid fdb5::DaosArray arr = cont.createArray(); - std::cout << "New automatically allocated Array OID: " << arr.name() << std::endl; + std::cout << "New automatically allocated Array OID: " << arr.name() << std::endl; fdb5::DaosKeyValue kv = cont.createKeyValue(); - std::cout << "New automatically allocated KeyValue OID: " << kv.name() << std::endl; + std::cout << "New automatically allocated KeyValue OID: " << kv.name() << std::endl; /// @todo: // arr.destroy(); // kv.destroy(); - } SECTION("DaosOID and named object") { @@ -231,8 +221,7 @@ CASE( "DaosContainer, DaosArray and DaosKeyValue" ) { EXPECT_THROWS_AS(fdb5::DaosArray obj(cont, oid_ne), fdb5::DaosEntityNotFoundException); /// @todo: - //write_obj.destroy(); - + // write_obj.destroy(); } /// @todo: DaosArray write and read @@ -244,10 +233,10 @@ CASE( "DaosContainer, DaosArray and DaosKeyValue" ) { std::string test_key{"test_key_1"}; char data[] = "test_data_1"; - kv.put(test_key, data, (uint64_t) sizeof(data)); + kv.put(test_key, data, (uint64_t)sizeof(data)); uint64_t size = kv.size(test_key); - EXPECT(size == (uint64_t) sizeof(data)); + EXPECT(size == (uint64_t)sizeof(data)); uint64_t res; char read_data[20] = ""; @@ -260,8 +249,7 @@ CASE( "DaosContainer, DaosArray and DaosKeyValue" ) { EXPECT_THROWS_AS(kv.get("nonexisting_key", nullptr, 0), fdb5::DaosEntityNotFoundException); /// @todo: - //kv.destroy(); - + // kv.destroy(); } SECTION("DaosName, DaosArrayName, DaosKeyValueName") { @@ -314,7 +302,7 @@ CASE( "DaosContainer, DaosArray and DaosKeyValue" ) { // EXPECT(nc_new.exists()); nc_new.destroy(); EXPECT_NOT(nc_new.exists()); - + std::string test_oid_2_str{"00000000000000020000000000000002"}; fdb5::DaosOID test_oid_2{test_oid_2_str}; fdb5::DaosName n2("a", "b", test_oid_2); @@ -338,7 +326,7 @@ CASE( "DaosContainer, DaosArray and DaosKeyValue" ) { // EXPECT_NOT(n4.exists()); fdb5::DaosKeyValueName nkv = nc.createKeyValueName(); - /// @todo: currently, existence of kvs is checked by attempting open. + /// @todo: currently, existence of kvs is checked by attempting open. /// But a DAOS kv open creates it if not exists. Should implement a better kv /// existency check not involving open/create before uncommenting the following test. // EXPECT_NOT(nkv.exists()); @@ -346,7 +334,7 @@ CASE( "DaosContainer, DaosArray and DaosKeyValue" ) { EXPECT(nkv.exists()); fdb5::DaosKeyValue kv{s, nkv}; char data[] = "test_value_3"; - kv.put("test_key_3", data, (uint64_t) sizeof(data)); + kv.put("test_key_3", data, (uint64_t)sizeof(data)); EXPECT(nkv.has("test_key_3")); /// @todo: // nkv.destroy(); @@ -356,7 +344,7 @@ CASE( "DaosContainer, DaosArray and DaosKeyValue" ) { /// @todo: deserialise fdb5::DaosName deserialisedname(pool_name, cont_name, n1.OID()); - + std::cout << "Object size is: " << deserialisedname.size() << std::endl; kv.put("test_key_4", data, sizeof(data)); @@ -365,7 +353,6 @@ CASE( "DaosContainer, DaosArray and DaosKeyValue" ) { EXPECT(keys.size() == 2); EXPECT(keys[0] == "test_key_3"); EXPECT(keys[1] == "test_key_4"); - } SECTION("DaosHandle write, append and read") { @@ -381,12 +368,12 @@ CASE( "DaosContainer, DaosArray and DaosKeyValue" ) { h.openForWrite(Length(sizeof(data))); { eckit::AutoClose closer(h); - res = h.write(data, (uint64_t) sizeof(data)); - EXPECT(res == (uint64_t) sizeof(data)); + res = h.write(data, (uint64_t)sizeof(data)); + EXPECT(res == (uint64_t)sizeof(data)); EXPECT(h.position() == Offset(sizeof(data))); - res = h.write(data, (uint64_t) sizeof(data)); - EXPECT(res == (uint64_t) sizeof(data)); + res = h.write(data, (uint64_t)sizeof(data)); + EXPECT(res == (uint64_t)sizeof(data)); EXPECT(h.position() == Offset(2 * sizeof(data))); } @@ -399,12 +386,12 @@ CASE( "DaosContainer, DaosArray and DaosKeyValue" ) { Length t = h2.openForRead(); EXPECT(t == Length(2 * sizeof(data))); EXPECT(h2.position() == Offset(0)); - std::vector read_data((size_t) t, 0); + std::vector read_data((size_t)t, 0); { eckit::AutoClose closer(h2); for (int i = 0; i < 2; ++i) { - res = h2.read(&read_data[0] + i * sizeof(data), (uint64_t) sizeof(data)); - EXPECT(res == (uint64_t) sizeof(data)); + res = h2.read(&read_data[0] + i * sizeof(data), (uint64_t)sizeof(data)); + EXPECT(res == (uint64_t)sizeof(data)); } EXPECT(h2.position() == Offset(2 * sizeof(data))); } @@ -413,48 +400,43 @@ CASE( "DaosContainer, DaosArray and DaosKeyValue" ) { std::unique_ptr h3(read_name.dataHandle()); Length t2 = h3->openForRead(); - std::vector read_data2((size_t) t2, 0); + std::vector read_data2((size_t)t2, 0); { eckit::AutoClose closer(*h3); for (int i = 0; i < 2; ++i) { - h3->read(&read_data2[0] + i * sizeof(data), (uint64_t) sizeof(data)); + h3->read(&read_data2[0] + i * sizeof(data), (uint64_t)sizeof(data)); } } EXPECT(std::memcmp(data, &read_data2[0], sizeof(data)) == 0); EXPECT(std::memcmp(data, &read_data2[0] + sizeof(data), sizeof(data)) == 0); fdb5::DaosArrayHandle dh_fail( - fdb5::DaosArrayName( - pool_name, cont_name, fdb5::DaosOID{1, 0, DAOS_OT_ARRAY, OC_S1} - ) - ); + fdb5::DaosArrayName(pool_name, cont_name, fdb5::DaosOID{1, 0, DAOS_OT_ARRAY, OC_S1})); EXPECT_THROWS_AS(dh_fail.openForRead(), fdb5::DaosEntityNotFoundException); - /// @todo: pool, container and object opening are optional before calling + /// @todo: pool, container and object opening are optional before calling /// DaosHandle::openForRead. Test it. /// @todo: container and object creation are optional before calling /// DaosHandle::openForWrite. Test it. /// @todo: test unopen/uncreated DaosObject::size() - } - /// @note: There's no actual need for container destruction as either pool or + /// @note: There's no actual need for container destruction as either pool or /// container are autodestroyed depending on if_HAVE_DAOS_ADMIN. - /// @todo: when enabling this, AutoPoolDestroy fails as a corresponding DaosContainer instance - /// still exists in the DaosPool, but the corresponding container does not exist and + /// @todo: when enabling this, AutoPoolDestroy fails as a corresponding DaosContainer instance + /// still exists in the DaosPool, but the corresponding container does not exist and /// dummy_daos destroy fails trying to rename an unexisting directory. The issue of pool /// and container destruction and invalidation needs to be addressed first. // cont.destroy(); /// @todo: test open pool destroy ensure it is closed - } /// @todo: test a new case where some DAOS operations are carried out with a DaosSession with specific config /// overriding (but not rewriting) DaosManager defaults -CASE( "DaosName and DaosHandle workflows" ) { +CASE("DaosName and DaosHandle workflows") { std::string cont_name{"test_cont_3"}; @@ -467,22 +449,17 @@ CASE( "DaosName and DaosHandle workflows" ) { fdb5::AutoPoolDestroy destroyer(pool); fdb5::DaosContainer& cont = pool.createContainer(cont_name); #else - #ifdef fdb5_HAVE_DUMMY_DAOS +#ifdef fdb5_HAVE_DUMMY_DAOS std::string pool_uuid{"00000000-0000-0000-0000-000000000002"}; std::string pool_name{"test_pool_3"}; (tmp_dummy_daos_root() / pool_uuid).mkdir(); - ::symlink( - (tmp_dummy_daos_root() / pool_uuid).path().c_str(), - (tmp_dummy_daos_root() / pool_name).path().c_str() - ); - #else + ::symlink((tmp_dummy_daos_root() / pool_uuid).path().c_str(), (tmp_dummy_daos_root() / pool_name).path().c_str()); +#else std::string pool_name; - pool_name = eckit::Resource( - "fdbDaosTestPool;$FDB_DAOS_TEST_POOL", pool_name - ); + pool_name = eckit::Resource("fdbDaosTestPool;$FDB_DAOS_TEST_POOL", pool_name); EXPECT(pool_name.length() > 0); - #endif - fdb5::DaosPool& pool = s.getPool(pool_name); +#endif + fdb5::DaosPool& pool = s.getPool(pool_name); fdb5::DaosContainer& cont = pool.createContainer(cont_name); fdb5::AutoContainerDestroy destroyer(cont); #endif @@ -508,7 +485,6 @@ CASE( "DaosName and DaosHandle workflows" ) { std::string oid_end{"000000030000000000000001"}; EXPECT(endsWith(na.OID().asString(), oid_end)); EXPECT(na.size() == len); - } SECTION("Array write to existing pool and container, with automatically generated OID") { @@ -523,7 +499,6 @@ CASE( "DaosName and DaosHandle workflows" ) { } std::cout << "Generated OID: " << na.OID().asString() << std::endl; EXPECT(na.size() == len); - } SECTION("Array write to existing pool and container, with URI") { @@ -538,8 +513,7 @@ CASE( "DaosName and DaosHandle workflows" ) { h->write(data, len); } std::cout << "Generated OID: " << na.OID().asString() << std::endl; - EXPECT(na.size() == len); - + EXPECT(na.size() == len); } SECTION("Array write/read to/from existing pool and container, with generated OID") { @@ -550,15 +524,14 @@ CASE( "DaosName and DaosHandle workflows" ) { fdb5::DaosArrayName na{pool_name, cont_name, oid}; // fdb5::DaosArrayName n{pool_name, cont_name, {"00110000000000000000000000000001"}}; std::unique_ptr h(na.dataHandle()); - h->openForWrite(len); /// @todo: creates it if not exists, should it fail? + h->openForWrite(len); /// @todo: creates it if not exists, should it fail? { eckit::AutoClose closer(*h); h->write(data, len); } std::string oid_end{"000000030000000000000002"}; EXPECT(endsWith(na.OID().asString(), oid_end)); - EXPECT(na.size() == len); - + EXPECT(na.size() == len); } SECTION("Array write to existing pool but non-existing container, with ungenerated OID") { @@ -566,7 +539,7 @@ CASE( "DaosName and DaosHandle workflows" ) { fdb5::DaosArrayOID oid{3, 3, OC_S1}; fdb5::DaosArrayName na{pool_name, "new_cont_1", oid}; EXPECT_NOT(fdb5::DaosName(pool_name, "new_cont_1").exists()); - std::unique_ptr h(na.dataHandle()); /// @todo: should dataHandle() assert pool exists? + std::unique_ptr h(na.dataHandle()); /// @todo: should dataHandle() assert pool exists? h->openForWrite(len); fdb5::AutoContainerDestroy d(pool.getContainer("new_cont_1")); { @@ -578,7 +551,6 @@ CASE( "DaosName and DaosHandle workflows" ) { EXPECT(endsWith(na.OID().asString(), oid_end)); EXPECT(na.size() == len); EXPECT(fdb5::DaosName(pool_name, "new_cont_1").exists()); - } SECTION("Array write to existing pool but non-existing container, with automatically generated OID") { @@ -597,7 +569,6 @@ CASE( "DaosName and DaosHandle workflows" ) { std::cout << "Generated OID: " << na.OID().asString() << std::endl; EXPECT(na.size() == len); EXPECT(fdb5::DaosName(pool_name, "new_cont_2").exists()); - } /// @todo: should use of a non-existing container plus generated OID be forbidden? @@ -619,7 +590,6 @@ CASE( "DaosName and DaosHandle workflows" ) { } EXPECT(nc.exists()); EXPECT(na.size() == len); - } SECTION("Array read from existing pool and container, with generated OID") { @@ -638,7 +608,7 @@ CASE( "DaosName and DaosHandle workflows" ) { fdb5::DaosArrayName na_read{pool_name, cont_name, na.OID()}; std::unique_ptr h2(na_read.dataHandle()); Length t = h2->openForRead(); - std::vector read_data((size_t) t, 0); + std::vector read_data((size_t)t, 0); { eckit::AutoClose closer(*h2); EXPECT(eckit::Length(read_data.size()) >= h2->size()); @@ -653,7 +623,6 @@ CASE( "DaosName and DaosHandle workflows" ) { fdb5::DaosArrayName na_read_ne{pool_name, cont_name, oid_ne}; std::unique_ptr h3(na_read_ne.dataHandle()); EXPECT_THROWS_AS(h3->openForRead(), fdb5::DaosEntityNotFoundException); - } SECTION("Array read from existing pool and container, with generated OID, reading a range") { @@ -684,7 +653,6 @@ CASE( "DaosName and DaosHandle workflows" ) { } EXPECT(res == len - eckit::Length(skip_bytes)); EXPECT(std::memcmp(data + skip_bytes, &read_data[0], sizeof(data) - skip_bytes) == 0); - } SECTION("Array read from existing pool and container, with ungenerated OID") { @@ -703,7 +671,7 @@ CASE( "DaosName and DaosHandle workflows" ) { fdb5::DaosArrayName na_read{pool_name, cont_name, oid}; std::unique_ptr h2(na_read.dataHandle()); Length t = h2->openForRead(); - std::vector read_data((size_t) t, 0); + std::vector read_data((size_t)t, 0); { eckit::AutoClose closer(*h2); EXPECT(eckit::Length(read_data.size()) >= h2->size()); @@ -718,27 +686,23 @@ CASE( "DaosName and DaosHandle workflows" ) { fdb5::DaosArrayName na_read_ne{pool_name, cont_name, oid_ne}; std::unique_ptr h3(na_read_ne.dataHandle()); EXPECT_THROWS_AS(h3->openForRead(), fdb5::DaosEntityNotFoundException); - } SECTION("Array read from existing pool and non-existing container") { - + fdb5::DaosArrayOID oid{3, 9, OC_S1}; fdb5::DaosArrayName na{pool_name, "new_cont_4", oid}; std::unique_ptr h(na.dataHandle()); EXPECT_THROWS_AS(h->openForRead(), fdb5::DaosEntityNotFoundException); EXPECT_NOT(fdb5::DaosName(pool_name, "new_cont_4").exists()); - } /// @todo: test analogous KeyValue workflows - } -} // namespace test -} // namespace fdb +} // namespace test +} // namespace fdb -int main(int argc, char **argv) -{ - return run_tests ( argc, argv ); +int main(int argc, char** argv) { + return run_tests(argc, argv); } diff --git a/tests/fdb/daos/test_daos_store.cc b/tests/fdb/daos/test_daos_store.cc index 5ab8651a0..30328e49f 100644 --- a/tests/fdb/daos/test_daos_store.cc +++ b/tests/fdb/daos/test_daos_store.cc @@ -12,56 +12,56 @@ #include #include "eckit/config/Resource.h" -#include "eckit/testing/Test.h" -#include "eckit/filesystem/URI.h" +#include "eckit/config/YAMLConfiguration.h" #include "eckit/filesystem/PathName.h" -#include "eckit/filesystem/TmpFile.h" #include "eckit/filesystem/TmpDir.h" +#include "eckit/filesystem/TmpFile.h" +#include "eckit/filesystem/URI.h" #include "eckit/io/FileHandle.h" #include "eckit/io/MemoryHandle.h" -#include "eckit/config/YAMLConfiguration.h" +#include "eckit/testing/Test.h" #include "metkit/mars/MarsRequest.h" -#include "fdb5/fdb5_config.h" -#include "fdb5/config/Config.h" #include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/config/Config.h" +#include "fdb5/fdb5_config.h" -#include "fdb5/toc/TocCatalogueWriter.h" #include "fdb5/toc/TocCatalogueReader.h" +#include "fdb5/toc/TocCatalogueWriter.h" -#include "fdb5/daos/DaosSession.h" -#include "fdb5/daos/DaosPool.h" #include "fdb5/daos/DaosArrayPartHandle.h" +#include "fdb5/daos/DaosPool.h" +#include "fdb5/daos/DaosSession.h" -#include "fdb5/daos/DaosStore.h" -#include "fdb5/daos/DaosFieldLocation.h" #include "fdb5/daos/DaosException.h" +#include "fdb5/daos/DaosFieldLocation.h" +#include "fdb5/daos/DaosStore.h" using namespace eckit::testing; using namespace eckit; namespace { - void deldir(eckit::PathName& p) { - if (!p.exists()) { - return; - } +void deldir(eckit::PathName& p) { + if (!p.exists()) { + return; + } - std::vector files; - std::vector dirs; - p.children(files, dirs); + std::vector files; + std::vector dirs; + p.children(files, dirs); - for (auto& f : files) { - f.unlink(); - } - for (auto& d : dirs) { - deldir(d); - } + for (auto& f : files) { + f.unlink(); + } + for (auto& d : dirs) { + deldir(d); + } - p.rmdir(); - }; -} + p.rmdir(); +}; +} // namespace #ifdef fdb5_HAVE_DUMMY_DAOS eckit::TmpDir& tmp_dummy_daos_root() { @@ -85,16 +85,17 @@ eckit::PathName& store_tests_tmp_root() { namespace fdb { namespace test { -CASE( "Setup" ) { +CASE("Setup") { #ifdef fdb5_HAVE_DUMMY_DAOS tmp_dummy_daos_root().mkdir(); ::setenv("DUMMY_DAOS_DATA_ROOT", tmp_dummy_daos_root().path().c_str(), 1); #endif - // ensure fdb root directory exists. If not, then that root is + // ensure fdb root directory exists. If not, then that root is // registered as non existing and Store tests fail. - if (store_tests_tmp_root().exists()) deldir(store_tests_tmp_root()); + if (store_tests_tmp_root().exists()) + deldir(store_tests_tmp_root()); store_tests_tmp_root().mkdir(); ::setenv("FDB_ROOT_DIRECTORY", store_tests_tmp_root().path().c_str(), 1); @@ -113,7 +114,6 @@ CASE( "Setup" ) { // LibFdb5::instance().defaultConfig().schema() is called // due to no specified schema file (e.g. in Key::registry()) ::setenv("FDB_SCHEMA_FILE", schema_file().path().c_str(), 1); - } CASE("DaosStore tests") { @@ -125,9 +125,7 @@ CASE("DaosStore tests") { std::string pool_name{"fdb_pool"}; #else std::string pool_name; - pool_name = eckit::Resource( - "fdbDaosTestPool;$FDB_DAOS_TEST_POOL", pool_name - ); + pool_name = eckit::Resource("fdbDaosTestPool;$FDB_DAOS_TEST_POOL", pool_name); EXPECT(pool_name.length() > 0); #endif @@ -135,23 +133,18 @@ CASE("DaosStore tests") { fdb5::UUID pool_uuid; { - fdb5::DaosManager::instance().configure( - eckit::LocalConfiguration(YAMLConfiguration( - "container_oids_per_alloc: " + std::to_string(container_oids_per_alloc) - )) - ); + fdb5::DaosManager::instance().configure(eckit::LocalConfiguration( + YAMLConfiguration("container_oids_per_alloc: " + std::to_string(container_oids_per_alloc)))); fdb5::DaosSession s{}; #ifdef fdb5_HAVE_DAOS_ADMIN fdb5::DaosPool& pool = s.createPool(pool_name); #else - #ifdef fdb5_HAVE_DUMMY_DAOS +#ifdef fdb5_HAVE_DUMMY_DAOS std::string pool_uuid_str{"00000000-0000-0000-0000-000000000003"}; (tmp_dummy_daos_root() / pool_uuid_str).mkdir(); - ::symlink( - (tmp_dummy_daos_root() / pool_uuid_str).path().c_str(), - (tmp_dummy_daos_root() / pool_name).path().c_str() - ); - #endif + ::symlink((tmp_dummy_daos_root() / pool_uuid_str).path().c_str(), + (tmp_dummy_daos_root() / pool_name).path().c_str()); +#endif fdb5::DaosPool& pool = s.getPool(pool_name); #endif pool_uuid = pool.uuid(); @@ -162,13 +155,17 @@ CASE("DaosStore tests") { std::string config_str{ "spaces:\n" "- roots:\n" - " - path: " + store_tests_tmp_root().asString() + "\n" + " - path: " + + store_tests_tmp_root().asString() + + "\n" "daos:\n" " store:\n" - " pool: " + pool_name + "\n" + " pool: " + + pool_name + + "\n" " client:\n" - " container_oids_per_alloc: " + std::to_string(container_oids_per_alloc) - }; + " container_oids_per_alloc: " + + std::to_string(container_oids_per_alloc)}; fdb5::Config config{YAMLConfiguration(config_str)}; @@ -189,14 +186,14 @@ CASE("DaosStore tests") { /// @todo: two cont create with label happen here /// @todo: again, daos_fini happening before cont and pool close - dstore.flush(); // not necessary for DAOS store + dstore.flush(); // not necessary for DAOS store // retrieve fdb5::Field field(std::move(loc), std::time(nullptr)); std::cout << "Read location: " << field.location() << std::endl; std::unique_ptr dh(store.retrieve(field)); EXPECT(dynamic_cast(dh.get())); - + eckit::MemoryHandle mh; dh->copyTo(mh); EXPECT(mh.size() == eckit::Length(sizeof(data))); @@ -219,7 +216,6 @@ CASE("DaosStore tests") { /// @todo: check that the URI is properly produced /// @todo: assert a new DaosSession shows newly configured container_oids_per_alloc - } SECTION("with POSIX Catalogue") { @@ -229,14 +225,20 @@ CASE("DaosStore tests") { std::string config_str{ "spaces:\n" "- roots:\n" - " - path: " + store_tests_tmp_root().asString() + "\n" - "schema : " + schema_file().path() + "\n" + " - path: " + + store_tests_tmp_root().asString() + + "\n" + "schema : " + + schema_file().path() + + "\n" "daos:\n" " store:\n" - " pool: " + pool_name + "\n" + " pool: " + + pool_name + + "\n" " client:\n" - " container_oids_per_alloc: " + std::to_string(container_oids_per_alloc) - }; + " container_oids_per_alloc: " + + std::to_string(container_oids_per_alloc)}; fdb5::Config config{YAMLConfiguration(config_str)}; @@ -269,11 +271,11 @@ CASE("DaosStore tests") { fdb5::Catalogue& cat = static_cast(tcat); cat.deselectIndex(); cat.selectIndex(index_key); - //const fdb5::Index& idx = tcat.currentIndex(); + // const fdb5::Index& idx = tcat.currentIndex(); static_cast(tcat).archive(index_key, field_key, std::move(loc)); /// flush store before flushing catalogue - dstore.flush(); // not necessary if using a DAOS store + dstore.flush(); // not necessary if using a DAOS store } // find data @@ -291,7 +293,7 @@ CASE("DaosStore tests") { std::unique_ptr dh(store.retrieve(field)); EXPECT(dynamic_cast(dh.get())); - + eckit::MemoryHandle mh; dh->copyTo(mh); EXPECT(mh.size() == eckit::Length(sizeof(data))); @@ -331,17 +333,23 @@ CASE("DaosStore tests") { std::string config_str{ "spaces:\n" "- roots:\n" - " - path: " + store_tests_tmp_root().asString() + "\n" + " - path: " + + store_tests_tmp_root().asString() + + "\n" "type: local\n" - "schema : " + schema_file().path() + "\n" + "schema : " + + schema_file().path() + + "\n" "engine: toc\n" "store: daos\n" "daos:\n" " store:\n" - " pool: " + pool_name + "\n" + " pool: " + + pool_name + + "\n" " client:\n" - " container_oids_per_alloc: " + std::to_string(container_oids_per_alloc_small) - }; + " container_oids_per_alloc: " + + std::to_string(container_oids_per_alloc_small)}; fdb5::Config config{YAMLConfiguration(config_str)}; @@ -351,21 +359,9 @@ CASE("DaosStore tests") { fdb5::Key db_key({{"a", "1"}, {"b", "2"}}); fdb5::Key index_key({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}}); - fdb5::FDBToolRequest full_req{ - request_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; - fdb5::FDBToolRequest index_req{ - index_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; - fdb5::FDBToolRequest db_req{ - db_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; + fdb5::FDBToolRequest full_req{request_key.request("retrieve"), false, std::vector{"a", "b"}}; + fdb5::FDBToolRequest index_req{index_key.request("retrieve"), false, std::vector{"a", "b"}}; + fdb5::FDBToolRequest db_req{db_key.request("retrieve"), false, std::vector{"a", "b"}}; // initialise store @@ -394,7 +390,8 @@ CASE("DaosStore tests") { char data[] = "test"; - /// @todo: here, DaosManager is being reconfigured with identical config, and it happens again multiple times below. + /// @todo: here, DaosManager is being reconfigured with identical config, and it happens again multiple times + /// below. // Should this be avoided? fdb.archive(request_key, data, sizeof(data)); @@ -404,7 +401,7 @@ CASE("DaosStore tests") { metkit::mars::MarsRequest r = request_key.request("retrieve"); std::unique_ptr dh(fdb.retrieve(r)); - + eckit::MemoryHandle mh; dh->copyTo(mh); EXPECT(mh.size() == eckit::Length(sizeof(data))); @@ -418,19 +415,22 @@ CASE("DaosStore tests") { auto wipeObject = fdb.wipe(full_req); count = 0; - while (wipeObject.next(elem)) count++; + while (wipeObject.next(elem)) + count++; EXPECT(count == 0); // dry run wipe index and store unit wipeObject = fdb.wipe(index_req); count = 0; - while (wipeObject.next(elem)) count++; + while (wipeObject.next(elem)) + count++; EXPECT(count > 0); // dry run wipe database wipeObject = fdb.wipe(db_req); count = 0; - while (wipeObject.next(elem)) count++; + while (wipeObject.next(elem)) + count++; EXPECT(count > 0); // ensure field still exists @@ -446,7 +446,8 @@ CASE("DaosStore tests") { // attempt to wipe with too specific request wipeObject = fdb.wipe(full_req, true); count = 0; - while (wipeObject.next(elem)) count++; + while (wipeObject.next(elem)) + count++; EXPECT(count == 0); /// @todo: really needed? fdb.flush(); @@ -454,7 +455,8 @@ CASE("DaosStore tests") { // wipe index and store unit (and DB container as there is only one index) wipeObject = fdb.wipe(index_req, true); count = 0; - while (wipeObject.next(elem)) count++; + while (wipeObject.next(elem)) + count++; EXPECT(count > 0); /// @todo: really needed? fdb.flush(); @@ -462,7 +464,8 @@ CASE("DaosStore tests") { // ensure field does not exist listObject = fdb.list(full_req); count = 0; - while (listObject.next(info)) count++; + while (listObject.next(info)) + count++; EXPECT(count == 0); /// @todo: ensure index and corresponding container do not exist @@ -471,7 +474,6 @@ CASE("DaosStore tests") { /// ensure DB still exists with one index /// @todo: ensure new DaosSession has updated daos client config - } /// @todo: if doing what's in this section at the end of the previous section reusing the same FDB object, @@ -483,17 +485,23 @@ CASE("DaosStore tests") { std::string config_str{ "spaces:\n" "- roots:\n" - " - path: " + store_tests_tmp_root().asString() + "\n" + " - path: " + + store_tests_tmp_root().asString() + + "\n" "type: local\n" - "schema : " + schema_file().path() + "\n" + "schema : " + + schema_file().path() + + "\n" "engine: toc\n" "store: daos\n" "daos:\n" " store:\n" - " pool: " + pool_name + "\n" + " pool: " + + pool_name + + "\n" " client:\n" - " container_oids_per_alloc: " + std::to_string(container_oids_per_alloc) - }; + " container_oids_per_alloc: " + + std::to_string(container_oids_per_alloc)}; fdb5::Config config{YAMLConfiguration(config_str)}; @@ -503,21 +511,9 @@ CASE("DaosStore tests") { fdb5::Key db_key({{"a", "1"}, {"b", "2"}}); fdb5::Key index_key({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}}); - fdb5::FDBToolRequest full_req{ - request_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; - fdb5::FDBToolRequest index_req{ - index_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; - fdb5::FDBToolRequest db_req{ - db_key.request("retrieve"), - false, - std::vector{"a", "b"} - }; + fdb5::FDBToolRequest full_req{request_key.request("retrieve"), false, std::vector{"a", "b"}}; + fdb5::FDBToolRequest index_req{index_key.request("retrieve"), false, std::vector{"a", "b"}}; + fdb5::FDBToolRequest db_req{db_key.request("retrieve"), false, std::vector{"a", "b"}}; // initialise store @@ -528,17 +524,18 @@ CASE("DaosStore tests") { char data[] = "test"; fdb.archive(request_key, data, sizeof(data)); - + fdb.flush(); size_t count; - + // wipe all database fdb5::WipeElement elem; auto wipeObject = fdb.wipe(db_req, true); count = 0; - while (wipeObject.next(elem)) count++; + while (wipeObject.next(elem)) + count++; EXPECT(count > 0); /// @todo: really needed? fdb.flush(); @@ -556,13 +553,12 @@ CASE("DaosStore tests") { EXPECT(count == 0); /// @todo: ensure DB and corresponding cont do not exist - } // teardown daos #ifdef fdb5_HAVE_DAOS_ADMIN - /// AutoPoolDestroy is not possible here because the pool is + /// AutoPoolDestroy is not possible here because the pool is /// created above with an ephemeral session fdb5::DaosSession().destroyPool(pool_uuid); #else @@ -570,13 +566,11 @@ CASE("DaosStore tests") { if (c == "1:2") fdb5::DaosSession().getPool(pool_uuid).destroyContainer(c); #endif - } -} // namespace test -} // namespace fdb +} // namespace test +} // namespace fdb -int main(int argc, char **argv) -{ - return run_tests ( argc, argv ); +int main(int argc, char** argv) { + return run_tests(argc, argv); } diff --git a/tests/fdb/daos/test_dummy_daos_write_read.cc b/tests/fdb/daos/test_dummy_daos_write_read.cc index 33d4f44ac..ce2961e29 100644 --- a/tests/fdb/daos/test_dummy_daos_write_read.cc +++ b/tests/fdb/daos/test_dummy_daos_write_read.cc @@ -14,12 +14,12 @@ #include #include -#include "eckit/testing/Test.h" #include "eckit/filesystem/TmpDir.h" +#include "eckit/testing/Test.h" #include "daos.h" -#include "dummy_daos.h" #include "daos/tests_lib.h" +#include "dummy_daos.h" using namespace eckit::testing; using namespace eckit; @@ -34,14 +34,13 @@ namespace test { //---------------------------------------------------------------------------------------------------------------------- -CASE( "Setup" ) { +CASE("Setup") { tmp_dummy_daos_root().mkdir(); ::setenv("DUMMY_DAOS_DATA_ROOT", tmp_dummy_daos_root().path().c_str(), 1); - } -CASE( "dummy_daos_write_then_read" ) { +CASE("dummy_daos_write_then_read") { int rc; @@ -54,7 +53,7 @@ CASE( "dummy_daos_write_then_read" ) { // create a pool with user-defined label std::string label = "test_pool"; - daos_prop_t *prop = NULL; + daos_prop_t* prop = NULL; prop = daos_prop_alloc(1); prop->dpp_entries[0].dpe_type = DAOS_PROP_PO_LABEL; D_STRNDUP(prop->dpp_entries[0].dpe_str, label.c_str(), DAOS_PROP_LABEL_MAX_LEN); @@ -94,7 +93,7 @@ CASE( "dummy_daos_write_then_read" ) { rc = daos_pool_connect(pool.c_str(), NULL, DAOS_PC_RW, &poh, NULL, NULL); EXPECT(rc == 0); EXPECT(dummy_daos_get_handle_path(poh) == pool_path); - + // create, open and close a container with auto-generated uuid uuid_t cont_uuid = {0}; @@ -158,7 +157,7 @@ CASE( "dummy_daos_write_then_read" ) { daos_obj_id_t oid_kv; rc = daos_cont_alloc_oids(coh, oid_alloc_size, &oid_kv.lo, NULL); EXPECT(rc == 0); - //EXPECT(oid_kv.lo == ((((uint64_t) getpid()) << 48) | (uint64_t) 0)); + // EXPECT(oid_kv.lo == ((((uint64_t) getpid()) << 48) | (uint64_t) 0)); rc = daos_obj_generate_oid(coh, &oid_kv, DAOS_OT_KV_HASHED, OC_S1, 0, 0); EXPECT(rc == 0); @@ -177,7 +176,7 @@ CASE( "dummy_daos_write_then_read" ) { rc = daos_kv_get(oh_kv, DAOS_TX_NONE, 0, key.c_str(), &size, NULL, NULL); EXPECT(rc == 0); - EXPECT(size == (daos_size_t) 0); + EXPECT(size == (daos_size_t)0); rc = daos_kv_put(oh_kv, DAOS_TX_NONE, 0, key.c_str(), std::strlen(value.c_str()), value.c_str(), NULL); EXPECT(rc == 0); @@ -186,10 +185,10 @@ CASE( "dummy_daos_write_then_read" ) { char kv_get_buf[100] = ""; rc = daos_kv_get(oh_kv, DAOS_TX_NONE, 0, key.c_str(), &size, NULL, NULL); EXPECT(rc == 0); - EXPECT(size == (daos_size_t) std::strlen(value.c_str())); + EXPECT(size == (daos_size_t)std::strlen(value.c_str())); rc = daos_kv_get(oh_kv, DAOS_TX_NONE, 0, key.c_str(), &size, kv_get_buf, NULL); EXPECT(rc == 0); - EXPECT(size == (daos_size_t) std::strlen(value.c_str())); + EXPECT(size == (daos_size_t)std::strlen(value.c_str())); EXPECT(std::strlen(kv_get_buf) == std::strlen(value.c_str())); EXPECT(std::string(kv_get_buf) == value); @@ -203,9 +202,9 @@ CASE( "dummy_daos_write_then_read" ) { daos_key_desc_t key_sizes[max_keys_per_rpc]; d_sg_list_t sgl_kv_list; d_iov_t iov_kv_list; - char *list_buf; + char* list_buf; int bufsize = 1024; - list_buf = (char*) malloc(bufsize); + list_buf = (char*)malloc(bufsize); d_iov_set(&iov_kv_list, list_buf, bufsize); sgl_kv_list.sg_nr = 1; sgl_kv_list.sg_nr_out = 0; @@ -240,7 +239,7 @@ CASE( "dummy_daos_write_then_read" ) { daos_obj_id_t oid; rc = daos_cont_alloc_oids(coh, oid_alloc_size, &oid.lo, NULL); EXPECT(rc == 0); - //EXPECT(oid.lo == ((((uint64_t) getpid()) << 48) | (uint64_t) 1)); + // EXPECT(oid.lo == ((((uint64_t) getpid()) << 48) | (uint64_t) 1)); rc = daos_array_generate_oid(coh, &oid, true, OC_S1, 0, 0); EXPECT(rc == 0); @@ -263,12 +262,12 @@ CASE( "dummy_daos_write_then_read" ) { d_iov_t iov; iod.arr_nr = 1; - rg.rg_len = (daos_size_t) sizeof(data); - rg.rg_idx = (daos_off_t) 0; + rg.rg_len = (daos_size_t)sizeof(data); + rg.rg_idx = (daos_off_t)0; iod.arr_rgs = &rg; sgl.sg_nr = 1; - d_iov_set(&iov, data, (size_t) sizeof(data)); + d_iov_set(&iov, data, (size_t)sizeof(data)); sgl.sg_iovs = &iov; rc = daos_array_write(oh, DAOS_TX_NONE, &iod, &sgl, NULL); @@ -284,22 +283,22 @@ CASE( "dummy_daos_write_then_read" ) { rc = daos_array_open(coh, oid, DAOS_TX_NONE, DAOS_OO_RW, &cell_size, &csize, &oh, NULL); EXPECT(rc == 0); - char *data_read; + char* data_read; daos_size_t array_size; rc = daos_array_get_size(oh, DAOS_TX_NONE, &array_size, NULL); EXPECT(rc == 0); EXPECT(array_size == rg.rg_len); - data_read = (char *) malloc(sizeof(char) * ((size_t) array_size)); + data_read = (char*)malloc(sizeof(char) * ((size_t)array_size)); iod.arr_nr = 1; rg.rg_len = array_size; - rg.rg_idx = (daos_off_t) 0; + rg.rg_idx = (daos_off_t)0; iod.arr_rgs = &rg; sgl.sg_nr = 1; - d_iov_set(&iov, data_read, (size_t) array_size); + d_iov_set(&iov, data_read, (size_t)array_size); sgl.sg_iovs = &iov; rc = daos_array_read(oh, DAOS_TX_NONE, &iod, &sgl, NULL); @@ -314,9 +313,8 @@ CASE( "dummy_daos_write_then_read" ) { // container list OIDs daos_epoch_t e; - rc = daos_cont_create_snap_opt( - coh, &e, NULL, (enum daos_snapshot_opts)(DAOS_SNAP_OPT_CR | DAOS_SNAP_OPT_OIT), NULL - ); + rc = + daos_cont_create_snap_opt(coh, &e, NULL, (enum daos_snapshot_opts)(DAOS_SNAP_OPT_CR | DAOS_SNAP_OPT_OIT), NULL); EXPECT(rc == 0); std::stringstream os_epoch; os_epoch << std::setw(16) << std::setfill('0') << std::hex << e; @@ -334,7 +332,8 @@ CASE( "dummy_daos_write_then_read" ) { uint32_t oids_nr = max_oids_per_rpc; rc = daos_oit_list(oith, oid_batch, &oids_nr, &anchor, NULL); EXPECT(rc == 0); - for (int i = 0; i < oids_nr; i++) oids.push_back(oid_batch[i]); + for (int i = 0; i < oids_nr; i++) + oids.push_back(oid_batch[i]); } EXPECT(oids.size() == 2); EXPECT(std::memcmp(&oids[0], &oid_kv, sizeof(daos_obj_id_t)) == 0); @@ -358,7 +357,7 @@ CASE( "dummy_daos_write_then_read" ) { daos_obj_close(oh_kv, NULL); EXPECT(rc == 0); - + rc = daos_array_open(coh, oid, DAOS_TX_NONE, DAOS_OO_RW, &cell_size, &csize, &oh, NULL); EXPECT(rc == 0); @@ -396,15 +395,13 @@ CASE( "dummy_daos_write_then_read" ) { D_FREE(svcl.rl_ranks); #endif - } //---------------------------------------------------------------------------------------------------------------------- -} // namespace test -} // namespace fdb +} // namespace test +} // namespace fdb -int main(int argc, char **argv) -{ - return run_tests ( argc, argv ); +int main(int argc, char** argv) { + return run_tests(argc, argv); } diff --git a/tests/fdb/database/test_indexaxis.cc b/tests/fdb/database/test_indexaxis.cc index de056df6d..ebf0ab399 100644 --- a/tests/fdb/database/test_indexaxis.cc +++ b/tests/fdb/database/test_indexaxis.cc @@ -10,23 +10,11 @@ namespace { -fdb5::Key EXAMPLE_K1{{ - {"class", "od"}, - {"expver", "0001"}, - {"date", "20240223"} -}}; - -fdb5::Key EXAMPLE_K2{{ - {"class", "rd"}, - {"expver", "0001"}, - {"time", "1200"} -}}; - -fdb5::Key EXAMPLE_K3{{ - {"class", "rd"}, - {"expver", "gotx"}, - {"time", "0000"} -}}; +fdb5::Key EXAMPLE_K1{{{"class", "od"}, {"expver", "0001"}, {"date", "20240223"}}}; + +fdb5::Key EXAMPLE_K2{{{"class", "rd"}, {"expver", "0001"}, {"time", "1200"}}}; + +fdb5::Key EXAMPLE_K3{{{"class", "rd"}, {"expver", "gotx"}, {"time", "0000"}}}; //---------------------------------------------------------------------------------------------------------------------- @@ -52,7 +40,7 @@ CASE("Insertion and comparison") { ia2.insert(EXAMPLE_K1); EXPECT(!(ia1 == ia2)); EXPECT(ia1 != ia2); - + ia1.sort(); ia2.sort(); EXPECT(ia1 == ia2); @@ -91,7 +79,9 @@ CASE("iostream and JSON output functions correctly") { std::ostringstream ss; eckit::JSON json(ss); json << ia; - EXPECT(ss.str() == "{\"class\":[\"od\",\"rd\"],\"date\":[\"20240223\"],\"expver\":[\"0001\",\"gotx\"],\"time\":[\"0000\",\"1200\"]}"); + EXPECT(ss.str() == + "{\"class\":[\"od\",\"rd\"],\"date\":[\"20240223\"],\"expver\":[\"0001\",\"gotx\"],\"time\":[\"0000\"," + "\"1200\"]}"); } } diff --git a/tests/fdb/test_fdb5_service.cc b/tests/fdb/test_fdb5_service.cc index 6956d79a5..361a346dc 100644 --- a/tests/fdb/test_fdb5_service.cc +++ b/tests/fdb/test_fdb5_service.cc @@ -17,26 +17,26 @@ #include #include +#include "eckit/io/AutoCloser.h" #include "eckit/io/DataHandle.h" #include "eckit/io/FileHandle.h" #include "eckit/io/MemoryHandle.h" #include "eckit/io/MultiHandle.h" #include "eckit/runtime/Main.h" +#include "eckit/serialisation/FileStream.h" +#include "eckit/serialisation/MemoryStream.h" #include "eckit/types/Types.h" #include "eckit/utils/Translator.h" -#include "eckit/serialisation/MemoryStream.h" -#include "eckit/serialisation/FileStream.h" -#include "eckit/io/AutoCloser.h" -#include "metkit/mars/MarsRequest.h" #include "metkit/mars/MarsExpension.h" +#include "metkit/mars/MarsRequest.h" #include "metkit/mars/TypeAny.h" -#include "fdb5/database/Key.h" -#include "fdb5/database/Archiver.h" -#include "fdb5/database/ArchiveVisitor.h" #include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/database/ArchiveVisitor.h" +#include "fdb5/database/Archiver.h" +#include "fdb5/database/Key.h" #include "eckit/testing/Test.h" @@ -45,7 +45,6 @@ using namespace eckit; using namespace eckit::testing; using namespace fdb5; - namespace fdb { namespace test { @@ -59,9 +58,8 @@ struct FixtureService { std::vector modelParams_; - FixtureService() : env("environ") - { - p["class"] = "rd"; + FixtureService() : env("environ") { + p["class"] = "rd"; p["stream"] = "oper"; p["domain"] = "g"; p["expver"] = "0001"; @@ -69,38 +67,31 @@ struct FixtureService { p["time"] = "0000"; p["type"] = "fc"; - modelParams_.push_back( "130.128" ); - modelParams_.push_back( "138.128" ); + modelParams_.push_back("130.128"); + modelParams_.push_back("138.128"); } - void write_cycle(fdb5::Archiver& fdb, StringDict& p ) - { - Translator str; + void write_cycle(fdb5::Archiver& fdb, StringDict& p) { + Translator str; std::vector::iterator param = modelParams_.begin(); - for( ; param != modelParams_.end(); ++param ) - { + for (; param != modelParams_.end(); ++param) { p["param"] = *param; p["levtype"] = "pl"; - for( size_t step = 0; step < 12; ++step ) - { - p["step"] = str(step*3); + for (size_t step = 0; step < 12; ++step) { + p["step"] = str(step * 3); - for( size_t level = 0; level < 10; ++level ) - { - p["levelist"] = str(level*100); + for (size_t level = 0; level < 10; ++level) { + p["levelist"] = str(level * 100); std::ostringstream data; data << "Raining cats and dogs -- " - << " param " << *param - << " step " << step - << " level " << level - << std::endl; + << " param " << *param << " step " << step << " level " << level << std::endl; std::string data_str = data.str(); fdb5::Key k{p}; - ArchiveVisitor visitor(fdb, k, static_cast(data_str.c_str()), data_str.size()); + ArchiveVisitor visitor(fdb, k, static_cast(data_str.c_str()), data_str.size()); fdb.archive(k, visitor); } } @@ -110,24 +101,24 @@ struct FixtureService { //---------------------------------------------------------------------------------------------------------------------- -CASE ( "test_fdb_stepunit_archive" ) { +CASE("test_fdb_stepunit_archive") { std::string data_str = "Raining cats and dogs"; fdb5::FDB fdb; Key key; - key.set("class","od"); - key.set("expver","0001"); - key.set("type","fc"); - key.set("stream","oper"); - key.set("date","20101010"); - key.set("time","0000"); - key.set("domain","g"); - key.set("levtype","sfc"); - key.set("step","60"); - key.set("param","130"); - fdb.archive(key, static_cast(data_str.c_str()), data_str.size()); + key.set("class", "od"); + key.set("expver", "0001"); + key.set("type", "fc"); + key.set("stream", "oper"); + key.set("date", "20101010"); + key.set("time", "0000"); + key.set("domain", "g"); + key.set("levtype", "sfc"); + key.set("step", "60"); + key.set("param", "130"); + fdb.archive(key, static_cast(data_str.c_str()), data_str.size()); fdb.flush(); metkit::mars::MarsRequest req = key.request(); @@ -141,9 +132,9 @@ CASE ( "test_fdb_stepunit_archive" ) { EXPECT(!iter.next(el)); } - key.set("step","2"); - key.set("stepunits","h"); - fdb.archive(key, static_cast(data_str.c_str()), data_str.size()); + key.set("step", "2"); + key.set("stepunits", "h"); + fdb.archive(key, static_cast(data_str.c_str()), data_str.size()); fdb.flush(); req.setValue("step", "2"); @@ -156,7 +147,7 @@ CASE ( "test_fdb_stepunit_archive" ) { EXPECT(!iter.next(el)); } - req.setValuesTyped(new metkit::mars::TypeAny("step"), std::vector{"2","60"}); + req.setValuesTyped(new metkit::mars::TypeAny("step"), std::vector{"2", "60"}); { fdb5::FDBToolRequest r(req); fdb5::ListIterator iter = fdb.list(r, true); @@ -168,9 +159,9 @@ CASE ( "test_fdb_stepunit_archive" ) { EXPECT(!iter.next(el)); } - key.set("step","30"); - key.set("stepunits","m"); - fdb.archive(key, static_cast(data_str.c_str()), data_str.size()); + key.set("step", "30"); + key.set("stepunits", "m"); + fdb.archive(key, static_cast(data_str.c_str()), data_str.size()); fdb.flush(); req.setValue("step", "30m"); @@ -183,7 +174,7 @@ CASE ( "test_fdb_stepunit_archive" ) { EXPECT(!iter.next(el)); } - req.values("step", {"0","to","2","by","30m"}); + req.values("step", {"0", "to", "2", "by", "30m"}); req.unsetValues("param"); { metkit::mars::MarsExpension expand{false}; @@ -200,16 +191,15 @@ CASE ( "test_fdb_stepunit_archive" ) { } } -CASE ( "test_fdb_service" ) { +CASE("test_fdb_service") { SETUP("Fixture") { FixtureService f; - SECTION( "test_fdb_service_write" ) - { + SECTION("test_fdb_service_write") { fdb5::Archiver fdb; - f.p["class"] = "rd"; + f.p["class"] = "rd"; f.p["stream"] = "oper"; f.p["domain"] = "g"; f.p["expver"] = "0001"; @@ -239,46 +229,39 @@ CASE ( "test_fdb_service" ) { f.write_cycle(fdb, f.p); } - - SECTION( "test_fdb_service_readtobuffer" ) - { + SECTION("test_fdb_service_readtobuffer") { fdb5::FDB retriever; Buffer buffer(1024); - Translator str; + Translator str; std::vector::iterator param = f.modelParams_.begin(); - for( ; param != f.modelParams_.end(); ++param ) - { + for (; param != f.modelParams_.end(); ++param) { f.p["param"] = *param; f.p["levtype"] = "pl"; - for( size_t step = 0; step < 2; ++step ) - { - f.p["step"] = str(step*3); + for (size_t step = 0; step < 2; ++step) { + f.p["step"] = str(step * 3); - for( size_t level = 0; level < 3; ++level ) - { - f.p["levelist"] = str(level*100); + for (size_t level = 0; level < 3; ++level) { + f.p["levelist"] = str(level * 100); Log::info() << "Looking for: " << f.p << std::endl; metkit::mars::MarsRequest r("retrieve", f.p); - std::unique_ptr dh ( retriever.retrieve(r) ); AutoClose closer1(*dh); + std::unique_ptr dh(retriever.retrieve(r)); + AutoClose closer1(*dh); ::memset(buffer, 0, buffer.size()); dh->openForRead(); dh->read(buffer, buffer.size()); - Log::info() << (char*) buffer << std::endl; + Log::info() << (char*)buffer << std::endl; std::ostringstream data; data << "Raining cats and dogs -- " - << " param " << *param - << " step " << step - << " level " << level - << std::endl; + << " param " << *param << " step " << step << " level " << level << std::endl; EXPECT(::memcmp(buffer, data.str().c_str(), data.str().size()) == 0); } @@ -286,15 +269,12 @@ CASE ( "test_fdb_service" ) { } } - - SECTION( "test_fdb_service_list" ) - { + SECTION("test_fdb_service_list") { fdb5::FDB lister; - Translator str; + Translator str; std::vector::iterator param = f.modelParams_.begin(); - for( ; param != f.modelParams_.end(); ++param ) - { + for (; param != f.modelParams_.end(); ++param) { f.p["param"] = *param; f.p["levtype"] = "pl"; f.p["step"] = str(0); @@ -313,43 +293,42 @@ CASE ( "test_fdb_service" ) { DIR* dirp = ::opendir(path.asString().c_str()); struct dirent* dp; while ((dp = ::readdir(dirp)) != nullptr) { - EXPECT_NOT(strstr( dp->d_name, "toc.")); + EXPECT_NOT(strstr(dp->d_name, "toc.")); } // consuming the rest of the queue - while (iter.next(el)); + while (iter.next(el)) + ; } } - SECTION( "test_fdb_service_marsreques" ) - { + SECTION("test_fdb_service_marsreques") { std::vector steps; - steps.push_back( "15" ); - steps.push_back( "18" ); - steps.push_back( "24" ); + steps.push_back("15"); + steps.push_back("18"); + steps.push_back("24"); std::vector levels; - levels.push_back( "100" ); - levels.push_back( "500" ); + levels.push_back("100"); + levels.push_back("500"); std::vector params; - params.push_back( "130.128" ); - params.push_back( "138.128" ); + params.push_back("130.128"); + params.push_back("138.128"); std::vector dates; - dates.push_back( "20120911" ); - dates.push_back( "20120912" ); - + dates.push_back("20120911"); + dates.push_back("20120912"); metkit::mars::MarsRequest r("retrieve"); - r.setValue("class","rd"); - r.setValue("expver","0001"); - r.setValue("type","fc"); - r.setValue("stream","oper"); - r.setValue("time","0000"); - r.setValue("domain","g"); - r.setValue("levtype","pl"); + r.setValue("class", "rd"); + r.setValue("expver", "0001"); + r.setValue("type", "fc"); + r.setValue("stream", "oper"); + r.setValue("time", "0000"); + r.setValue("domain", "g"); + r.setValue("levtype", "pl"); r.setValuesTyped(new metkit::mars::TypeAny("param"), params); r.setValuesTyped(new metkit::mars::TypeAny("step"), steps); @@ -360,9 +339,9 @@ CASE ( "test_fdb_service" ) { fdb5::FDB retriever; - std::unique_ptr dh ( retriever.retrieve(r) ); + std::unique_ptr dh(retriever.retrieve(r)); - PathName path ( "data_mars_request.data" ); + PathName path("data_mars_request.data"); path.unlink(); dh->saveInto(path); @@ -370,9 +349,7 @@ CASE ( "test_fdb_service" ) { } } - - -CASE ( "test_fdb_service_subtoc" ) { +CASE("test_fdb_service_subtoc") { SETUP("Fixture") { FixtureService f; @@ -384,11 +361,10 @@ CASE ( "test_fdb_service_subtoc" ) { fdb5::Config config(expanded, userConf); - SECTION( "test_fdb_service_subtoc_write" ) - { + SECTION("test_fdb_service_subtoc_write") { fdb5::Archiver fdb(config); - f.p["class"] = "rd"; + f.p["class"] = "rd"; f.p["stream"] = "oper"; f.p["domain"] = "g"; f.p["expver"] = "0002"; @@ -418,47 +394,41 @@ CASE ( "test_fdb_service_subtoc" ) { f.write_cycle(fdb, f.p); } - SECTION( "test_fdb_service_subtoc_readtobuffer" ) - { + SECTION("test_fdb_service_subtoc_readtobuffer") { fdb5::FDB retriever; Buffer buffer(1024); f.p["expver"] = "0002"; - Translator str; + Translator str; std::vector::iterator param = f.modelParams_.begin(); - for( ; param != f.modelParams_.end(); ++param ) - { + for (; param != f.modelParams_.end(); ++param) { f.p["param"] = *param; f.p["levtype"] = "pl"; - for( size_t step = 0; step < 2; ++step ) - { - f.p["step"] = str(step*3); + for (size_t step = 0; step < 2; ++step) { + f.p["step"] = str(step * 3); - for( size_t level = 0; level < 3; ++level ) - { - f.p["levelist"] = str(level*100); + for (size_t level = 0; level < 3; ++level) { + f.p["levelist"] = str(level * 100); Log::info() << "Looking for: " << f.p << std::endl; metkit::mars::MarsRequest r("retrieve", f.p); - std::unique_ptr dh ( retriever.retrieve(r) ); AutoClose closer1(*dh); + std::unique_ptr dh(retriever.retrieve(r)); + AutoClose closer1(*dh); ::memset(buffer, 0, buffer.size()); dh->openForRead(); dh->read(buffer, buffer.size()); - Log::info() << (char*) buffer << std::endl; + Log::info() << (char*)buffer << std::endl; std::ostringstream data; data << "Raining cats and dogs -- " - << " param " << *param - << " step " << step - << " level " << level - << std::endl; + << " param " << *param << " step " << step << " level " << level << std::endl; EXPECT(::memcmp(buffer, data.str().c_str(), data.str().size()) == 0); } @@ -466,17 +436,14 @@ CASE ( "test_fdb_service_subtoc" ) { } } - - SECTION( "test_fdb_service_subtoc_list" ) - { + SECTION("test_fdb_service_subtoc_list") { fdb5::FDB lister; f.p["expver"] = "0002"; - Translator str; + Translator str; std::vector::iterator param = f.modelParams_.begin(); - for( ; param != f.modelParams_.end(); ++param ) - { + for (; param != f.modelParams_.end(); ++param) { f.p["param"] = *param; f.p["levtype"] = "pl"; @@ -497,46 +464,45 @@ CASE ( "test_fdb_service_subtoc" ) { struct dirent* dp; bool subtoc = false; while ((dp = ::readdir(dirp)) != nullptr) { - if (strstr( dp->d_name, "toc.")) { + if (strstr(dp->d_name, "toc.")) { subtoc = true; } } EXPECT(subtoc); // consuming the rest of the queue - while (iter.next(el)); + while (iter.next(el)) + ; } } - SECTION( "test_fdb_service_subtoc_marsreques" ) - { + SECTION("test_fdb_service_subtoc_marsreques") { std::vector steps; - steps.push_back( "15" ); - steps.push_back( "18" ); - steps.push_back( "24" ); + steps.push_back("15"); + steps.push_back("18"); + steps.push_back("24"); std::vector levels; - levels.push_back( "100" ); - levels.push_back( "500" ); + levels.push_back("100"); + levels.push_back("500"); std::vector params; - params.push_back( "130.128" ); - params.push_back( "138.128" ); + params.push_back("130.128"); + params.push_back("138.128"); std::vector dates; - dates.push_back( "20120911" ); - dates.push_back( "20120912" ); - + dates.push_back("20120911"); + dates.push_back("20120912"); metkit::mars::MarsRequest r("retrieve"); - r.setValue("class","rd"); - r.setValue("expver","0002"); - r.setValue("type","fc"); - r.setValue("stream","oper"); - r.setValue("time","0000"); - r.setValue("domain","g"); - r.setValue("levtype","pl"); + r.setValue("class", "rd"); + r.setValue("expver", "0002"); + r.setValue("type", "fc"); + r.setValue("stream", "oper"); + r.setValue("time", "0000"); + r.setValue("domain", "g"); + r.setValue("levtype", "pl"); r.setValuesTyped(new metkit::mars::TypeAny("param"), params); r.setValuesTyped(new metkit::mars::TypeAny("step"), steps); @@ -547,9 +513,9 @@ CASE ( "test_fdb_service_subtoc" ) { fdb5::FDB retriever; - std::unique_ptr dh ( retriever.retrieve(r) ); + std::unique_ptr dh(retriever.retrieve(r)); - PathName path ( "data_mars_request.data" ); + PathName path("data_mars_request.data"); path.unlink(); dh->saveInto(path); @@ -557,9 +523,9 @@ CASE ( "test_fdb_service_subtoc" ) { } } -CASE( "schemaSerialisation" ) { +CASE("schemaSerialisation") { - PathName filename = PathName::unique("data"); + PathName filename = PathName::unique("data"); std::string filepath = filename.asString(); std::string original; @@ -595,11 +561,10 @@ CASE( "schemaSerialisation" ) { //----------------------------------------------------------------------------- -} // namespace test -} // namespace fdb +} // namespace test +} // namespace fdb -int main(int argc, char **argv) -{ +int main(int argc, char** argv) { eckit::Main::initialise(argc, argv, "FDB_HOME"); - return run_tests ( argc, argv, false ); + return run_tests(argc, argv, false); } diff --git a/tests/fdb/type/test_toKey.cc b/tests/fdb/type/test_toKey.cc index 22e255f61..779f7e0c5 100644 --- a/tests/fdb/type/test_toKey.cc +++ b/tests/fdb/type/test_toKey.cc @@ -13,22 +13,21 @@ #include "eckit/testing/Test.h" #include "fdb5/config/Config.h" -#include "fdb5/database/Archiver.h" #include "fdb5/database/ArchiveVisitor.h" +#include "fdb5/database/Archiver.h" #include "fdb5/rules/Rule.h" using namespace eckit::testing; using namespace eckit; - namespace fdb { namespace test { fdb5::Config config; char data[4]; -CASE( "ClimateDaily - no expansion" ) { +CASE("ClimateDaily - no expansion") { fdb5::Key key{}; EXPECT(key.valuesToString() == ""); @@ -42,10 +41,9 @@ CASE( "ClimateDaily - no expansion" ) { key.set("stream", "dacl"); EXPECT(key.canonicalValue("date") == "20210427"); EXPECT(key.valuesToString() == "20210427:dacl"); - } -CASE( "Step & ClimateDaily - expansion" ) { +CASE("Step & ClimateDaily - expansion") { fdb5::TypedKey key(config.schema().registry()); EXPECT(key.valuesToString() == ""); @@ -159,8 +157,7 @@ CASE( "Step & ClimateDaily - expansion" ) { EXPECT(key.canonicalValue("step") == "1-2"); } - -CASE( "Levelist" ) { +CASE("Levelist") { eckit::DenseSet values; values.insert("100"); values.insert("200"); @@ -217,15 +214,16 @@ CASE( "Levelist" ) { // this works (probably shouldn't), simply becasue to_string uses the same precision as printf %f (default 6) /// @note don't use to_string when canonicalising Keys - key.set("levelist", std::to_string(double(1./3.))); + key.set("levelist", std::to_string(double(1. / 3.))); EXPECT(key.canonicalValue("levelist") == "0.333333"); EXPECT(key.match("levelist", values)); } -CASE( "Expver, Time & ClimateDaily - string ctor - expansion" ) { +CASE("Expver, Time & ClimateDaily - string ctor - expansion") { fdb5::TypedKey key = fdb5::TypedKey::parseString( - "class=ei,expver=1,stream=dacl,domain=g,type=pb,levtype=pl,date=20210427,time=6,step=0,quantile=99:100,levelist=50,param=129.128", + "class=ei,expver=1,stream=dacl,domain=g,type=pb,levtype=pl,date=20210427,time=6,step=0,quantile=99:100," + "levelist=50,param=129.128", config.schema().registry()); EXPECT(key.canonicalValue("date") == "20210427"); @@ -241,7 +239,7 @@ CASE( "Expver, Time & ClimateDaily - string ctor - expansion" ) { EXPECT(key.valuesToString() == "ei:0001:dacl:g:pb:pl:0427:0600:0:99:100:50:129.128"); } -CASE( "ClimateMonthly - string ctor - expansion" ) { +CASE("ClimateMonthly - string ctor - expansion") { fdb5::TypedKey key = fdb5::TypedKey::parseString( "class=op,expver=1,stream=mnth,domain=g,type=cl,levtype=pl,date=20210427,time=0000,levelist=50,param=129.128", @@ -257,21 +255,20 @@ CASE( "ClimateMonthly - string ctor - expansion" ) { EXPECT(key.canonicalValue("date") == "4"); EXPECT(key.valuesToString() == "op:0001:mnth:g:cl:pl:4:0000:50:129.128"); - } // do we need to keep this behaviour? should we rely on metkit for date expansion and remove it from TypedKey? -CASE( "Date - string ctor - expansion" ) { +CASE("Date - string ctor - expansion") { fdb5::TypedKey key = fdb5::TypedKey::parseString( - "class=od,expver=1,stream=oper,type=ofb,date=-2,time=0000,obsgroup=MHS,reportype=3001", - config.schema().registry()); + "class=od,expver=1,stream=oper,type=ofb,date=-2,time=0000,obsgroup=MHS,reportype=3001", + config.schema().registry()); eckit::Date now(-2); eckit::Translator t; - EXPECT(key.canonicalValue("date") == t(now.yyyymmdd())); - EXPECT(key.valuesToString() == "od:0001:oper:ofb:"+t(now.yyyymmdd())+":0000:mhs:3001"); + EXPECT(key.canonicalValue("date") == t(now.yyyymmdd())); + EXPECT(key.valuesToString() == "od:0001:oper:ofb:" + t(now.yyyymmdd()) + ":0000:mhs:3001"); fdb5::Archiver archiver; fdb5::ArchiveVisitor visitor(archiver, key.canonical(), data, 4); @@ -279,17 +276,14 @@ CASE( "Date - string ctor - expansion" ) { key.registry(visitor.rule()->registry()); EXPECT(key.canonicalValue("date") == t(now.yyyymmdd())); - EXPECT(key.valuesToString() == "od:0001:oper:ofb:"+t(now.yyyymmdd())+":0000:mhs:3001"); - + EXPECT(key.valuesToString() == "od:0001:oper:ofb:" + t(now.yyyymmdd()) + ":0000:mhs:3001"); } - //---------------------------------------------------------------------------------------------------------------------- -} // namespace test -} // namespace fdb +} // namespace test +} // namespace fdb -int main(int argc, char **argv) -{ - return run_tests ( argc, argv ); +int main(int argc, char** argv) { + return run_tests(argc, argv); } diff --git a/tests/regressions/FDB-310/fdb-url.cc b/tests/regressions/FDB-310/fdb-url.cc index f4933b657..2ecd2a611 100644 --- a/tests/regressions/FDB-310/fdb-url.cc +++ b/tests/regressions/FDB-310/fdb-url.cc @@ -15,33 +15,31 @@ #include "eckit/option/CmdArgs.h" #include "eckit/option/SimpleOption.h" -#include "metkit/mars/MarsRequest.h" -#include "metkit/mars/MarsParser.h" #include "metkit/mars/MarsExpension.h" +#include "metkit/mars/MarsParser.h" +#include "metkit/mars/MarsRequest.h" #include "fdb5/LibFdb5.h" #include "fdb5/api/FDB.h" -#include "fdb5/message/MessageDecoder.h" #include "fdb5/io/HandleGatherer.h" +#include "fdb5/message/MessageDecoder.h" #include "fdb5/tools/FDBTool.h" - class FDBUrl : public fdb5::FDBTool { - virtual void execute(const eckit::option::CmdArgs &args); - virtual void usage(const std::string &tool) const; + virtual void execute(const eckit::option::CmdArgs& args); + virtual void usage(const std::string& tool) const; virtual int numberOfPositionalArguments() const { return 2; } - public: - FDBUrl(int argc, char **argv): fdb5::FDBTool(argc, argv) { + +public: + FDBUrl(int argc, char** argv) : fdb5::FDBTool(argc, argv) { options_.push_back(new eckit::option::SimpleOption("uri", "Inspect -> URI -> Read")); options_.push_back(new eckit::option::SimpleOption("extract", "Extract request from a GRIB file")); options_.push_back(new eckit::option::SimpleOption("raw", "Uses the raw request, without expansion")); - options_.push_back( - new eckit::option::SimpleOption("statistics", - "Report timing statistics")); + options_.push_back(new eckit::option::SimpleOption("statistics", "Report timing statistics")); } }; -void FDBUrl::usage(const std::string &tool) const { +void FDBUrl::usage(const std::string& tool) const { eckit::Log::info() << std::endl << "Usage: " << tool << " request.mars target.grib" << std::endl << " " << tool << " --raw request.mars target.grib" << std::endl @@ -51,7 +49,7 @@ void FDBUrl::usage(const std::string &tool) const { fdb5::FDBTool::usage(tool); } -void FDBUrl::execute(const eckit::option::CmdArgs &args) { +void FDBUrl::execute(const eckit::option::CmdArgs& args) { bool extract = args.getBool("extract", false); bool raw = args.getBool("raw", false); @@ -120,9 +118,7 @@ void FDBUrl::execute(const eckit::option::CmdArgs &args) { dh->saveInto(out); } - -int main(int argc, char **argv) { +int main(int argc, char** argv) { FDBUrl app(argc, argv); return app.start(); } -