FDB-303 added decoupling between CanonicalKey and fully typed 'ApiKey'

ecmwf · Jun 17, 2024 · d0be656 · d0be656
1 parent c029792
commit d0be656
Show file tree

Hide file tree

Showing 196 changed files with 1,209 additions and 1,047 deletions.
diff --git a/src/fdb5/api/DistFDB.cc b/src/fdb5/api/DistFDB.cc
@@ -70,7 +70,7 @@ DistFDB::DistFDB(const Config& config, const std::string& name) :
 
 DistFDB::~DistFDB() {}
 
-void DistFDB::archive(const Key& key, const void* data, size_t length) {
+void DistFDB::archive(const CanonicalKey& key, const void* data, size_t length) {
 
     std::vector<size_t> laneIndices;
 

diff --git a/src/fdb5/api/DistFDB.h b/src/fdb5/api/DistFDB.h
@@ -41,7 +41,7 @@ class DistFDB : public FDBBase {
     DistFDB(const Config& config, const std::string& name);
     ~DistFDB() override;
 
-    void archive(const Key& key, const void* data, size_t length) override;
+    void archive(const CanonicalKey& key, const void* data, size_t length) override;
 
     ListIterator inspect(const metkit::mars::MarsRequest& request) override;
 

diff --git a/src/fdb5/api/FDB.cc b/src/fdb5/api/FDB.cc
@@ -50,7 +50,7 @@ FDB::~FDB() {
 }
 
 void FDB::archive(eckit::message::Message msg) {
-    fdb5::Key key = MessageDecoder::messageToKey(msg);
+    fdb5::CanonicalKey key = MessageDecoder::messageToKey(msg);
     archive(key, msg.data(), msg.length());
 }
 void FDB::archive(eckit::DataHandle& handle) {
@@ -73,7 +73,7 @@ void FDB::archive(const metkit::mars::MarsRequest& request, eckit::DataHandle& h
     metkit::hypercube::HyperCube cube(request);
 
     while ( (msg = reader.next()) ) {
-        fdb5::Key key = MessageDecoder::messageToKey(msg);
+        fdb5::CanonicalKey key = MessageDecoder::messageToKey(msg);
         if (!cube.clear(key.request())) {
             std::stringstream ss;
             ss << "FDB archive - found unexpected message" << std::endl;
@@ -97,13 +97,13 @@ void FDB::archive(const metkit::mars::MarsRequest& request, eckit::DataHandle& h
     }
 }
 
-void FDB::archive(const Key& key, const void* data, size_t length) {
+void FDB::archive(const CanonicalKey& key, const void* data, size_t length) {
     eckit::Timer timer;
     timer.start();
 
     // This is the API entrypoint. Keys supplied by the user may not have type registry info attached (so
     // serialisation won't work properly...)
-    Key keyInternal(key);
+    CanonicalKey keyInternal(key);
     keyInternal.registry(config().schema().registry());
 
     // step in archival requests from the model is just an integer. We need to include the stepunit

diff --git a/src/fdb5/api/FDB.h b/src/fdb5/api/FDB.h
@@ -48,7 +48,7 @@ namespace fdb5 {
 
 class FDBBase;
 class FDBToolRequest;
-class Key;
+class CanonicalKey;
 
 //----------------------------------------------------------------------------------------------------------------------
 
@@ -75,7 +75,7 @@ class FDB {
     // warning: not high-perf API - makes sure that all the requested fields are archived and there are no data exceeding the request
     void archive(const metkit::mars::MarsRequest& request, eckit::DataHandle& handle);
     // disclaimer: this is a low-level API. The provided key and the corresponding data are not checked for consistency
-    void archive(const Key& key, const void* data, size_t length);
+    void archive(const CanonicalKey& key, const void* data, size_t length);
 
     /// Flushes all buffers and closes all data handles into a consistent DB state
     /// @note always safe to call

diff --git a/src/fdb5/api/FDBFactory.h b/src/fdb5/api/FDBFactory.h
@@ -48,7 +48,7 @@ namespace metkit { class MarsRequest; }
 
 namespace fdb5 {
 
-class Key;
+class CanonicalKey;
 class FDBToolRequest;
 
 //----------------------------------------------------------------------------------------------------------------------
@@ -64,7 +64,7 @@ class FDBBase : private eckit::NonCopyable {
 
     // -------------- Primary API functions ----------------------------
 
-    virtual void archive(const Key& key, const void* data, size_t length) = 0;
+    virtual void archive(const CanonicalKey& key, const void* data, size_t length) = 0;
 
     virtual void flush() = 0;
 

diff --git a/src/fdb5/api/LocalFDB.cc b/src/fdb5/api/LocalFDB.cc
@@ -46,7 +46,7 @@ using namespace eckit;
 
 
 namespace fdb5 {
-void LocalFDB::archive(const Key& key, const void* data, size_t length) {
+void LocalFDB::archive(const CanonicalKey& key, const void* data, size_t length) {
 
     if (!archiver_) {
         LOG_DEBUG_LIB(LibFdb5) << *this << ": Constructing new archiver" << std::endl;

diff --git a/src/fdb5/api/LocalFDB.h b/src/fdb5/api/LocalFDB.h
@@ -37,7 +37,7 @@ class LocalFDB : public FDBBase {
     using FDBBase::FDBBase;
     using FDBBase::stats;
 
-    void archive(const Key& key, const void* data, size_t length) override;
+    void archive(const CanonicalKey& key, const void* data, size_t length) override;
 
     ListIterator inspect(const metkit::mars::MarsRequest& request) override;
 

diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
@@ -757,7 +757,7 @@ MoveIterator RemoteFDB::move(const FDBToolRequest& request, const eckit::URI& de
 // -----------------------------------------------------------------------------------------------------
 
 // Here we do archive/flush related stuff
-void RemoteFDB::archive(const Key& key, const void* data, size_t length) {
+void RemoteFDB::archive(const CanonicalKey& key, const void* data, size_t length) {
 
     connect();
 
@@ -836,9 +836,9 @@ FDBStats RemoteFDB::archiveThreadLoop(uint32_t requestID) {
 
     // We can pop multiple elements off the archive queue simultaneously, if
     // configured
-    std::vector<std::pair<Key, Buffer>> elements;
+    std::vector<std::pair<CanonicalKey, Buffer>> elements;
     for (size_t i = 0; i < maxArchiveBatchSize_; ++i) {
-        elements.emplace_back(std::make_pair(Key{}, Buffer{0}));
+        elements.emplace_back(std::make_pair(CanonicalKey{}, Buffer{0}));
     }
 
     try {
@@ -876,7 +876,7 @@ FDBStats RemoteFDB::archiveThreadLoop(uint32_t requestID) {
     // They will be released when flush() is called.
 }
 
-long RemoteFDB::sendArchiveData(uint32_t id, const std::vector<std::pair<Key, Buffer>>& elements, size_t count) {
+long RemoteFDB::sendArchiveData(uint32_t id, const std::vector<std::pair<CanonicalKey, Buffer>>& elements, size_t count) {
 
     if (count == 1) {
         sendArchiveData(id, elements[0].first, elements[0].second.data(), elements[0].second.size());
@@ -922,7 +922,7 @@ long RemoteFDB::sendArchiveData(uint32_t id, const std::vector<std::pair<Key, Bu
 }
 
 
-void RemoteFDB::sendArchiveData(uint32_t id, const Key& key, const void* data, size_t length) {
+void RemoteFDB::sendArchiveData(uint32_t id, const CanonicalKey& key, const void* data, size_t length) {
 
     ASSERT(data);
     ASSERT(length != 0);
@@ -1094,10 +1094,10 @@ class FDBRemoteDataHandle : public DataHandle {
 
 
 eckit::DataHandle* RemoteFDB::dataHandle(const FieldLocation& fieldLocation) {
-    return dataHandle(fieldLocation, Key());
+    return dataHandle(fieldLocation, CanonicalKey());
 }
 
-eckit::DataHandle* RemoteFDB::dataHandle(const FieldLocation& fieldLocation, const Key& remapKey) {
+eckit::DataHandle* RemoteFDB::dataHandle(const FieldLocation& fieldLocation, const CanonicalKey& remapKey) {
 
     connect();
 

diff --git a/src/fdb5/api/RemoteFDB.h b/src/fdb5/api/RemoteFDB.h
@@ -45,7 +45,7 @@ class RemoteFDB : public FDBBase {
 
     using StoredMessage = std::pair<remote::MessageHeader, eckit::Buffer>;
     using MessageQueue = eckit::Queue<StoredMessage>;
-    using ArchiveQueue = eckit::Queue<std::pair<fdb5::Key, eckit::Buffer>>;
+    using ArchiveQueue = eckit::Queue<std::pair<fdb5::CanonicalKey, eckit::Buffer>>;
 
 public: // method
 
@@ -55,10 +55,10 @@ class RemoteFDB : public FDBBase {
     ~RemoteFDB() override;
 
     /// Archive writes data into aggregation buffer
-    void archive(const Key& key, const void* data, size_t length) override;
+    void archive(const CanonicalKey& key, const void* data, size_t length) override;
 
     eckit::DataHandle* dataHandle(const FieldLocation& fieldLocation);
-    eckit::DataHandle* dataHandle(const FieldLocation& fieldLocation, const Key& remapKey);
+    eckit::DataHandle* dataHandle(const FieldLocation& fieldLocation, const CanonicalKey& remapKey);
 
     ListIterator inspect(const metkit::mars::MarsRequest& request) override;
 
@@ -123,8 +123,8 @@ class RemoteFDB : public FDBBase {
 
     FDBStats archiveThreadLoop(uint32_t requestID);
 
-    void sendArchiveData(uint32_t id, const Key& key, const void* data, size_t length);
-    long sendArchiveData(uint32_t id, const std::vector<std::pair<Key, eckit::Buffer>>& elements, size_t count);
+    void sendArchiveData(uint32_t id, const CanonicalKey& key, const void* data, size_t length);
+    long sendArchiveData(uint32_t id, const std::vector<std::pair<CanonicalKey, eckit::Buffer>>& elements, size_t count);
 
     virtual void print(std::ostream& s) const override;
 

diff --git a/src/fdb5/api/SelectFDB.cc b/src/fdb5/api/SelectFDB.cc
@@ -80,7 +80,7 @@ SelectFDB::SelectFDB(const Config& config, const std::string& name) :
 
 SelectFDB::~SelectFDB() {}
 
-void SelectFDB::archive(const Key& key, const void* data, size_t length) {
+void SelectFDB::archive(const CanonicalKey& key, const void* data, size_t length) {
 
     for (auto& iter : subFdbs_) {
 
@@ -208,7 +208,7 @@ void SelectFDB::print(std::ostream &s) const {
     s << "SelectFDB()";
 }
 
-bool SelectFDB::matches(const Key &key, const SelectMap &select, bool requireMissing) const {
+bool SelectFDB::matches(const CanonicalKey& key, const SelectMap &select, bool requireMissing) const {
 
     for (const auto& kv : select) {
 

diff --git a/src/fdb5/api/SelectFDB.h b/src/fdb5/api/SelectFDB.h
@@ -46,7 +46,7 @@ class SelectFDB : public FDBBase {
 
     ~SelectFDB() override;
 
-    void archive(const Key& key, const void* data, size_t length) override;
+    void archive(const CanonicalKey& key, const void* data, size_t length) override;
 
     ListIterator inspect(const metkit::mars::MarsRequest& request) override;
 
@@ -74,7 +74,7 @@ class SelectFDB : public FDBBase {
 
     void print(std::ostream& s) const override;
 
-    bool matches(const Key& key, const SelectMap& select, bool requireMissing) const;
+    bool matches(const CanonicalKey& key, const SelectMap& select, bool requireMissing) const;
     bool matches(const metkit::mars::MarsRequest& request, const SelectMap& select, bool requireMissing) const;
 
     template <typename QueryFN>

diff --git a/src/fdb5/api/fdb_c.cc b/src/fdb5/api/fdb_c.cc
@@ -37,8 +37,8 @@ struct fdb_handle_t : public FDB {
     using FDB::FDB;
 };
 
-struct fdb_key_t : public Key {
-    using Key::Key;
+struct fdb_key_t : public CanonicalKey {
+    using CanonicalKey::CanonicalKey;
 };
 
 struct fdb_request_t {
@@ -91,15 +91,15 @@ struct fdb_split_key_t {
 public:
     fdb_split_key_t() : key_(nullptr), level_(-1) {}
 
-    void set(const std::vector<Key>& key) {
+    void set(const std::vector<CanonicalKey>& key) {
         key_ = &key;
         level_ = -1;
     }
 
     int next_metadata(const char** k, const char** v, size_t* level) {
         if (key_ == nullptr) {
             std::stringstream ss;
-            ss << "fdb_split_key_t not valid. Key not configured";
+            ss << "fdb_split_key_t not valid. CanonicalKey not configured";
             throw eckit::UserError(ss.str(), Here());
         }
         if (level_ == -1) {
@@ -129,9 +129,9 @@ struct fdb_split_key_t {
     }
 
 private:
-    const std::vector<Key>* key_;
+    const std::vector<CanonicalKey>* key_;
     int level_;
-    Key::const_iterator it_;
+    CanonicalKey::const_iterator it_;
 };
 
 struct fdb_listiterator_t {

diff --git a/src/fdb5/api/fdb_c.h b/src/fdb5/api/fdb_c.h
@@ -87,29 +87,29 @@ int fdb_set_failure_handler(fdb_failure_handler_t handler, void* context);
 /** @} */
 
 
-/** \defgroup Key */
+/** \defgroup CanonicalKey */
 /** @{ */
 
 struct fdb_key_t;
-/** Opaque type for the Key object. Holds the metadata of a Key. */
+/** Opaque type for the CanonicalKey object. Holds the metadata of a CanonicalKey. */
 typedef struct fdb_key_t fdb_key_t;
 
-/** Creates a Key instance.
- * \param key Key instance. Returned instance must be deleted using #fdb_delete_key.
+/** Creates a CanonicalKey instance.
+ * \param key CanonicalKey instance. Returned instance must be deleted using #fdb_delete_key.
  * \returns Return code (#FdbErrorValues)
  */
 int fdb_new_key(fdb_key_t** key);
 
-/** Adds a metadata pair to a Key
- * \param key Key instance
+/** Adds a metadata pair to a CanonicalKey
+ * \param key CanonicalKey instance
  * \param param Metadata name
  * \param value Metadata value
  * \returns Return code (#FdbErrorValues)
  */
 int fdb_key_add(fdb_key_t* key, const char* param, const char* value);
 
-/** Deallocates Key object and associated resources.
- * \param key Key instance
+/** Deallocates CanonicalKey object and associated resources.
+ * \param key CanonicalKey instance
  * \returns Return code (#FdbErrorValues)
  */
 int fdb_delete_key(fdb_key_t* key);
@@ -179,9 +179,9 @@ int fdb_new_splitkey(fdb_split_key_t** key);
 /** Returns the next set of metadata in a SplitKey object. For a given ListElement, the SplitKey represents the Keys associated with each level of the FDB index.
  * Supports multiple fdb_split_key_t iterating over the same key.
  * \param it SplitKey instance
- * \param key Key metadata name
- * \param value Key metadata value
- * \param level level in the iondex of the current Key
+ * \param key CanonicalKey metadata name
+ * \param value CanonicalKey metadata value
+ * \param level level in the iondex of the current CanonicalKey
  * \returns Return code (#FdbErrorValues)
  */
 int fdb_splitkey_next_metadata(fdb_split_key_t* it, const char** key, const char** value, size_t* level);
@@ -313,7 +313,7 @@ int fdb_new_handle(fdb_handle_t** fdb);
 /** Archives binary data to a FDB instance.
  * \warning this is a low-level API. The provided key and the corresponding data are not checked for consistency
  * \param fdb FDB instance.
- * \param key Key used for indexing and archiving the data
+ * \param key CanonicalKey used for indexing and archiving the data
  * \param data Pointer to the binary data to archive
  * \param length Size of the data to archive with the given #key
  * \returns Return code (#FdbErrorValues)

diff --git a/src/fdb5/api/helpers/AxesIterator.cc b/src/fdb5/api/helpers/AxesIterator.cc
@@ -19,7 +19,7 @@ namespace fdb5 {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-AxesElement::AxesElement(Key&& dbKey, IndexAxis&& axes) :
+AxesElement::AxesElement(CanonicalKey&& dbKey, IndexAxis&& axes) :
         dbKey_(std::move(dbKey)), axes_(std::move(axes)) {}
 
 AxesElement::AxesElement(eckit::Stream& s) {

diff --git a/src/fdb5/api/helpers/AxesIterator.h b/src/fdb5/api/helpers/AxesIterator.h
@@ -30,11 +30,11 @@ class AxesElement {
 public: // methods
 
     AxesElement() = default;
-    AxesElement(Key&& dbKey, IndexAxis&& axis);
+    AxesElement(CanonicalKey&& dbKey, IndexAxis&& axis);
     explicit AxesElement(eckit::Stream& s);
 
     [[ nodiscard ]]
-    const Key& key() const { return dbKey_; }
+    const CanonicalKey& key() const { return dbKey_; }
 
     [[ nodiscard ]]
     const IndexAxis& axes() const { return axes_; }
@@ -57,7 +57,7 @@ class AxesElement {
 
 private: // members
 
-    Key dbKey_;
+    CanonicalKey dbKey_;
     IndexAxis axes_;
 };
 

diff --git a/src/fdb5/api/helpers/ControlIterator.h b/src/fdb5/api/helpers/ControlIterator.h
@@ -137,7 +137,7 @@ struct ControlElement {
     ControlElement(eckit::Stream& s);
 
     // Database key
-    Key key;
+    CanonicalKey key;
 
     // The location of the Database this response is for
     eckit::URI location;

diff --git a/src/fdb5/api/helpers/ListIterator.cc b/src/fdb5/api/helpers/ListIterator.cc
@@ -21,7 +21,7 @@ namespace fdb5 {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-ListElement::ListElement(const std::vector<Key>& keyParts, std::shared_ptr<const FieldLocation> location, time_t timestamp) :
+ListElement::ListElement(const std::vector<CanonicalKey>& keyParts, std::shared_ptr<const FieldLocation> location, time_t timestamp) :
     keyParts_(keyParts), location_(location), timestamp_(timestamp) {}
 
 ListElement::ListElement(eckit::Stream &s) {
@@ -30,10 +30,10 @@ ListElement::ListElement(eckit::Stream &s) {
     s >> timestamp_;
 }
 
-Key ListElement::combinedKey() const {
-    Key combined = keyParts_[2];
+CanonicalKey ListElement::combinedKey() const {
+    CanonicalKey combined = keyParts_[2];
 
-    for (const Key& partKey : keyParts_) {
+    for (const CanonicalKey& partKey : keyParts_) {
         for (const auto& kv : partKey) {
             combined.set(kv.first, kv.second);
         }