fix: Merge branch 'develop' into feature/backend-s3

src/fdb5/database/Index.h

@@ -15,27 +15,26 @@
 #ifndef fdb5_Index_H
 #define fdb5_Index_H
 
-#include <memory>
-#include <optional>
-#include <functional>
-
 #include "eckit/eckit.h"
-
 #include "eckit/io/Length.h"
 #include "eckit/io/Offset.h"
+#include "eckit/memory/Counted.h"
 #include "eckit/memory/NonCopyable.h"
 #include "eckit/types/FixedString.h"
 #include "eckit/types/Types.h"
-#include "eckit/memory/Counted.h"
-
 #include "fdb5/database/EntryVisitMechanism.h"
 #include "fdb5/database/Field.h"
-#include "fdb5/database/IndexStats.h"
 #include "fdb5/database/IndexAxis.h"
 #include "fdb5/database/IndexLocation.h"
+#include "fdb5/database/IndexStats.h"
 #include "fdb5/database/Indexer.h"
 #include "fdb5/database/Key.h"
 
+#include <functional>
+#include <memory>
+#include <optional>
+#include <vector>
+
 namespace eckit {
 class Stream;
 }
@@ -154,7 +153,7 @@ class Index {
 
     const IndexLocation& location() const { return content_->location(); }
 
-    const std::vector<eckit::URI> dataURIs() const { return content_->dataURIs(); }
+    std::vector<eckit::URI> dataURIs() const { return content_->dataURIs(); }
 
     bool dirty() const { return content_->dirty(); }
 

src/fdb5/s3/S3Common.cc

@@ -8,16 +8,19 @@
  * does it submit to any jurisdiction.
  */
 
-#include <algorithm>
+#include "fdb5/s3/S3Common.h"
 
-#include "eckit/io/s3/S3Name.h"
-#include "eckit/io/s3/S3Credential.h"
+#include "eckit/config/LocalConfiguration.h"
+#include "eckit/exception/Exceptions.h"
+#include "eckit/filesystem/URI.h"
+#include "eckit/io/s3/S3Config.h"
 #include "eckit/io/s3/S3Session.h"
+#include "eckit/utils/Tokenizer.h"
+#include "fdb5/config/Config.h"
+#include "fdb5/database/Key.h"
 
-#include "fdb5/s3/S3Common.h"
-
-// #include "eckit/exception/Exceptions.h"
-#include "eckit/config/Resource.h"
+#include <algorithm>
+#include <string>
 
 namespace fdb5 {
 
@@ -27,17 +30,12 @@ S3Common::S3Common(const fdb5::Config& config, const std::string& component, con
 
     parseConfig(config);
 
-
-
     /// @note: code for bucket per DB
 
     std::string keyStr = key.valuesToString();
     std::replace(keyStr.begin(), keyStr.end(), ':', '-');
     db_bucket_ = prefix_ + keyStr;
 
-
-
-
     /// @note: code for single bucket for all DBs
 
     // std::vector<std::string> valid{"catalogue", "store"};
@@ -63,11 +61,7 @@ S3Common::S3Common(const fdb5::Config& config, const std::string& component, con
     // if (c.has("client"))
     //     fdb5::DaosManager::instance().configure(c.getSubConfiguration("client"));
 
-
-
-
     /// @todo: check that the bucket name complies with name restrictions
-
 }
 
 S3Common::S3Common(const fdb5::Config& config, const std::string& component, const eckit::URI& uri) {
@@ -77,19 +71,15 @@ S3Common::S3Common(const fdb5::Config& config, const std::string& component, con
 
     parseConfig(config);
 
-    endpoint_ = eckit::net::Endpoint{uri.host(), uri.port()};
-
-
+    endpoint_ = eckit::net::Endpoint {uri.host(), uri.port()};
 
     /// @note: code for bucket per DB
 
     const auto parts = eckit::Tokenizer("/").tokenize(uri.name());
-    const auto n = parts.size();
+    const auto n     = parts.size();
     ASSERT(n == 1 | n == 2);
     db_bucket_ = parts[0];
 
-
-
     /// @note: code for single bucket for all DBs
 
     // eckit::S3Name n{uri};
@@ -104,38 +94,42 @@ S3Common::S3Common(const fdb5::Config& config, const std::string& component, con
 
     // db_prefix_ = bits[0];
 
-
     // // eckit::LocalConfiguration c{};
 
     // // if (config.has("s3")) c = config.getSubConfiguration("s3");
 
     // // if (c.has("client"))
     // //     fdb5::DaosManager::instance().configure(c.getSubConfiguration("client"));
-
 }
 
 void S3Common::parseConfig(const fdb5::Config& config) {
 
-    eckit::LocalConfiguration s3{};
+    eckit::LocalConfiguration s3 {};
 
     if (config.has("s3")) {
         s3 = config.getSubConfiguration("s3");
-        
+
         std::string credentialsPath;
         if (s3.has("credential")) { credentialsPath = s3.getString("credential"); }
-        eckit::S3Session::instance().readCredentials(credentialsPath);
+        eckit::S3Session::instance().loadCredentials(credentialsPath);
+    }
+
+    if (!s3.has("endpoint")) {
+        throw eckit::UserError("Missing \"endpoint\" in configuration: " + config.configPath());
     }
-
 
-    endpoint_ = eckit::net::Endpoint{s3.getString("endpoint", "127.0.0.1:9000")};
+    endpoint_ = eckit::net::Endpoint {s3.getString("endpoint", "127.0.0.1:9000")};
+
+    eckit::S3Config s3Config(endpoint_);
 
+    if (s3.has("region")) { s3Config.region = s3.getString("region"); }
 
+    eckit::S3Session::instance().addClient(s3Config);
 
     /// @note: code for bucket per DB only
     prefix_ = s3.getString("bucketPrefix", prefix_);
-
 }
 
 //----------------------------------------------------------------------------------------------------------------------
 
-} // namespace fdb5
+}  // namespace fdb5

src/fdb5/s3/S3FieldLocation.cc

@@ -33,14 +33,14 @@ S3FieldLocation::S3FieldLocation(const eckit::URI &uri, eckit::Offset offset, ec
 S3FieldLocation::S3FieldLocation(eckit::Stream& s) :
     FieldLocation(s) {}
 
-std::shared_ptr<FieldLocation> S3FieldLocation::make_shared() const {
-    return std::make_shared<S3FieldLocation>(std::move(*this));
+std::shared_ptr<const FieldLocation> S3FieldLocation::make_shared() const {
+    return std::make_shared<const S3FieldLocation>(std::move(*this));
 }
 
 eckit::DataHandle* S3FieldLocation::dataHandle() const {
 
     return eckit::S3ObjectName(uri_).dataHandle(offset());
-    
+
 }
 
 void S3FieldLocation::print(std::ostream &out) const {
@@ -53,4 +53,4 @@ void S3FieldLocation::visit(FieldLocationVisitor& visitor) const {
 
 static FieldLocationBuilder<S3FieldLocation> builder("s3");
 
-} // namespace fdb5
+} // namespace fdb5

src/fdb5/s3/S3FieldLocation.h

@@ -32,7 +32,7 @@ class S3FieldLocation : public FieldLocation {
 
     eckit::DataHandle* dataHandle() const override;
 
-    virtual std::shared_ptr<FieldLocation> make_shared() const override;
+    virtual std::shared_ptr<const FieldLocation> make_shared() const override;
 
     virtual void visit(FieldLocationVisitor& visitor) const override;
 
@@ -56,4 +56,4 @@ class S3FieldLocation : public FieldLocation {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-} // namespace fdb5
+} // namespace fdb5