feat(S3): parse config for store [skip ci]

CMakeLists.txt

@@ -57,9 +57,9 @@ ecbuild_add_option( FEATURE TOCFDB  # option defined in fdb5_config.h
 
 ### FDB S3 Store backend
 ecbuild_add_option( FEATURE S3FDB
-                    CONDITION eckit_HAVE_AWS_S3 
-                    DEFAULT ON
-                    DESCRIPTION "S3 support for FDB Store" )
+                    DEFAULT OFF
+                    CONDITION eckit_HAVE_S3
+                    DESCRIPTION "S3 storage support for FDB Store" )
 
 ### support for Lustre API control of file stripping
 

src/fdb5/CMakeLists.txt

@@ -425,16 +425,7 @@ else()
     set( UUID_INCLUDE_DIRS "" )
 endif()
 
-if( HAVE_S3FDB )
-    list( APPEND fdb5_srcs
-        s3/S3FieldLocation.h
-        s3/S3FieldLocation.cc
-        s3/S3Store.h
-        s3/S3Store.cc
-        s3/S3Common.h
-        s3/S3Common.cc
-    )
-endif()
+add_subdirectory( s3 )
 
 ecbuild_add_library(
 

src/fdb5/s3/CMakeLists.txt

@@ -0,0 +1,18 @@
+if( HAVE_S3FDB )
+
+    list( APPEND fdb5_srcs
+        s3/S3Common.cc
+        s3/S3Common.h
+        s3/S3FieldLocation.cc
+        s3/S3FieldLocation.h
+        s3/S3Root.cc
+        s3/S3Root.h
+        s3/S3RootManager.cc
+        s3/S3RootManager.h
+        s3/S3Store.cc
+        s3/S3Store.h
+    )
+
+    set( fdb5_srcs ${fdb5_srcs} PARENT_SCOPE )
+
+endif()

src/fdb5/s3/S3Common.cc

@@ -8,68 +8,20 @@
  * does it submit to any jurisdiction.
  */
 
-#include "fdb5/s3/S3Common.h"
+/*
+ * This software was developed as part of the Horizon Europe programme funded project DaFab
+ * (Grant agreement: 101128693) https://www.dafab-ai.eu/
+ */
 
-#include "eckit/config/LocalConfiguration.h"
-#include "eckit/exception/Exceptions.h"
-#include "eckit/filesystem/URI.h"
-#include "eckit/io/s3/S3BucketName.h"
-#include "eckit/io/s3/S3Config.h"
-#include "eckit/io/s3/S3Session.h"
-#include "fdb5/config/Config.h"
-#include "fdb5/database/Key.h"
+#include "fdb5/s3/S3Common.h"
 
-#include <algorithm>
-#include <string>
+#include "fdb5/s3/S3RootManager.h"
 
 namespace fdb5 {
 
 //----------------------------------------------------------------------------------------------------------------------
 
-S3Common::S3Common(const fdb5::Config& config, const std::string& /*component*/, const fdb5::Key& key) {
-
-    parseConfig(config);
-
-    std::string keyStr = key.valuesToString();
-    std::replace(keyStr.begin(), keyStr.end(), ':', '-');
-    db_bucket_ = prefix_ + keyStr;
-}
-
-S3Common::S3Common(const fdb5::Config& config, const std::string& /*component*/, const eckit::URI& uri) {
-
-    parseConfig(config);
-
-    endpoint_ = eckit::net::Endpoint {uri.host(), uri.port()};
-
-    db_bucket_ = eckit::S3BucketName::parse(uri.name());
-}
-
-void S3Common::parseConfig(const fdb5::Config& config) {
-
-    eckit::LocalConfiguration s3 {};
-
-    if (config.has("s3")) {
-        s3 = config.getSubConfiguration("s3");
-
-        std::string credentialsPath;
-        if (s3.has("credential")) { credentialsPath = s3.getString("credential"); }
-        eckit::S3Session::instance().loadCredentials(credentialsPath);
-    }
-
-    if (!s3.has("endpoint")) {
-        throw eckit::UserError("Missing \"endpoint\" in configuration: " + config.configPath());
-    }
-
-    endpoint_ = eckit::net::Endpoint {s3.getString("endpoint", "127.0.0.1:9000")};
-
-    eckit::S3Config s3Config(endpoint_);
-
-    if (s3.has("region")) { s3Config.region = s3.getString("region"); }
-
-    eckit::S3Session::instance().addClient(s3Config);
-
-    prefix_ = s3.getString("bucketPrefix", prefix_);
-}
+S3Common::S3Common(const Key& databaseKey, const Config& config) : root_ {S3RootManager(config).root(databaseKey)} { }
 
 //----------------------------------------------------------------------------------------------------------------------
 

src/fdb5/s3/S3Common.h

@@ -8,33 +8,32 @@
  * does it submit to any jurisdiction.
  */
 
-/// @file   S3Common.h
-/// @author Nicolau Manubens
-/// @date   Feb 2024
+/*
+ * This software was developed as part of the Horizon Europe programme funded project DaFab
+ * (Grant agreement: 101128693) https://www.dafab-ai.eu/
+ */
+
+/// @file   S3Config.h
+/// @author Metin Cakircali
+/// @date   Dec 2024
 
 #pragma once
 
-#include "eckit/filesystem/URI.h"
-#include "fdb5/config/Config.h"
-#include "fdb5/database/Key.h"
+#include "eckit/io/s3/S3BucketName.h"
 
 namespace fdb5 {
 
-class S3Common {
+class Key;
+class Config;
 
-public:  // methods
-    S3Common(const fdb5::Config&, const std::string& component, const fdb5::Key&);
-    S3Common(const fdb5::Config&, const std::string& component, const eckit::URI&);
+//----------------------------------------------------------------------------------------------------------------------
 
-private:  // methods
-    void parseConfig(const fdb5::Config& config);
+struct S3Common {
+    S3Common(const Key& databaseKey, const Config& config);
 
-protected:  // members
-    eckit::net::Endpoint endpoint_;
-    std::string          db_bucket_;
-
-private:  // members
-    std::string prefix_;
+    eckit::S3BucketName root_;
 };
 
+//----------------------------------------------------------------------------------------------------------------------
+
 }  // namespace fdb5

src/fdb5/s3/S3Root.cc

@@ -0,0 +1,29 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/*
+ * This software was developed as part of the Horizon Europe programme funded project DaFab
+ * (Grant agreement: 101128693) https://www.dafab-ai.eu/
+ */
+
+#include "fdb5/s3/S3Root.h"
+
+#include "eckit/config/LocalConfiguration.h"
+
+namespace fdb5 {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+S3Root::S3Root(const eckit::LocalConfiguration& root)
+    : endpoint_ {root.getString("endpoint")}, bucket_ {root.getString("bucket")} { }
+
+//----------------------------------------------------------------------------------------------------------------------
+
+}  // namespace fdb5

src/fdb5/s3/S3Root.h

@@ -0,0 +1,41 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/*
+ * This software was developed as part of the Horizon Europe programme funded project DaFab
+ * (Grant agreement: 101128693) https://www.dafab-ai.eu/
+ */
+
+/// @file   S3Config.h
+/// @author Metin Cakircali
+/// @date   Dec 2024
+
+#pragma once
+
+#include "eckit/config/LocalConfiguration.h"
+#include "eckit/net/Endpoint.h"
+
+#include <string>
+
+namespace fdb5 {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+struct S3Root {
+
+    explicit S3Root(const eckit::LocalConfiguration& root);
+
+    eckit::net::Endpoint endpoint_;
+    std::string          bucket_;
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
+}  // namespace fdb5

src/fdb5/s3/S3RootManager.cc

@@ -0,0 +1,100 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/*
+ * This software was developed as part of the Horizon Europe programme funded project DaFab
+ * (Grant agreement: 101128693) https://www.dafab-ai.eu/
+ */
+
+#include "fdb5/s3/S3RootManager.h"
+
+#include "eckit/config/LocalConfiguration.h"
+#include "eckit/exception/Exceptions.h"
+#include "eckit/io/s3/S3BucketName.h"
+#include "eckit/io/s3/S3Config.h"
+#include "eckit/io/s3/S3Session.h"
+#include "eckit/log/CodeLocation.h"
+#include "eckit/log/Log.h"
+#include "fdb5/LibFdb5.h"
+#include "fdb5/config/Config.h"
+#include "fdb5/s3/S3Root.h"
+
+#include <ostream>
+#include <string>
+#include <vector>
+
+namespace fdb5 {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+namespace {
+
+auto loadS3(const Config& config) -> eckit::LocalConfiguration {
+
+    eckit::Log::debug<LibFdb5>() << "Loading S3 configuration from: " << config.configPath() << std::endl;
+
+    const auto s3Config = config.getSubConfiguration("s3");
+
+    if (s3Config.empty()) { throw eckit::UserError("No S3 configuration found in: " + config.configPath()); }
+
+    {
+        auto& session = eckit::S3Session::instance();
+
+        // credentials
+
+        if (s3Config.has("credentials")) {
+            const auto credentialsPath = s3Config.getString("credentials");
+            session.loadCredentials(credentialsPath);
+        }
+
+        // configuration clients
+
+        std::string configPath;
+        if (s3Config.has("configuration")) { configPath = s3Config.getString("configuration"); }
+        session.loadClients(configPath);
+
+        //         const auto roots = s3Config.getSubConfigurations("roots");
+        //         eckit::Log::info() << "----------> serversPath: " << serversPath << std::endl;
+        //         session.loadClients(serversPath);
+    }
+
+    return s3Config;
+}
+
+}  // namespace
+
+//----------------------------------------------------------------------------------------------------------------------
+
+S3RootManager::S3RootManager(const Config& config) : config_ {loadS3(config)} { }
+
+//----------------------------------------------------------------------------------------------------------------------
+
+eckit::S3BucketName S3RootManager::root(const Key& /*databaseKey*/) const {
+    /// @todo implement databaseKey to root selection
+    return parseRoots().front();
+}
+
+std::vector<eckit::S3BucketName> S3RootManager::parseRoots() const {
+
+    const auto roots = config_.getSubConfigurations("roots");
+
+    if (roots.empty()) { throw eckit::UserError("No S3 roots found in configuration!", Here()); }
+
+    std::vector<eckit::S3BucketName> result;
+    result.reserve(roots.size());
+
+    for (const auto& root : roots) { result.emplace_back(root.getString("endpoint"), root.getString("bucket")); }
+
+    return result;
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+
+}  // namespace fdb5