Merge branch 'main' of baltig.infn.it:cnafsd/storm-tape into 28-upgra…

…de-to-vcpkg-release-2024-02-14

.gitlab-ci.yml

@@ -31,6 +31,7 @@ build-storm-tape:debug:
   before_script:
     - mkdir -p $VCPKG_DEFAULT_BINARY_CACHE
   script:
+    - git config --global --add safe.directory $CI_PROJECT_DIR
     - cmake --preset debug -DCPACK_RPM_BUILD_SOURCE_DIRS_PREFIX=/storm
     - cmake --build build/debug --target package --parallel
     - build/debug/tests/all.t
@@ -59,6 +60,7 @@ build-storm-tape:release:
   before_script:
     - mkdir -p $VCPKG_DEFAULT_BINARY_CACHE
   script:
+    - git config --global --add safe.directory $CI_PROJECT_DIR
     - cmake --preset release -DCPACK_RPM_BUILD_SOURCE_DIRS_PREFIX=/storm
     - cmake --build build/release --target package --parallel
   artifacts:
@@ -79,6 +81,7 @@ package-storm-tape:
   before_script:
     - mkdir -p $VCPKG_DEFAULT_BINARY_CACHE
   script:
+    - git config --global --add safe.directory $CI_PROJECT_DIR
     - cmake -S . -B build/mc -G "Ninja Multi-Config" -DBUILD_TESTING=OFF -DCPACK_RPM_BUILD_SOURCE_DIRS_PREFIX=/storm
     - cd build/mc
     - cmake --build . --parallel --config Debug

src/configuration.cpp

@@ -143,7 +143,7 @@ static StorageArea load_storage_area(YAML::Node const& sa)
   return {name, root, ap};
 }
 
-static void check_root(StorageArea const& sa)
+static void check_root(StorageArea const& sa, bool mirror_mode)
 {
   auto const& [name, root, _] = sa;
 
@@ -172,7 +172,7 @@ static void check_root(StorageArea const& sa)
                     root.string(), name)};
   }
 
-  if (!storm_has_all_permissions(root)) {
+  if (!mirror_mode && !storm_has_all_permissions(root)) {
     throw std::runtime_error{
         fmt::format("root '{}' of storage area '{}' has invalid permissions",
                     root.string(), name)};
@@ -266,11 +266,6 @@ static StorageAreas load_storage_areas(YAML::Node const& sas)
     }
   }
 
-  {
-    std::for_each(result.begin(), result.end(),
-                  [](auto const& sa) { check_root(sa); });
-  }
-
   return result;
 }
 
@@ -312,6 +307,23 @@ static std::optional<LogLevel> load_log_level(YAML::Node const& node)
   throw std::runtime_error{"invalid 'log-level' entry in configuration"};
 }
 
+static std::optional<bool> load_mirror_mode(YAML::Node const& node)
+{
+  if (!node.IsDefined()) {
+    return {};
+  }
+
+  if (node.IsNull()) {
+    throw std::runtime_error{fmt::format("mirror-mode is null")};
+  }
+
+  bool value;
+  if (YAML::convert<bool>::decode(node, value)) {
+    return value;
+  }
+  throw std::runtime_error{"invalid 'mirror-mode' entry in configuration"};
+}
+
 static Configuration load(YAML::Node const& node)
 {
   const auto sas_key = "storage-areas";
@@ -339,13 +351,30 @@ static Configuration load(YAML::Node const& node)
     config.log_level = *maybe_log_level;
   }
 
+  {
+    auto const key    = "mirror-mode";
+    auto const& value = node[key];
+    auto const maybe  = load_mirror_mode(value);
+    if (maybe.has_value()) {
+      config.mirror_mode = *maybe;
+    }
+  }
+
   return config;
 }
 
+auto check_sa_roots(StorageAreas const& sas, bool mirror_mode)
+{
+  std::for_each(sas.begin(), sas.end(),
+                [=](auto& sa) { check_root(sa, mirror_mode); });
+}
+
 Configuration load_configuration(std::istream& is)
 {
-  YAML::Node const config = YAML::Load(is);
-  return load(config);
+  YAML::Node const node = YAML::Load(is);
+  auto config           = load(node);
+  check_sa_roots(config.storage_areas, config.mirror_mode);
+  return config;
 }
 
 Configuration load_configuration(fs::path const& p)

src/configuration.hpp

@@ -25,6 +25,7 @@ struct Configuration
   std::uint16_t port   = 8080;
   StorageAreas storage_areas;
   LogLevel log_level = 1;
+  bool mirror_mode = false;
 };
 
 Configuration load_configuration(std::istream& is);

src/in_progress_request.hpp

@@ -0,0 +1,16 @@
+#ifndef STORM_IN_PROGRESS_REQUEST_HPP
+#define STORM_IN_PROGRESS_REQUEST_HPP
+
+#include <cstddef>
+
+namespace storm {
+
+struct InProgressRequest
+{
+  inline static constexpr struct Tag {} tag{};
+  std::size_t n_files{1'000};
+  int precise{0};
+};
+
+} // namespace storm
+#endif

src/io.cpp

@@ -62,14 +62,8 @@ crow::response to_crow_response(StatusResponse const& resp)
   std::transform( //
       m_files.begin(), m_files.end(), std::back_inserter(files),
       [](File const& file) {
-        boost::json::object result{{"path", file.logical_path.c_str()}};
-        if (file.locality == Locality::disk
-            || file.locality == Locality::disk_and_tape) {
-          result.emplace("onDisk", true);
-        } else {
-          result.emplace("state", to_string(file.state));
-        }
-        return result;
+        return boost::json::object{{"path", file.logical_path.c_str()},
+                                   {"state", to_string(file.state)}};
       });
   boost::json::object jbody;
   jbody["id"]          = id;
@@ -344,4 +338,26 @@ std::size_t from_body_params(std::string_view body, TakeOverRequest::Tag)
   }
   return n_files;
 }
+
+InProgressRequest from_query_params(crow::query_string const& qs,
+                                    InProgressRequest::Tag)
+{
+  InProgressRequest result{};
+
+  if (auto v = qs.get("n")) {
+    int n;
+    if (boost::conversion::try_lexical_convert(std::string{v}, n) && n > 0) {
+      result.n_files = static_cast<std::size_t>(n);
+    }
+  }
+  if (auto v = qs.get("precise")) {
+    int precise;
+    if (boost::conversion::try_lexical_convert(std::string{v}, precise) && precise >= 0) {
+      result.precise = precise;
+    }
+  }
+
+  return result;
+}
+
 } // namespace storm

src/io.hpp

@@ -6,11 +6,13 @@
 #include "requests_with_paths.hpp"
 #include "stage_request.hpp"
 #include "takeover_request.hpp"
+#include "in_progress_request.hpp"
 #include <boost/json.hpp>
 
 namespace crow {
 class request;
 class response;
+class query_string;
 } // namespace crow
 
 namespace storm {
@@ -56,6 +58,7 @@ void fill_hostinfo_from_forwarded(HostInfo& info, std::string const& http_forwar
 HostInfo get_hostinfo(crow::request const& req, Configuration const& conf);
 
 std::size_t from_body_params(std::string_view body, TakeOverRequest::Tag);
+InProgressRequest from_query_params(crow::query_string const& qs, InProgressRequest::Tag);
 
 } // namespace storm
 

src/local_storage.cpp

@@ -1,53 +1,42 @@
 #include "local_storage.hpp"
 #include "extended_attributes.hpp"
-#include <sys/stat.h>
 #include "profiler.hpp"
+#include <sys/stat.h>
 
 namespace storm {
 
-Locality LocalStorage::locality(PhysicalPath const& path)
+Result<bool> LocalStorage::is_in_progress(PhysicalPath const& path)
 {
-  PROFILE_FUNCTION();
+  std::error_code ec;
+  auto result = has_xattr(path, XAttrName{"user.TSMRecT"}, ec);
+  if (ec == std::error_code{}) {
+    return result;
+  } else {
+    return ec;
+  }
+}
 
+Result<FileSizeInfo> LocalStorage::file_size_info(PhysicalPath const& path)
+{
   struct stat sb = {};
 
   if (::stat(path.c_str(), &sb) == -1) {
-    return Locality::unavailable;
-  }
-
-  if (sb.st_size == 0) {
-    return Locality::none;
+    return std::make_error_code(std::errc{errno});
   }
 
   constexpr auto bytes_per_block{512L};
-  bool const is_stub{sb.st_blocks * bytes_per_block < sb.st_size};
+  return FileSizeInfo{static_cast<std::size_t>(sb.st_size),
+                      sb.st_blocks * bytes_per_block < sb.st_size};
+}
 
+Result<bool> LocalStorage::is_on_tape(PhysicalPath const& path)
+{
   std::error_code ec;
-
-  bool const is_being_recalled{[&] {
-    XAttrName const tsm_rect{"user.TSMRecT"};
-    return has_xattr(path, tsm_rect, ec);
-  }()};
-
-  if (ec != std::error_code{}) {
-    return Locality::unavailable;
-  }
-
-  bool const is_on_disk{!(is_stub || is_being_recalled)};
-
-  bool const is_on_tape{[&] {
-    XAttrName const storm_migrated{"user.storm.migrated"};
-    return has_xattr(path, storm_migrated, ec);
-  }()};
-
-  if (ec != std::error_code{}) {
-    return Locality::unavailable;
-  }
-
-  if (is_on_disk) {
-    return is_on_tape ? Locality::disk_and_tape : Locality::disk;
+  auto result = has_xattr(path, XAttrName{"user.storm.migrated"}, ec);
+  if (ec == std::error_code{}) {
+    return result;
   } else {
-    return is_on_tape ? Locality::tape : Locality::lost;
+    return ec;
   }
 }
 

src/local_storage.hpp

@@ -7,7 +7,9 @@ namespace storm {
 
 struct LocalStorage : Storage
 {
-  Locality locality(PhysicalPath const&) override;
+  Result<bool> is_in_progress(PhysicalPath const& path) override;
+  Result<FileSizeInfo> file_size_info(PhysicalPath const& path) override;
+  Result<bool> is_on_tape(PhysicalPath const& path) override;
 };
 
 } // namespace storm

src/profiler.hpp

@@ -12,7 +12,7 @@
 #  define COMBINE_HELPER(X,Y) X##Y
 #  define COMBINE(X,Y) COMBINE_HELPER(X,Y)
 #  define PROFILE_SCOPE(name) storm::InstrumentationTimer COMBINE(timer,__LINE__)(name)
-#  define PROFILE_FUNCTION()  PROFILE_SCOPE(__FUNCTION__)
+#  define PROFILE_FUNCTION()  PROFILE_SCOPE(__PRETTY_FUNCTION__)
 #else
 #  define PROFILE_SCOPE(name)
 #  define PROFILE_FUNCTION()

src/routes.cpp

@@ -210,7 +210,8 @@ void create_internal_routes(CrowApp& app, storm::Configuration const&,
     PROFILE_SCOPE("IN_PROGRESS");
     app.get_context<AccessLogger>(req).operation = "IN_PROGRESS";
     try {
-      auto resp = service.in_progress();
+      auto in_progress = from_query_params(req.url_params, InProgressRequest::tag);
+      auto resp = service.in_progress(in_progress);
       return to_crow_response(resp);
     } catch (HttpError const& e) {
       CROW_LOG_ERROR << e.what() << '\n';

src/storage.hpp

@@ -7,8 +7,10 @@ namespace storm {
 
 struct Storage
 {
-  virtual ~Storage()                                   = default;
-  virtual Locality locality(PhysicalPath const& path) = 0;
+  virtual ~Storage()                                            = default;
+  virtual Result<bool> is_in_progress(PhysicalPath const& path) = 0;
+  virtual Result<FileSizeInfo> file_size_info(PhysicalPath const& path) = 0;
+  virtual Result<bool> is_on_tape(PhysicalPath const& path)             = 0;
 };
 
 } // namespace storm