Merge branch 'hotfix/5.13.4'

ecmwf · Nov 28, 2024 · 156e3d8 · 156e3d8
2 parents c7f351f + a84c033
commit 156e3d8
Show file tree

Hide file tree

Showing 28 changed files with 88 additions and 47 deletions.
diff --git a/.clang-tidy b/.clang-tidy
@@ -0,0 +1,8 @@
+---
+Checks: "-*, readability-delete-null-pointer"
+WarningsAsErrors:             true
+HeaderFileExtensions:         ['', 'h','hh','hpp','hxx']
+ImplementationFileExtensions: ['c','cc','cpp','cxx']
+HeaderFilterRegex:            ''
+FormatStyle:                  none
+InheritParentConfig:          false
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-5.13.3
+5.13.4
diff --git a/src/dummy_daos/daos.h b/src/dummy_daos/daos.h
@@ -20,7 +20,6 @@
 #include <stdint.h>
 #include <unistd.h>
 #include <uuid/uuid.h>
-#include <stdbool.h>
 #include <string.h>
 
 //----------------------------------------------------------------------------------------------------------------------

diff --git a/src/fdb5/api/RandomFDB.cc b/src/fdb5/api/RandomFDB.cc
@@ -19,7 +19,6 @@
 #include "fdb5/api/FDBFactory.h"
 #include "fdb5/LibFdb5.h"
 
-#include <stdlib.h>
 #include <random>
 
 using namespace eckit;

diff --git a/src/fdb5/api/fdb_c.cc b/src/fdb5/api/fdb_c.cc
@@ -9,7 +9,6 @@
  */
 
 #include "eckit/io/MemoryHandle.h"
-#include "eckit/io/FileDescHandle.h"
 #include "eckit/message/Message.h"
 #include "eckit/runtime/Main.h"
 #include "eckit/config/YAMLConfiguration.h"
@@ -66,8 +65,8 @@ struct fdb_request_t {
         Tokenizer parse("/");
 
         for (int i=0; i<numValues; i++) {
-        	std::vector<std::string> result;
-        	parse(values[i], result);
+            std::vector<std::string> result;
+            parse(values[i], result);
             vv.insert(std::end(vv), std::begin(result), std::end(result));
         }
         request_.values(n, vv);
@@ -110,7 +109,7 @@ struct fdb_split_key_t {
             } else {
                 return FDB_ITERATION_COMPLETE;
             }
-        } 
+        }
         while (it_ == key_->at(level_).end()) {
             if (level_<key_->size()-1) {
                 level_++;
@@ -193,12 +192,15 @@ struct fdb_datareader_t {
         ASSERT(dh_);
         return dh_->read(buf, length);
     }
+    long size() {
+        ASSERT(dh_);
+        return dh_->size();
+    }
     void set(DataHandle* dh) {
-        if (dh_)
-            delete dh_;
+        delete dh_;
         dh_ = dh;
     }
-    
+
 private:
     DataHandle* dh_;
 };
@@ -554,6 +556,12 @@ int fdb_datareader_read(fdb_datareader_t* dr, void *buf, long count, long* read)
         *read = dr->read(buf, count);
     });
 }
+int fdb_datareader_size(fdb_datareader_t* dr, long* size) {
+    return wrapApiFunction([=]{
+        ASSERT(dr);
+        *size = dr->size();
+    });
+}
 int fdb_delete_datareader(fdb_datareader_t* dr) {
     return wrapApiFunction([dr]{
         ASSERT(dr);

diff --git a/src/fdb5/api/fdb_c.h b/src/fdb5/api/fdb_c.h
@@ -279,6 +279,13 @@ int fdb_datareader_seek(fdb_datareader_t* dr, long pos);
  */
 int fdb_datareader_skip(fdb_datareader_t* dr, long count);
 
+/** Return size of internal datahandle in bytes.
+ * \param dr DataReader instance
+ * \param size Size of the DataReader
+ * \returns Return code (#FdbErrorValues)
+ */
+int fdb_datareader_size(fdb_datareader_t* dr, long* size);
+
 /** Read binary data from a DataReader to a given memory buffer.
  * \param dr DataReader instance
  * \param buf Pointer of the target memory buffer.

diff --git a/src/fdb5/daos/DaosCatalogueWriter.cc b/src/fdb5/daos/DaosCatalogueWriter.cc
@@ -8,7 +8,7 @@
  * does it submit to any jurisdiction.
  */
 
-#include <limits.h>
+#include <climits>
 #include <numeric>
 
 #include "eckit/io/FileHandle.h"

diff --git a/src/fdb5/daos/DaosIndex.cc b/src/fdb5/daos/DaosIndex.cc
@@ -8,7 +8,7 @@
  * does it submit to any jurisdiction.
  */
 
-#include <limits.h>  // for PATH_MAX
+#include <climits>  // for PATH_MAX
 
 #include "eckit/io/MemoryHandle.h"
 #include "eckit/serialisation/MemoryStream.h"

diff --git a/src/fdb5/database/Archiver.cc b/src/fdb5/database/Archiver.cc
@@ -11,6 +11,8 @@
 
 #include "fdb5/database/Archiver.h"
 
+#include <ctime>
+
 #include "eckit/config/Resource.h"
 
 #include "fdb5/LibFdb5.h"

diff --git a/src/fdb5/database/Archiver.h b/src/fdb5/database/Archiver.h
@@ -16,7 +16,6 @@
 #ifndef fdb5_Archiver_H
 #define fdb5_Archiver_H
 
-#include <time.h>
 #include <utility>
 
 #include "eckit/memory/NonCopyable.h"

diff --git a/src/fdb5/database/Report.h b/src/fdb5/database/Report.h
@@ -17,6 +17,7 @@
 #define fdb5_Report_H
 
 #include <map>
+#include <set>
 
 #include "eckit/memory/NonCopyable.h"
 

diff --git a/src/fdb5/io/LustreFileHandle.h b/src/fdb5/io/LustreFileHandle.h
@@ -56,18 +56,19 @@ class LustreFileHandle : public HANDLE {
 
     void openForAppend(const eckit::Length& len) override {
 
-        std::string path = HANDLE::path_;
+        std::string pathStr = HANDLE::path_;
+        eckit::PathName path{pathStr};
 
-        if(eckit::PathName(path).exists()) return; //< Lustre API outputs ioctl error messages when called on files exist
+        if(path.exists()) return; //< Lustre API outputs ioctl error messages when called on files exist
 
         /* From the docs: llapi_file_create closes the file descriptor. You must re-open the file afterwards */
 
-        LOG_DEBUG_LIB(LibFdb5) << "Creating Lustre file " << path
+        LOG_DEBUG_LIB(LibFdb5) << "Creating Lustre file " << pathStr
                                     << " with " << stripe_.count_ << " stripes "
                                     << "of " << eckit::Bytes(stripe_.size_)
                                     << std::endl;
 
-        int err = fdb5LustreapiFileCreate(path.c_str(), stripe_);
+        int err = fdb5LustreapiFileCreate(path, stripe_);
 
         if(err == EINVAL) {
 

diff --git a/src/fdb5/io/LustreSettings.cc b/src/fdb5/io/LustreSettings.cc
@@ -15,8 +15,11 @@
 #include "fdb5/fdb5_config.h"
 #include "fdb5/LibFdb5.h"
 
+#define LL_SUPER_MAGIC  0x0BD00BD0
 
 #if defined(fdb5_HAVE_LUSTRE)
+#include <sys/vfs.h>
+
 extern "C" {
 void fdb5_lustreapi_silence_msg();
 int  fdb5_lustreapi_file_create(const char* path, size_t stripesize, size_t stripecount);
@@ -35,19 +38,24 @@ bool fdb5LustreapiSupported() {
 #endif
 }
 
-int fdb5LustreapiFileCreate(const char* path, LustreStripe stripe) {
+int fdb5LustreapiFileCreate(const eckit::PathName& path, LustreStripe stripe) {
 
 #if defined(fdb5_HAVE_LUSTRE)
 
-    static bool lustreapi_silence = false;
+    struct statfs buf;
 
-    if(not lustreapi_silence) {
-        fdb5_lustreapi_silence_msg();
-        lustreapi_silence = true;
-    }
+    statfs(path.dirName().localPath(), &buf);
+    if (buf.f_type == LL_SUPER_MAGIC) {
 
-    return fdb5_lustreapi_file_create(path, stripe.size_, stripe.count_);
+        static bool lustreapi_silence = false;
 
+        if(not lustreapi_silence) {
+            fdb5_lustreapi_silence_msg();
+            lustreapi_silence = true;
+        }
+
+        return fdb5_lustreapi_file_create(path.localPath(), stripe.size_, stripe.count_);
+    }
 #endif
 
     /// @note since fdb5LustreapiSupported() should be guarding all calls to this function,

diff --git a/src/fdb5/io/LustreSettings.h b/src/fdb5/io/LustreSettings.h
@@ -16,6 +16,7 @@
 
 #include <cstddef>
 
+#include "eckit/filesystem/PathName.h"
 namespace fdb5 {
 
 //----------------------------------------------------------------------------------------------------------------------
@@ -42,7 +43,7 @@ LustreStripe stripeDataLustreSettings();
 
 //----------------------------------------------------------------------------------------------------------------------
 
-int fdb5LustreapiFileCreate(const char* path, LustreStripe stripe);
+int fdb5LustreapiFileCreate(const eckit::PathName& path, LustreStripe stripe);
 
 bool fdb5LustreapiSupported();
 

diff --git a/src/fdb5/io/SingleGribMungePartFileHandle.cc b/src/fdb5/io/SingleGribMungePartFileHandle.cc
@@ -15,7 +15,7 @@
 
 #include "eckit/log/Log.h"
 
-#include <stdio.h>
+#include <cstdio>
 #include "eccodes.h"
 
 using namespace eckit;

diff --git a/src/fdb5/pmem/DataPoolManager.h b/src/fdb5/pmem/DataPoolManager.h
@@ -20,7 +20,7 @@
 #ifndef fdb5_pmem_DataPoolManager_H
 #define fdb5_pmem_DataPoolManager_H
 
-#include <stdint.h>
+#include <cstdint>
 #include <iosfwd>
 #include <map>
 

diff --git a/src/fdb5/remote/AvailablePortList.h b/src/fdb5/remote/AvailablePortList.h
@@ -20,7 +20,7 @@
 #define fdb5_remote_AvailablePortList_H
 
 #include <sys/types.h>
-#include <time.h>
+#include <ctime>
 #include <utility>
 
 #include "eckit/container/SharedMemArray.h"

diff --git a/src/fdb5/remote/FdbServer.cc b/src/fdb5/remote/FdbServer.cc
@@ -13,11 +13,13 @@
  * (Project ID: 671951) www.nextgenio.eu
  */
 
+#include "fdb5/remote/FdbServer.h"
+
+#include <cstdlib>
+
 #include "eckit/thread/Thread.h"
 #include "eckit/thread/ThreadControler.h"
 
-#include "fdb5/remote/FdbServer.h"
-
 #include "fdb5/remote/AvailablePortList.h"
 #include "fdb5/remote/Handler.h"
 

diff --git a/src/fdb5/remote/FdbServer.h b/src/fdb5/remote/FdbServer.h
@@ -16,7 +16,6 @@
 #define fdb5_remote_FdbServer_H
 
 #include <unistd.h>
-#include <stdlib.h>
 #include <thread>
 
 #include "eckit/net/Port.h"

diff --git a/src/fdb5/toc/TocCatalogueWriter.cc b/src/fdb5/toc/TocCatalogueWriter.cc
@@ -59,7 +59,7 @@ bool TocCatalogueWriter::selectIndex(const Key& idxKey) {
 
         // Enforce lustre striping if requested
         if (stripeLustre()) {
-            fdb5LustreapiFileCreate(indexPath.localPath(), stripeIndexLustreSettings());
+            fdb5LustreapiFileCreate(indexPath, stripeIndexLustreSettings());
         }
 
         indexes_[idxKey] = Index(new TocIndex(idxKey, *this, indexPath, 0, TocIndex::WRITE));
@@ -81,7 +81,7 @@ bool TocCatalogueWriter::selectIndex(const Key& idxKey) {
 
             // Enforce lustre striping if requested
             if (stripeLustre()) {
-                fdb5LustreapiFileCreate(indexPath.localPath(), stripeIndexLustreSettings());
+                fdb5LustreapiFileCreate(indexPath, stripeIndexLustreSettings());
             }
 
             fullIndexes_[idxKey] = Index(new TocIndex(idxKey, *this, indexPath, 0, TocIndex::WRITE));

diff --git a/src/fdb5/toc/TocHandler.cc b/src/fdb5/toc/TocHandler.cc
@@ -724,7 +724,7 @@ class SubtocPreloader {
                 int fd;
                 SYSCALL2((fd = ::open(path.localPath(), iomode)), path);
                 closers.emplace_back(AutoFDCloser{fd});
-                eckit::Length tocSize = 2*1024*1024;
+                eckit::Length tocSize = path.size();
 
                 aiocb& aio(aiocbs[i]);
                 zero(aio);
@@ -801,9 +801,8 @@ class SubtocPreloader {
             }
         }
 #else
-    NOTIMP
+    NOTIMP;
 #endif // eckit_HAVE_AIO
-
         return std::move(subTocReadCache_);
     }
 
@@ -938,7 +937,7 @@ void TocHandler::writeInitRecord(const Key& key) {
 
     // enforce lustre striping if requested
     if (stripeLustre() && !tocPath_.exists()) {
-        fdb5LustreapiFileCreate(tocPath_.localPath(), stripeIndexLustreSettings());
+        fdb5LustreapiFileCreate(tocPath_, stripeIndexLustreSettings());
     }
 
     ASSERT(fd_ == -1);
@@ -976,7 +975,7 @@ void TocHandler::writeInitRecord(const Key& key) {
             // LustreFileHandle<eckit::FileHandle> out(tmp, stripeIndexLustreSettings());
 
             if (stripeLustre()) {
-                fdb5LustreapiFileCreate(tmp.localPath(), stripeIndexLustreSettings());
+                fdb5LustreapiFileCreate(tmp, stripeIndexLustreSettings());
             }
             eckit::FileHandle out(tmp);
             in.copyTo(out);

diff --git a/src/fdb5/toc/TocMoveVisitor.cc b/src/fdb5/toc/TocMoveVisitor.cc
@@ -23,7 +23,6 @@
 #include "fdb5/toc/RootManager.h"
 
 #include <dirent.h>
-#include <errno.h>
 #include <fdb5/LibFdb5.h>
 #include <sys/types.h>
 #include <sys/file.h>

diff --git a/src/fdb5/toc/TocRecord.h b/src/fdb5/toc/TocRecord.h
@@ -15,7 +15,7 @@
 #define fdb5_TocRecord_H
 
 #include <unistd.h>
-#include <time.h>
+#include <ctime>
 #include <sys/time.h>
 
 #include "eckit/types/FixedString.h"

diff --git a/src/fdb5/toc/TocWipeVisitor.cc b/src/fdb5/toc/TocWipeVisitor.cc
@@ -19,7 +19,7 @@
 #include "fdb5/toc/TocWipeVisitor.h"
 
 #include <dirent.h>
-#include <errno.h>
+#include <cerrno>
 #include <fdb5/LibFdb5.h>
 #include <sys/types.h>
 #include <cstring>

diff --git a/tests/fdb/api/test_auxiliary.cc b/tests/fdb/api/test_auxiliary.cc
@@ -1,4 +1,6 @@
 #include "eckit/testing/Test.h"
+#include "eckit/filesystem/TmpDir.h"
+#include "eckit/filesystem/LocalPathName.h"
 #include "metkit/mars/MarsRequest.h"
 #include "fdb5/api/FDB.h"
 #include "fdb5/api/helpers/FDBToolRequest.h"
@@ -63,9 +65,10 @@ std::set<eckit::PathName> setup(FDB& fdb) {
 //----------------------------------------------------------------------------------------------------------------------
 
 CASE("Wipe with extensions") {
+    eckit::TmpDir tmpdir(eckit::LocalPathName::cwd().c_str());
+    eckit::testing::SetEnv env_config{"FDB_ROOT_DIRECTORY", tmpdir.asString().c_str()};
 
-    ::setenv("FDB_AUX_EXTENSIONS", "foo,bar", 1);
-
+    eckit::testing::SetEnv env_extensions{"FDB_AUX_EXTENSIONS", "foo,bar"};
     FDB fdb;
     std::set<eckit::PathName> auxPaths = setup(fdb);
     EXPECT(auxPaths.size() == 6);
@@ -90,9 +93,10 @@ CASE("Wipe with extensions") {
 }
 
 CASE("Purge with extensions") {
+    eckit::TmpDir tmpdir(eckit::LocalPathName::cwd().c_str());
+    eckit::testing::SetEnv env_config{"FDB_ROOT_DIRECTORY", tmpdir.asString().c_str()};
 
-    ::setenv("FDB_AUX_EXTENSIONS", "foo,bar", 1);
-
+    eckit::testing::SetEnv env_extensions{"FDB_AUX_EXTENSIONS", "foo,bar"};
     std::set<eckit::PathName> auxPathsDelete;
 
     // Archive the same data three times