From b0ef4d38057c7504600e30dd85a7bfda90c104b3 Mon Sep 17 00:00:00 2001 From: Chris Bradley Date: Thu, 3 Oct 2024 18:54:52 +0100 Subject: [PATCH 01/11] Use tmpdir in callback and auxiliary tests --- tests/fdb/api/test_auxiliary.cc | 9 +++++++-- tests/fdb/api/test_callback.cc | 5 +++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/fdb/api/test_auxiliary.cc b/tests/fdb/api/test_auxiliary.cc index d559c8b29..67ccfbee9 100644 --- a/tests/fdb/api/test_auxiliary.cc +++ b/tests/fdb/api/test_auxiliary.cc @@ -1,4 +1,5 @@ #include "eckit/testing/Test.h" +#include "eckit/filesystem/TmpDir.h" #include "metkit/mars/MarsRequest.h" #include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" @@ -64,8 +65,10 @@ std::set setup(FDB& fdb) { CASE("Wipe with extensions") { - ::setenv("FDB_AUX_EXTENSIONS", "foo,bar", 1); + eckit::TmpDir tmpdir; + eckit::testing::SetEnv env_config{"FDB_ROOT_DIRECTORY", tmpdir.asString().c_str()}; + eckit::testing::SetEnv env_extensions{"FDB_AUX_EXTENSIONS", "foo,bar"}; FDB fdb; std::set auxPaths = setup(fdb); EXPECT(auxPaths.size() == 6); @@ -91,8 +94,10 @@ CASE("Wipe with extensions") { CASE("Purge with extensions") { - ::setenv("FDB_AUX_EXTENSIONS", "foo,bar", 1); + eckit::TmpDir tmpdir; + eckit::testing::SetEnv env_config{"FDB_ROOT_DIRECTORY", tmpdir.asString().c_str()}; + eckit::testing::SetEnv env_extensions{"FDB_AUX_EXTENSIONS", "foo,bar"}; std::set auxPathsDelete; // Archive the same data three times diff --git a/tests/fdb/api/test_callback.cc b/tests/fdb/api/test_callback.cc index d27d81d73..2fdcfd556 100644 --- a/tests/fdb/api/test_callback.cc +++ b/tests/fdb/api/test_callback.cc @@ -1,10 +1,15 @@ #include "eckit/testing/Test.h" +#include "eckit/filesystem/TmpDir.h" #include "fdb5/api/FDB.h" namespace fdb5::test { //---------------------------------------------------------------------------------------------------------------------- CASE("Archive and flush callback") { + + eckit::TmpDir tmpdir; + eckit::testing::SetEnv env_config{"FDB_ROOT_DIRECTORY", tmpdir.asString().c_str()}; + FDB fdb; std::string data_str = "Raining cats and dogs"; From 1b962e51b5a239fa2e126c7f8c62007bd0bf8305 Mon Sep 17 00:00:00 2001 From: Chris Bradley Date: Mon, 18 Nov 2024 15:35:15 +0000 Subject: [PATCH 02/11] Add datareader_size to C api --- src/fdb5/api/fdb_c.cc | 10 ++++++++++ src/fdb5/api/fdb_c.h | 7 +++++++ 2 files changed, 17 insertions(+) diff --git a/src/fdb5/api/fdb_c.cc b/src/fdb5/api/fdb_c.cc index 9e49f384f..6a34344e7 100644 --- a/src/fdb5/api/fdb_c.cc +++ b/src/fdb5/api/fdb_c.cc @@ -193,6 +193,10 @@ struct fdb_datareader_t { ASSERT(dh_); return dh_->read(buf, length); } + long size() { + ASSERT(dh_); + return dh_->size(); + } void set(DataHandle* dh) { if (dh_) delete dh_; @@ -554,6 +558,12 @@ int fdb_datareader_read(fdb_datareader_t* dr, void *buf, long count, long* read) *read = dr->read(buf, count); }); } +int fdb_datareader_size(fdb_datareader_t* dr, long* size) { + return wrapApiFunction([=]{ + ASSERT(dr); + *size = dr->size(); + }); +} int fdb_delete_datareader(fdb_datareader_t* dr) { return wrapApiFunction([dr]{ ASSERT(dr); diff --git a/src/fdb5/api/fdb_c.h b/src/fdb5/api/fdb_c.h index df569fabe..acaa1186f 100644 --- a/src/fdb5/api/fdb_c.h +++ b/src/fdb5/api/fdb_c.h @@ -279,6 +279,13 @@ int fdb_datareader_seek(fdb_datareader_t* dr, long pos); */ int fdb_datareader_skip(fdb_datareader_t* dr, long count); +/** Return size of internal datahandle in bytes. + * \param dr DataReader instance + * \param size Size of the DataReader + * \returns Return code (#FdbErrorValues) + */ +int fdb_datareader_size(fdb_datareader_t* dr, long* size); + /** Read binary data from a DataReader to a given memory buffer. * \param dr DataReader instance * \param buf Pointer of the target memory buffer. From bc732bdad2be910923dca5cb8bebdcce938c7bf5 Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Tue, 19 Nov 2024 10:44:42 +0000 Subject: [PATCH 03/11] replaced c headers --- src/dummy_daos/daos.h | 8 ++++---- src/fdb5/api/RandomFDB.cc | 2 +- src/fdb5/daos/DaosCatalogueWriter.cc | 2 +- src/fdb5/daos/DaosIndex.cc | 2 +- src/fdb5/database/Archiver.h | 2 +- src/fdb5/io/SingleGribMungePartFileHandle.cc | 2 +- src/fdb5/pmem/DataPoolManager.h | 2 +- src/fdb5/remote/AvailablePortList.h | 2 +- src/fdb5/remote/FdbServer.h | 2 +- src/fdb5/toc/TocMoveVisitor.cc | 3 +-- src/fdb5/toc/TocRecord.h | 2 +- src/fdb5/toc/TocWipeVisitor.cc | 2 +- tests/fdb/api/test_fdb_c.cc | 2 +- tests/fdb/type/test_toKey.cc | 2 +- 14 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/dummy_daos/daos.h b/src/dummy_daos/daos.h index 23c57fc65..8dd18e987 100644 --- a/src/dummy_daos/daos.h +++ b/src/dummy_daos/daos.h @@ -16,12 +16,12 @@ #pragma once -#include -#include +#include +#include #include #include -#include -#include +#include +#include //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/api/RandomFDB.cc b/src/fdb5/api/RandomFDB.cc index 69c6946cc..fef26ac97 100644 --- a/src/fdb5/api/RandomFDB.cc +++ b/src/fdb5/api/RandomFDB.cc @@ -19,7 +19,7 @@ #include "fdb5/api/FDBFactory.h" #include "fdb5/LibFdb5.h" -#include +#include #include using namespace eckit; diff --git a/src/fdb5/daos/DaosCatalogueWriter.cc b/src/fdb5/daos/DaosCatalogueWriter.cc index 1caed881d..adacd07b1 100644 --- a/src/fdb5/daos/DaosCatalogueWriter.cc +++ b/src/fdb5/daos/DaosCatalogueWriter.cc @@ -8,7 +8,7 @@ * does it submit to any jurisdiction. */ -#include +#include #include #include "eckit/io/FileHandle.h" diff --git a/src/fdb5/daos/DaosIndex.cc b/src/fdb5/daos/DaosIndex.cc index bab6b152c..05a017e67 100644 --- a/src/fdb5/daos/DaosIndex.cc +++ b/src/fdb5/daos/DaosIndex.cc @@ -8,7 +8,7 @@ * does it submit to any jurisdiction. */ -#include // for PATH_MAX +#include // for PATH_MAX #include "eckit/io/MemoryHandle.h" #include "eckit/serialisation/MemoryStream.h" diff --git a/src/fdb5/database/Archiver.h b/src/fdb5/database/Archiver.h index 0d90a4524..24f1826cd 100644 --- a/src/fdb5/database/Archiver.h +++ b/src/fdb5/database/Archiver.h @@ -16,7 +16,7 @@ #ifndef fdb5_Archiver_H #define fdb5_Archiver_H -#include +#include #include #include "eckit/memory/NonCopyable.h" diff --git a/src/fdb5/io/SingleGribMungePartFileHandle.cc b/src/fdb5/io/SingleGribMungePartFileHandle.cc index 3d654922f..591c4f7d2 100644 --- a/src/fdb5/io/SingleGribMungePartFileHandle.cc +++ b/src/fdb5/io/SingleGribMungePartFileHandle.cc @@ -15,7 +15,7 @@ #include "eckit/log/Log.h" -#include +#include #include "eccodes.h" using namespace eckit; diff --git a/src/fdb5/pmem/DataPoolManager.h b/src/fdb5/pmem/DataPoolManager.h index c0cc4fbd8..7dd4ecb6b 100644 --- a/src/fdb5/pmem/DataPoolManager.h +++ b/src/fdb5/pmem/DataPoolManager.h @@ -20,7 +20,7 @@ #ifndef fdb5_pmem_DataPoolManager_H #define fdb5_pmem_DataPoolManager_H -#include +#include #include #include diff --git a/src/fdb5/remote/AvailablePortList.h b/src/fdb5/remote/AvailablePortList.h index 7aa0698a9..64aa805af 100644 --- a/src/fdb5/remote/AvailablePortList.h +++ b/src/fdb5/remote/AvailablePortList.h @@ -20,7 +20,7 @@ #define fdb5_remote_AvailablePortList_H #include -#include +#include #include #include "eckit/container/SharedMemArray.h" diff --git a/src/fdb5/remote/FdbServer.h b/src/fdb5/remote/FdbServer.h index 4eeef21d6..6e568d811 100644 --- a/src/fdb5/remote/FdbServer.h +++ b/src/fdb5/remote/FdbServer.h @@ -16,7 +16,7 @@ #define fdb5_remote_FdbServer_H #include -#include +#include #include #include "eckit/net/Port.h" diff --git a/src/fdb5/toc/TocMoveVisitor.cc b/src/fdb5/toc/TocMoveVisitor.cc index fd234aa76..0d4ba437c 100644 --- a/src/fdb5/toc/TocMoveVisitor.cc +++ b/src/fdb5/toc/TocMoveVisitor.cc @@ -22,8 +22,7 @@ #include "fdb5/toc/TocMoveVisitor.h" #include "fdb5/toc/RootManager.h" -#include -#include +#include #include #include #include diff --git a/src/fdb5/toc/TocRecord.h b/src/fdb5/toc/TocRecord.h index d38b4ad86..3cf193664 100644 --- a/src/fdb5/toc/TocRecord.h +++ b/src/fdb5/toc/TocRecord.h @@ -15,7 +15,7 @@ #define fdb5_TocRecord_H #include -#include +#include #include #include "eckit/types/FixedString.h" diff --git a/src/fdb5/toc/TocWipeVisitor.cc b/src/fdb5/toc/TocWipeVisitor.cc index c5682691a..26d69b350 100644 --- a/src/fdb5/toc/TocWipeVisitor.cc +++ b/src/fdb5/toc/TocWipeVisitor.cc @@ -19,7 +19,7 @@ #include "fdb5/toc/TocWipeVisitor.h" #include -#include +#include #include #include #include diff --git a/tests/fdb/api/test_fdb_c.cc b/tests/fdb/api/test_fdb_c.cc index 57c1f486c..7d8d34c5f 100644 --- a/tests/fdb/api/test_fdb_c.cc +++ b/tests/fdb/api/test_fdb_c.cc @@ -8,7 +8,7 @@ * does it submit to any jurisdiction. */ -#include +#include #include "eckit/config/Resource.h" #include "eckit/filesystem/PathName.h" diff --git a/tests/fdb/type/test_toKey.cc b/tests/fdb/type/test_toKey.cc index f026cb958..d37abca6c 100644 --- a/tests/fdb/type/test_toKey.cc +++ b/tests/fdb/type/test_toKey.cc @@ -9,7 +9,7 @@ */ #include -#include +#include #include "eckit/testing/Test.h" From d8d6fe6607779a0c809bbf14482e96e50cc8fd74 Mon Sep 17 00:00:00 2001 From: Simon Smart Date: Tue, 19 Nov 2024 14:10:06 +0000 Subject: [PATCH 04/11] Enable teams notifications --- .github/workflows/ci.yml | 18 ++++++++++++++++++ .github/workflows/notify-issues.yml | 14 ++++++++++++++ .github/workflows/notify-pull-request.yml | 14 ++++++++++++++ 3 files changed, 46 insertions(+) create mode 100644 .github/workflows/notify-issues.yml create mode 100644 .github/workflows/notify-pull-request.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 14265a520..c27e0c1cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,3 +74,21 @@ jobs: repository: private-downstream-ci event_type: downstream-ci-hpc payload: '{"fdb": "ecmwf/fdb@${{ github.event.pull_request.head.sha || github.sha }}"}' + + notify: + runs-on: ubuntu-latest + needs: + - downstream-ci + - private-downstream-ci + - downstream-ci-hpc + - private-downstream-ci-hpc + if: ${{ always() && !github.event.pull_request.head.repo.fork && (github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci') }} + steps: + - name: Trigger Teams notification + if: failure() && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop') + uses: ecmwf-actions/notify-teams@v1 + with: + incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }} + needs_context: ${{ toJSON(needs) }} + notify_on: | + failure diff --git a/.github/workflows/notify-issues.yml b/.github/workflows/notify-issues.yml new file mode 100644 index 000000000..282bb705d --- /dev/null +++ b/.github/workflows/notify-issues.yml @@ -0,0 +1,14 @@ +name: Notify Issues + +on: + issues: + types: [opened, reopened] + +jobs: + notify: + runs-on: ubuntu-latest + steps: + - name: Notify Issues + uses: ecmwf-actions/notify-teams-issue@v1 + with: + incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }} diff --git a/.github/workflows/notify-pull-request.yml b/.github/workflows/notify-pull-request.yml new file mode 100644 index 000000000..7d1ab72bc --- /dev/null +++ b/.github/workflows/notify-pull-request.yml @@ -0,0 +1,14 @@ +name: Notify Pull Request + +on: + pull_request_target: + types: [opened, reopened] + +jobs: + notify: + runs-on: ubuntu-latest + steps: + - name: Notify Pull Request + uses: ecmwf-actions/notify-teams-pr@v1 + with: + incoming_webhook: ${{ secrets.MS_TEAMS_INCOMING_WEBHOOK }} From ced977cb9c9f90d0ee7d89fe50f56fbd2e88b8d5 Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Tue, 19 Nov 2024 18:46:54 +0000 Subject: [PATCH 05/11] addressed PR comments --- src/dummy_daos/daos.h | 7 +++---- src/fdb5/api/RandomFDB.cc | 1 - src/fdb5/database/Archiver.cc | 2 ++ src/fdb5/database/Archiver.h | 1 - src/fdb5/remote/FdbServer.cc | 6 ++++-- src/fdb5/remote/FdbServer.h | 1 - src/fdb5/toc/TocMoveVisitor.cc | 2 +- tests/fdb/type/test_toKey.cc | 1 - 8 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/dummy_daos/daos.h b/src/dummy_daos/daos.h index 8dd18e987..201c42d80 100644 --- a/src/dummy_daos/daos.h +++ b/src/dummy_daos/daos.h @@ -16,12 +16,11 @@ #pragma once -#include -#include +#include +#include #include #include -#include -#include +#include //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/api/RandomFDB.cc b/src/fdb5/api/RandomFDB.cc index fef26ac97..286993b5a 100644 --- a/src/fdb5/api/RandomFDB.cc +++ b/src/fdb5/api/RandomFDB.cc @@ -19,7 +19,6 @@ #include "fdb5/api/FDBFactory.h" #include "fdb5/LibFdb5.h" -#include #include using namespace eckit; diff --git a/src/fdb5/database/Archiver.cc b/src/fdb5/database/Archiver.cc index 23a08efe7..e5b4de3ad 100644 --- a/src/fdb5/database/Archiver.cc +++ b/src/fdb5/database/Archiver.cc @@ -11,6 +11,8 @@ #include "fdb5/database/Archiver.h" +#include + #include "eckit/config/Resource.h" #include "fdb5/LibFdb5.h" diff --git a/src/fdb5/database/Archiver.h b/src/fdb5/database/Archiver.h index 24f1826cd..45a67c77f 100644 --- a/src/fdb5/database/Archiver.h +++ b/src/fdb5/database/Archiver.h @@ -16,7 +16,6 @@ #ifndef fdb5_Archiver_H #define fdb5_Archiver_H -#include #include #include "eckit/memory/NonCopyable.h" diff --git a/src/fdb5/remote/FdbServer.cc b/src/fdb5/remote/FdbServer.cc index e45b0ca20..358bf2b0a 100644 --- a/src/fdb5/remote/FdbServer.cc +++ b/src/fdb5/remote/FdbServer.cc @@ -13,11 +13,13 @@ * (Project ID: 671951) www.nextgenio.eu */ +#include "fdb5/remote/FdbServer.h" + +#include + #include "eckit/thread/Thread.h" #include "eckit/thread/ThreadControler.h" -#include "fdb5/remote/FdbServer.h" - #include "fdb5/remote/AvailablePortList.h" #include "fdb5/remote/Handler.h" diff --git a/src/fdb5/remote/FdbServer.h b/src/fdb5/remote/FdbServer.h index 6e568d811..cf48a6910 100644 --- a/src/fdb5/remote/FdbServer.h +++ b/src/fdb5/remote/FdbServer.h @@ -16,7 +16,6 @@ #define fdb5_remote_FdbServer_H #include -#include #include #include "eckit/net/Port.h" diff --git a/src/fdb5/toc/TocMoveVisitor.cc b/src/fdb5/toc/TocMoveVisitor.cc index 0d4ba437c..d0f1d1b42 100644 --- a/src/fdb5/toc/TocMoveVisitor.cc +++ b/src/fdb5/toc/TocMoveVisitor.cc @@ -22,7 +22,7 @@ #include "fdb5/toc/TocMoveVisitor.h" #include "fdb5/toc/RootManager.h" -#include +#include #include #include #include diff --git a/tests/fdb/type/test_toKey.cc b/tests/fdb/type/test_toKey.cc index d37abca6c..1ed792351 100644 --- a/tests/fdb/type/test_toKey.cc +++ b/tests/fdb/type/test_toKey.cc @@ -9,7 +9,6 @@ */ #include -#include #include "eckit/testing/Test.h" From 0abc7a0d87238ce713ed826c0c7ef73068808afe Mon Sep 17 00:00:00 2001 From: Kai Kratz Date: Fri, 22 Nov 2024 17:16:01 +0100 Subject: [PATCH 06/11] Add .clang-tidy config with minimal checks The commited .clang-tidy file contains only a single check and no automated tidy runs are done. The added check is readability-delete-null-pointer, i.e. tidy warns on superflous nullptr check before calling delete. This commit also fixes a missing include in Report.h because this clang-tidy would otherwise report this tempalte instanciation as erroneous. --- .clang-tidy | 8 ++++++++ src/fdb5/api/fdb_c.cc | 12 +++++------- src/fdb5/database/Report.h | 1 + 3 files changed, 14 insertions(+), 7 deletions(-) create mode 100644 .clang-tidy diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 000000000..0b704176b --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,8 @@ +--- +Checks: "-*, readability-delete-null-pointer" +WarningsAsErrors: true +HeaderFileExtensions: ['', 'h','hh','hpp','hxx'] +ImplementationFileExtensions: ['c','cc','cpp','cxx'] +HeaderFilterRegex: '' +FormatStyle: none +InheritParentConfig: false diff --git a/src/fdb5/api/fdb_c.cc b/src/fdb5/api/fdb_c.cc index 6a34344e7..eb69bc9d5 100644 --- a/src/fdb5/api/fdb_c.cc +++ b/src/fdb5/api/fdb_c.cc @@ -9,7 +9,6 @@ */ #include "eckit/io/MemoryHandle.h" -#include "eckit/io/FileDescHandle.h" #include "eckit/message/Message.h" #include "eckit/runtime/Main.h" #include "eckit/config/YAMLConfiguration.h" @@ -66,8 +65,8 @@ struct fdb_request_t { Tokenizer parse("/"); for (int i=0; i result; - parse(values[i], result); + std::vector result; + parse(values[i], result); vv.insert(std::end(vv), std::begin(result), std::end(result)); } request_.values(n, vv); @@ -110,7 +109,7 @@ struct fdb_split_key_t { } else { return FDB_ITERATION_COMPLETE; } - } + } while (it_ == key_->at(level_).end()) { if (level_size()-1) { level_++; @@ -198,11 +197,10 @@ struct fdb_datareader_t { return dh_->size(); } void set(DataHandle* dh) { - if (dh_) - delete dh_; + delete dh_; dh_ = dh; } - + private: DataHandle* dh_; }; diff --git a/src/fdb5/database/Report.h b/src/fdb5/database/Report.h index 9b65ed6e3..268f945fe 100644 --- a/src/fdb5/database/Report.h +++ b/src/fdb5/database/Report.h @@ -17,6 +17,7 @@ #define fdb5_Report_H #include +#include #include "eckit/memory/NonCopyable.h" From 8a31f21c0c5a29f0d226552f735611dd790c750a Mon Sep 17 00:00:00 2001 From: Chris Bradley Date: Wed, 27 Nov 2024 13:44:33 +0000 Subject: [PATCH 07/11] Base tmpdir on cwd in auxiliary/callback tests --- tests/fdb/api/test_auxiliary.cc | 7 +++---- tests/fdb/api/test_callback.cc | 3 ++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/fdb/api/test_auxiliary.cc b/tests/fdb/api/test_auxiliary.cc index 67ccfbee9..1f88983d6 100644 --- a/tests/fdb/api/test_auxiliary.cc +++ b/tests/fdb/api/test_auxiliary.cc @@ -1,5 +1,6 @@ #include "eckit/testing/Test.h" #include "eckit/filesystem/TmpDir.h" +#include "eckit/filesystem/LocalPathName.h" #include "metkit/mars/MarsRequest.h" #include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" @@ -64,8 +65,7 @@ std::set setup(FDB& fdb) { //---------------------------------------------------------------------------------------------------------------------- CASE("Wipe with extensions") { - - eckit::TmpDir tmpdir; + eckit::TmpDir tmpdir(eckit::LocalPathName::cwd().c_str()); eckit::testing::SetEnv env_config{"FDB_ROOT_DIRECTORY", tmpdir.asString().c_str()}; eckit::testing::SetEnv env_extensions{"FDB_AUX_EXTENSIONS", "foo,bar"}; @@ -93,8 +93,7 @@ CASE("Wipe with extensions") { } CASE("Purge with extensions") { - - eckit::TmpDir tmpdir; + eckit::TmpDir tmpdir(eckit::LocalPathName::cwd().c_str()); eckit::testing::SetEnv env_config{"FDB_ROOT_DIRECTORY", tmpdir.asString().c_str()}; eckit::testing::SetEnv env_extensions{"FDB_AUX_EXTENSIONS", "foo,bar"}; diff --git a/tests/fdb/api/test_callback.cc b/tests/fdb/api/test_callback.cc index 2fdcfd556..dbcfcd1f7 100644 --- a/tests/fdb/api/test_callback.cc +++ b/tests/fdb/api/test_callback.cc @@ -1,5 +1,6 @@ #include "eckit/testing/Test.h" #include "eckit/filesystem/TmpDir.h" +#include "eckit/filesystem/LocalPathName.h" #include "fdb5/api/FDB.h" namespace fdb5::test { @@ -7,7 +8,7 @@ namespace fdb5::test { //---------------------------------------------------------------------------------------------------------------------- CASE("Archive and flush callback") { - eckit::TmpDir tmpdir; + eckit::TmpDir tmpdir(eckit::LocalPathName::cwd().c_str()); eckit::testing::SetEnv env_config{"FDB_ROOT_DIRECTORY", tmpdir.asString().c_str()}; FDB fdb; From b8a1030f2acde6bbc0a8d38f943d1f43acc50733 Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Wed, 27 Nov 2024 23:17:57 +0000 Subject: [PATCH 08/11] FDB-330 lustre FS identification --- src/fdb5/io/LustreFileHandle.h | 9 +++++---- src/fdb5/io/LustreSettings.cc | 22 +++++++++++++++------- src/fdb5/io/LustreSettings.h | 3 ++- src/fdb5/toc/TocCatalogueWriter.cc | 4 ++-- src/fdb5/toc/TocHandler.cc | 4 ++-- 5 files changed, 26 insertions(+), 16 deletions(-) diff --git a/src/fdb5/io/LustreFileHandle.h b/src/fdb5/io/LustreFileHandle.h index 1baa26cd4..7adcde756 100644 --- a/src/fdb5/io/LustreFileHandle.h +++ b/src/fdb5/io/LustreFileHandle.h @@ -56,18 +56,19 @@ class LustreFileHandle : public HANDLE { void openForAppend(const eckit::Length& len) override { - std::string path = HANDLE::path_; + std::string pathStr = HANDLE::path_; + eckit::PathName path{pathStr}; - if(eckit::PathName(path).exists()) return; //< Lustre API outputs ioctl error messages when called on files exist + if(path.exists()) return; //< Lustre API outputs ioctl error messages when called on files exist /* From the docs: llapi_file_create closes the file descriptor. You must re-open the file afterwards */ - LOG_DEBUG_LIB(LibFdb5) << "Creating Lustre file " << path + LOG_DEBUG_LIB(LibFdb5) << "Creating Lustre file " << pathStr << " with " << stripe_.count_ << " stripes " << "of " << eckit::Bytes(stripe_.size_) << std::endl; - int err = fdb5LustreapiFileCreate(path.c_str(), stripe_); + int err = fdb5LustreapiFileCreate(path, stripe_); if(err == EINVAL) { diff --git a/src/fdb5/io/LustreSettings.cc b/src/fdb5/io/LustreSettings.cc index 898306e68..7845113a6 100644 --- a/src/fdb5/io/LustreSettings.cc +++ b/src/fdb5/io/LustreSettings.cc @@ -15,8 +15,11 @@ #include "fdb5/fdb5_config.h" #include "fdb5/LibFdb5.h" +#define LL_SUPER_MAGIC 0x0BD00BD0 #if defined(fdb5_HAVE_LUSTRE) +#include + extern "C" { void fdb5_lustreapi_silence_msg(); int fdb5_lustreapi_file_create(const char* path, size_t stripesize, size_t stripecount); @@ -35,19 +38,24 @@ bool fdb5LustreapiSupported() { #endif } -int fdb5LustreapiFileCreate(const char* path, LustreStripe stripe) { +int fdb5LustreapiFileCreate(eckit::PathName path, LustreStripe stripe) { #if defined(fdb5_HAVE_LUSTRE) - static bool lustreapi_silence = false; + struct statfs buf; - if(not lustreapi_silence) { - fdb5_lustreapi_silence_msg(); - lustreapi_silence = true; - } + statfs(path.dirName().localPath(), &buf); + if (buf.type == LL_SUPER_MAGIC) { - return fdb5_lustreapi_file_create(path, stripe.size_, stripe.count_); + static bool lustreapi_silence = false; + if(not lustreapi_silence) { + fdb5_lustreapi_silence_msg(); + lustreapi_silence = true; + } + + return fdb5_lustreapi_file_create(path.localPath(), stripe.size_, stripe.count_); + } #endif /// @note since fdb5LustreapiSupported() should be guarding all calls to this function, diff --git a/src/fdb5/io/LustreSettings.h b/src/fdb5/io/LustreSettings.h index 40bb03262..464f4c673 100644 --- a/src/fdb5/io/LustreSettings.h +++ b/src/fdb5/io/LustreSettings.h @@ -16,6 +16,7 @@ #include +#include "eckit/filesystem/PathName.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- @@ -42,7 +43,7 @@ LustreStripe stripeDataLustreSettings(); //---------------------------------------------------------------------------------------------------------------------- -int fdb5LustreapiFileCreate(const char* path, LustreStripe stripe); +int fdb5LustreapiFileCreate(eckit::PathName path, LustreStripe stripe); bool fdb5LustreapiSupported(); diff --git a/src/fdb5/toc/TocCatalogueWriter.cc b/src/fdb5/toc/TocCatalogueWriter.cc index ec91155f5..ed09a618b 100644 --- a/src/fdb5/toc/TocCatalogueWriter.cc +++ b/src/fdb5/toc/TocCatalogueWriter.cc @@ -59,7 +59,7 @@ bool TocCatalogueWriter::selectIndex(const Key& idxKey) { // Enforce lustre striping if requested if (stripeLustre()) { - fdb5LustreapiFileCreate(indexPath.localPath(), stripeIndexLustreSettings()); + fdb5LustreapiFileCreate(indexPath, stripeIndexLustreSettings()); } indexes_[idxKey] = Index(new TocIndex(idxKey, *this, indexPath, 0, TocIndex::WRITE)); @@ -81,7 +81,7 @@ bool TocCatalogueWriter::selectIndex(const Key& idxKey) { // Enforce lustre striping if requested if (stripeLustre()) { - fdb5LustreapiFileCreate(indexPath.localPath(), stripeIndexLustreSettings()); + fdb5LustreapiFileCreate(indexPath, stripeIndexLustreSettings()); } fullIndexes_[idxKey] = Index(new TocIndex(idxKey, *this, indexPath, 0, TocIndex::WRITE)); diff --git a/src/fdb5/toc/TocHandler.cc b/src/fdb5/toc/TocHandler.cc index 86bf55cb9..b2fbadf08 100644 --- a/src/fdb5/toc/TocHandler.cc +++ b/src/fdb5/toc/TocHandler.cc @@ -938,7 +938,7 @@ void TocHandler::writeInitRecord(const Key& key) { // enforce lustre striping if requested if (stripeLustre() && !tocPath_.exists()) { - fdb5LustreapiFileCreate(tocPath_.localPath(), stripeIndexLustreSettings()); + fdb5LustreapiFileCreate(tocPath_, stripeIndexLustreSettings()); } ASSERT(fd_ == -1); @@ -976,7 +976,7 @@ void TocHandler::writeInitRecord(const Key& key) { // LustreFileHandle out(tmp, stripeIndexLustreSettings()); if (stripeLustre()) { - fdb5LustreapiFileCreate(tmp.localPath(), stripeIndexLustreSettings()); + fdb5LustreapiFileCreate(tmp, stripeIndexLustreSettings()); } eckit::FileHandle out(tmp); in.copyTo(out); From 4311b2d29a4c6063c4f8a1a25b3f2ce7ba37536b Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Wed, 27 Nov 2024 23:42:47 +0000 Subject: [PATCH 09/11] FDB-330 lustre FS identification --- src/fdb5/io/LustreSettings.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fdb5/io/LustreSettings.cc b/src/fdb5/io/LustreSettings.cc index 7845113a6..4b3decb87 100644 --- a/src/fdb5/io/LustreSettings.cc +++ b/src/fdb5/io/LustreSettings.cc @@ -45,7 +45,7 @@ int fdb5LustreapiFileCreate(eckit::PathName path, LustreStripe stripe) { struct statfs buf; statfs(path.dirName().localPath(), &buf); - if (buf.type == LL_SUPER_MAGIC) { + if (buf.f_type == LL_SUPER_MAGIC) { static bool lustreapi_silence = false; From ffa93c23023169409affc6f2df7ce666f7ab6c2f Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Thu, 28 Nov 2024 11:42:32 +0000 Subject: [PATCH 10/11] addressed PR + fix toc async read --- src/fdb5/io/LustreSettings.cc | 2 +- src/fdb5/io/LustreSettings.h | 2 +- src/fdb5/toc/TocHandler.cc | 5 ++--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/fdb5/io/LustreSettings.cc b/src/fdb5/io/LustreSettings.cc index 4b3decb87..5af6ced71 100644 --- a/src/fdb5/io/LustreSettings.cc +++ b/src/fdb5/io/LustreSettings.cc @@ -38,7 +38,7 @@ bool fdb5LustreapiSupported() { #endif } -int fdb5LustreapiFileCreate(eckit::PathName path, LustreStripe stripe) { +int fdb5LustreapiFileCreate(const eckit::PathName& path, LustreStripe stripe) { #if defined(fdb5_HAVE_LUSTRE) diff --git a/src/fdb5/io/LustreSettings.h b/src/fdb5/io/LustreSettings.h index 464f4c673..c3c12d47e 100644 --- a/src/fdb5/io/LustreSettings.h +++ b/src/fdb5/io/LustreSettings.h @@ -43,7 +43,7 @@ LustreStripe stripeDataLustreSettings(); //---------------------------------------------------------------------------------------------------------------------- -int fdb5LustreapiFileCreate(eckit::PathName path, LustreStripe stripe); +int fdb5LustreapiFileCreate(const eckit::PathName& path, LustreStripe stripe); bool fdb5LustreapiSupported(); diff --git a/src/fdb5/toc/TocHandler.cc b/src/fdb5/toc/TocHandler.cc index b2fbadf08..cec2035f8 100644 --- a/src/fdb5/toc/TocHandler.cc +++ b/src/fdb5/toc/TocHandler.cc @@ -724,7 +724,7 @@ class SubtocPreloader { int fd; SYSCALL2((fd = ::open(path.localPath(), iomode)), path); closers.emplace_back(AutoFDCloser{fd}); - eckit::Length tocSize = 2*1024*1024; + eckit::Length tocSize = path.size(); aiocb& aio(aiocbs[i]); zero(aio); @@ -801,9 +801,8 @@ class SubtocPreloader { } } #else - NOTIMP + NOTIMP; #endif // eckit_HAVE_AIO - return std::move(subTocReadCache_); } From a84c033740c2ac2f91555fa26903a06bf0356d16 Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Thu, 28 Nov 2024 13:15:08 +0000 Subject: [PATCH 11/11] version bump --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index e2b3d6a8a..d135a3a8f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -5.13.3 \ No newline at end of file +5.13.4