Skip to content

Commit a789091

Browse files
danovarodanovaro
committed
Merge branch 'develop' into hotfix/5.13.2
2 parents 2fccbdc + 1001d61 commit a789091

File tree

205 files changed

+2105
-1392
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

205 files changed

+2105
-1392
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
11
.vscode
2+
build

‎src/fdb5/LibFdb5.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
#include "eckit/system/Library.h"
2121

2222
#include "fdb5/database/DB.h"
23-
#include "fdb5/types/TypesRegistry.h"
2423
#include "fdb5/api/helpers/Callback.h"
2524

2625
namespace fdb5 {

‎src/fdb5/api/DistFDB.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ class DistFDB : public FDBBase {
6969

7070
private: // methods
7171

72-
virtual void print(std::ostream& s) const override;
72+
void print(std::ostream& s) const override;
7373

7474
template <typename QueryFN>
7575
auto queryInternal(const FDBToolRequest& request, const QueryFN& fn) -> decltype(fn(*(FDB*)(nullptr), request));

‎src/fdb5/api/FDB.cc

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,6 @@
2020
#include "eckit/message/Message.h"
2121
#include "eckit/message/Reader.h"
2222

23-
#include "eckit/system/Plugin.h"
24-
#include "eckit/system/LibraryManager.h"
25-
2623
#include "metkit/hypercube/HyperCubePayloaded.h"
2724

2825
#include "fdb5/LibFdb5.h"
@@ -34,6 +31,8 @@
3431
#include "fdb5/message/MessageDecoder.h"
3532
#include "fdb5/types/Type.h"
3633

34+
#include <memory>
35+
3736
namespace fdb5 {
3837

3938
//----------------------------------------------------------------------------------------------------------------------
@@ -109,15 +108,15 @@ void FDB::archive(const Key& key, const void* data, size_t length) {
109108
// This is the API entrypoint. Keys supplied by the user may not have type registry info attached (so
110109
// serialisation won't work properly...)
111110
Key keyInternal(key);
112-
keyInternal.registry(config().schema().registry());
113111

114112
// step in archival requests from the model is just an integer. We need to include the stepunit
115113
auto stepunit = keyInternal.find("stepunits");
116114
if (stepunit != keyInternal.end()) {
117-
if (stepunit->second.size()>0 && stepunit->second[0]!='h') {
115+
if (stepunit->second.size()>0 && static_cast<char>(tolower(stepunit->second[0])) != 'h') {
118116
auto step = keyInternal.find("step");
119117
if (step != keyInternal.end()) {
120-
std::string canonicalStep = keyInternal.registry().lookupType("step").toKey("step", step->second+stepunit->second);
118+
std::string canonicalStep = config().schema().registry().lookupType("step").toKey(step->second + static_cast<char>(tolower(stepunit->second[0])));
119+
keyInternal.set("step", canonicalStep);
121120
}
122121
}
123122
keyInternal.unset("stepunits");
@@ -154,21 +153,20 @@ class ListElementDeduplicator : public metkit::hypercube::Deduplicator<ListEleme
154153
};
155154

156155
eckit::DataHandle* FDB::read(const eckit::URI& uri) {
157-
FieldLocation* loc = FieldLocationFactory::instance().build(uri.scheme(), uri);
158-
return loc->dataHandle();
156+
auto location = std::unique_ptr<FieldLocation>(FieldLocationFactory::instance().build(uri.scheme(), uri));
157+
return location->dataHandle();
159158
}
160159

161160
eckit::DataHandle* FDB::read(const std::vector<eckit::URI>& uris, bool sorted) {
162161
HandleGatherer result(sorted);
163162

164163
for (const eckit::URI& uri : uris) {
165-
FieldLocation* loc = FieldLocationFactory::instance().build(uri.scheme(), uri);
166-
result.add(loc->dataHandle());
167-
delete loc;
164+
auto location = std::unique_ptr<FieldLocation>(FieldLocationFactory::instance().build(uri.scheme(), uri));
165+
result.add(location->dataHandle());
168166
}
167+
169168
return result.dataHandle();
170169
}
171-
172170

173171
eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) {
174172
eckit::Timer timer;
@@ -303,13 +301,17 @@ void FDB::flush() {
303301
IndexAxis FDB::axes(const FDBToolRequest& request, int level) {
304302
IndexAxis axes;
305303
AxesElement elem;
306-
auto it = internal_->axes(request, level);
304+
auto it = axesIterator(request, level);
307305
while (it.next(elem)) {
308306
axes.merge(elem.axes());
309307
}
310308
return axes;
311309
}
312310

311+
AxesIterator FDB::axesIterator(const FDBToolRequest& request, int level) {
312+
return internal_->axesIterator(request, level);
313+
}
314+
313315
bool FDB::dirty() const {
314316
return dirty_;
315317
}

‎src/fdb5/api/FDB.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
#include "fdb5/api/helpers/StatusIterator.h"
3434
#include "fdb5/api/helpers/WipeIterator.h"
3535
#include "fdb5/api/helpers/MoveIterator.h"
36+
#include "fdb5/api/helpers/AxesIterator.h"
3637
#include "fdb5/config/Config.h"
3738
#include "fdb5/api/helpers/Callback.h"
3839

@@ -115,6 +116,8 @@ class FDB {
115116

116117
IndexAxis axes(const FDBToolRequest& request, int level=3);
117118

119+
AxesIterator axesIterator(const FDBToolRequest& request, int level=3);
120+
118121
bool enabled(const ControlIdentifier& controlIdentifier) const;
119122

120123
bool dirty() const;

‎src/fdb5/api/FDBFactory.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ class FDBBase : private eckit::NonCopyable {
8989

9090
virtual MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) = 0;
9191

92-
virtual AxesIterator axes(const FDBToolRequest& request, int axes) { NOTIMP; }
92+
virtual AxesIterator axesIterator(const FDBToolRequest& request, int axes) { NOTIMP; }
9393

9494
void registerArchiveCallback(ArchiveCallback callback) {callback_ = callback;}
9595

@@ -185,7 +185,7 @@ class FDBBuilder : public FDBBuilderBase {
185185

186186
private: // methods
187187

188-
virtual std::unique_ptr<FDBBase> make(const Config& config) const override {
188+
std::unique_ptr<FDBBase> make(const Config& config) const override {
189189
return std::unique_ptr<T>(new T(config, name_));
190190
}
191191
};

‎src/fdb5/api/LocalFDB.cc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -125,8 +125,8 @@ ControlIterator LocalFDB::control(const FDBToolRequest& request,
125125
return queryInternal<ControlVisitor>(request, action, identifiers);
126126
}
127127

128-
AxesIterator LocalFDB::axes(const FDBToolRequest& request, int level) {
129-
LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::axes() : " << request << std::endl;
128+
AxesIterator LocalFDB::axesIterator(const FDBToolRequest& request, int level) {
129+
LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::axesIterator() : " << request << std::endl;
130130
return queryInternal<AxesVisitor>(request, config_, level);
131131
}
132132

‎src/fdb5/api/LocalFDB.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ class LocalFDB : public FDBBase {
5959

6060
MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) override;
6161

62-
AxesIterator axes(const FDBToolRequest& request, int axes) override;
62+
AxesIterator axesIterator(const FDBToolRequest& request, int axes) override;
6363

6464
void flush() override;
6565

‎src/fdb5/api/RemoteFDB.cc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -968,7 +968,7 @@ class FDBRemoteDataHandle : public DataHandle {
968968
overallPosition_(0),
969969
currentBuffer_(0),
970970
complete_(false) {}
971-
virtual bool canSeek() const override { return false; }
971+
bool canSeek() const override { return false; }
972972

973973
private: // methods
974974

‎src/fdb5/api/RemoteFDB.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -126,9 +126,9 @@ class RemoteFDB : public FDBBase {
126126
void sendArchiveData(uint32_t id, const Key& key, const void* data, size_t length);
127127
long sendArchiveData(uint32_t id, const std::vector<std::pair<Key, eckit::Buffer>>& elements, size_t count);
128128

129-
virtual void print(std::ostream& s) const override;
129+
void print(std::ostream& s) const override;
130130

131-
virtual FDBStats stats() const override;
131+
FDBStats stats() const override;
132132

133133
private: // members
134134

‎src/fdb5/api/SelectFDB.cc

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,14 @@ ControlIterator SelectFDB::control(const FDBToolRequest& request,
196196
});
197197
}
198198

199+
AxesIterator SelectFDB::axesIterator(const FDBToolRequest& request, int level) {
200+
LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::axesIterator() >> " << request << std::endl;
201+
return queryInternal(request,
202+
[level](FDB& fdb, const FDBToolRequest& request) {
203+
return fdb.axesIterator(request, level);
204+
});
205+
}
206+
199207
void SelectFDB::flush() {
200208
for (auto& iter : subFdbs_) {
201209
FDB& fdb(iter.second);
@@ -208,7 +216,7 @@ void SelectFDB::print(std::ostream &s) const {
208216
s << "SelectFDB()";
209217
}
210218

211-
bool SelectFDB::matches(const Key &key, const SelectMap &select, bool requireMissing) const {
219+
bool SelectFDB::matches(const Key& key, const SelectMap &select, bool requireMissing) const {
212220

213221
for (const auto& kv : select) {
214222

‎src/fdb5/api/SelectFDB.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@ class SelectFDB : public FDBBase {
6868

6969
MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) override { NOTIMP; }
7070

71+
AxesIterator axesIterator(const FDBToolRequest& request, int level) override;
72+
7173
void flush() override;
7274

7375
private: // methods

‎src/fdb5/api/fdb_c.cc

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
#include "eckit/io/FileDescHandle.h"
1313
#include "eckit/message/Message.h"
1414
#include "eckit/runtime/Main.h"
15+
#include "eckit/config/YAMLConfiguration.h"
1516

1617
#include "metkit/mars/MarsRequest.h"
1718
#include "metkit/mars/MarsExpension.h"
@@ -331,6 +332,16 @@ int fdb_new_handle(fdb_handle_t** fdb) {
331332
});
332333
}
333334

335+
int fdb_new_handle_from_yaml(fdb_handle_t** fdb, const char* system_config, const char* user_config) {
336+
return wrapApiFunction([fdb, system_config, user_config] {
337+
Config cfg{YAMLConfiguration(std::string(system_config)), YAMLConfiguration(std::string(user_config))};
338+
cfg.set("configSource", "yaml");
339+
cfg.expandConfig();
340+
*fdb = new fdb_handle_t(cfg);
341+
});
342+
343+
}
344+
334345
int fdb_archive(fdb_handle_t* fdb, fdb_key_t* key, const char* data, size_t length) {
335346
return wrapApiFunction([fdb, key, data, length] {
336347
ASSERT(fdb);

‎src/fdb5/api/fdb_c.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,14 @@ typedef struct fdb_handle_t fdb_handle_t;
310310
*/
311311
int fdb_new_handle(fdb_handle_t** fdb);
312312

313+
/** Creates a FDB instance from a YAML configuration.
314+
* \param fdb FDB instance. Returned instance must be deleted using #fdb_delete_handle.
315+
* \param system_config Override the system config with this YAML string
316+
* \param user_config Supply user level config with this YAML string
317+
* \returns Return code (#FdbErrorValues)
318+
*/
319+
int fdb_new_handle_from_yaml(fdb_handle_t** fdb, const char* system_config, const char* user_config);
320+
313321
/** Archives binary data to a FDB instance.
314322
* \warning this is a low-level API. The provided key and the corresponding data are not checked for consistency
315323
* \param fdb FDB instance.

‎src/fdb5/api/helpers/APIIterator.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -83,9 +83,9 @@ class APIAggregateIterator : public APIIteratorBase<ValueType> {
8383
APIAggregateIterator(std::queue<APIIterator<ValueType>>&& iterators) :
8484
iterators_(std::move(iterators)) {}
8585

86-
virtual ~APIAggregateIterator() override {}
86+
~APIAggregateIterator() override {}
8787

88-
virtual bool next(ValueType& elem) override {
88+
bool next(ValueType& elem) override {
8989

9090
while (!iterators_.empty()) {
9191
if (iterators_.front().next(elem)) {
@@ -136,15 +136,15 @@ class APIAsyncIterator : public APIIteratorBase<ValueType> {
136136
workerThread_ = std::thread(fullWorker);
137137
}
138138

139-
virtual ~APIAsyncIterator() override {
139+
~APIAsyncIterator() override {
140140
if (!queue_.closed()) {
141141
queue_.interrupt(std::make_exception_ptr(eckit::SeriousBug("Destructing incomplete async queue", Here())));
142142
}
143143
ASSERT(workerThread_.joinable());
144144
workerThread_.join();
145145
}
146146

147-
virtual bool next(ValueType& elem) override {
147+
bool next(ValueType& elem) override {
148148
return !(queue_.pop(elem) == -1);
149149
}
150150

‎src/fdb5/api/local/AxesVisitor.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ class AxesVisitor : public QueryVisitor<AxesElement> {
4040
// bool preVisitDatabase(const eckit::URI& uri) override;
4141
bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
4242
bool visitIndex(const Index&) override;
43-
void visitDatum(const Field&, const Key&) override { NOTIMP; }
43+
void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
4444

4545
private: // members
4646

‎src/fdb5/api/local/ControlVisitor.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ class ControlVisitor : public QueryVisitor<ControlElement> {
3939

4040
bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
4141
bool visitIndex(const Index&) override { NOTIMP; }
42-
void visitDatum(const Field&, const Key&) override { NOTIMP; }
42+
void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
4343

4444
private: // members
4545

‎src/fdb5/api/local/DumpVisitor.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ class DumpVisitor : public QueryVisitor<DumpElement> {
4949
return true;
5050
}
5151
bool visitIndex(const Index&) override { NOTIMP; }
52-
void visitDatum(const Field&, const Key&) override { NOTIMP; }
52+
void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
5353

5454
void visitDatum(const Field& field, const std::string& keyFingerprint) override {
5555
EntryVisitor::visitDatum(field, keyFingerprint);

‎src/fdb5/api/local/ListVisitor.h

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
#include "fdb5/database/DB.h"
2323
#include "fdb5/database/Index.h"
24+
#include "fdb5/database/Key.h"
2425
#include "fdb5/api/local/QueryVisitor.h"
2526
#include "fdb5/api/helpers/ListIterator.h"
2627

@@ -79,12 +80,12 @@ struct ListVisitor : public QueryVisitor<ListElement> {
7980
}
8081

8182
/// Test if entry matches the current request. If so, add to the output queue.
82-
void visitDatum(const Field& field, const Key& key) override {
83+
void visitDatum(const Field& field, const TypedKey& datumKey) override {
8384
ASSERT(currentCatalogue_);
8485
ASSERT(currentIndex_);
8586

86-
if (key.match(datumRequest_)) {
87-
queue_.emplace(ListElement({currentCatalogue_->key(), currentIndex_->key(), key}, field.stableLocation(), field.timestamp()));
87+
if (datumKey.match(datumRequest_)) {
88+
queue_.emplace(ListElement({currentCatalogue_->key(), currentIndex_->key(), datumKey.canonical()}, field.stableLocation(), field.timestamp()));
8889
}
8990
}
9091

‎src/fdb5/api/local/MoveVisitor.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ class MoveVisitor : public QueryVisitor<MoveElement> {
4444

4545
bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
4646
bool visitIndex(const Index&) override { NOTIMP; }
47-
void visitDatum(const Field&, const Key&) override { NOTIMP; }
47+
void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
4848
void visitDatum(const Field& field, const std::string& keyFingerprint) override { NOTIMP; }
4949

5050
private: // members

‎src/fdb5/api/local/PurgeVisitor.cc

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,8 +82,6 @@ void PurgeVisitor::visitDatum(const Field& field, const std::string& keyFingerpr
8282
internalVisitor_->visitDatum(field, keyFingerprint);
8383
}
8484

85-
void PurgeVisitor::visitDatum(const Field&, const Key&) { NOTIMP; }
86-
8785
void PurgeVisitor::catalogueComplete(const Catalogue& catalogue) {
8886
internalVisitor_->catalogueComplete(catalogue);
8987

‎src/fdb5/api/local/PurgeVisitor.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ class PurgeVisitor : public QueryVisitor<PurgeElement> {
4848
bool visitIndex(const Index& index) override;
4949
void catalogueComplete(const Catalogue& catalogue) override;
5050
void visitDatum(const Field& field, const std::string& keyFingerprint) override;
51-
void visitDatum(const Field&, const Key&) override;
51+
void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
5252

5353
private: // members
5454

‎src/fdb5/api/local/StatsVisitor.cc

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,6 @@ void StatsVisitor::visitDatum(const Field& field, const std::string& keyFingerpr
4747
internalVisitor_->visitDatum(field, keyFingerprint);
4848
}
4949

50-
void StatsVisitor::visitDatum(const Field&, const Key&) { NOTIMP; }
51-
5250
void StatsVisitor::catalogueComplete(const Catalogue& catalogue) {
5351
internalVisitor_->catalogueComplete(catalogue);
5452

‎src/fdb5/api/local/StatsVisitor.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ class StatsVisitor : public QueryVisitor<StatsElement> {
4242
bool visitIndex(const Index& index) override;
4343
void catalogueComplete(const Catalogue& catalogue) override;
4444
void visitDatum(const Field& field, const std::string& keyFingerprint) override;
45-
void visitDatum(const Field&, const Key&) override;
45+
void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
4646

4747
private: // members
4848

‎src/fdb5/api/local/StatusVisitor.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ class StatusVisitor : public QueryVisitor<StatusElement> {
3838
bool visitEntries() override { return false; }
3939
bool visitDatabase(const Catalogue& catalogue, const Store& store) override { queue_.emplace(catalogue); return true; }
4040
bool visitIndex(const Index&) override { NOTIMP; }
41-
void visitDatum(const Field&, const Key&) override { NOTIMP; }
41+
void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
4242

4343
void visitDatum(const Field& field, const std::string& keyFingerprint) override {
4444
EntryVisitor::visitDatum(field, keyFingerprint);

0 commit comments

Comments
 (0)