Skip to content

Commit 3eb793b

Browse files
ChrisspyBChrisspyB
committed
Initial ArchiveCallback work
1 parent 29420df commit 3eb793b

15 files changed

+170
-18
lines changed

‎src/fdb5/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ list( APPEND fdb5_srcs
4848
api/helpers/PurgeIterator.h
4949
api/helpers/StatsIterator.cc
5050
api/helpers/StatsIterator.h
51+
api/helpers/ArchiveCallback.h
5152

5253
api/local/QueryVisitor.h
5354
api/local/QueueStringLogTarget.h

‎src/fdb5/api/FDB.cc

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,9 @@ FDB::~FDB() {
4949
}
5050
}
5151

52-
void FDB::archive(eckit::message::Message msg) {
52+
Key FDB::archive(eckit::message::Message msg, ArchiveCallback callback) {
5353
fdb5::Key key = MessageDecoder::messageToKey(msg);
54-
archive(key, msg.data(), msg.length());
54+
return archive(key, msg.data(), msg.length(), callback);
5555
}
5656
void FDB::archive(eckit::DataHandle& handle) {
5757
eckit::message::Message msg;
@@ -97,7 +97,7 @@ void FDB::archive(const metkit::mars::MarsRequest& request, eckit::DataHandle& h
9797
}
9898
}
9999

100-
void FDB::archive(const Key& key, const void* data, size_t length) {
100+
Key FDB::archive(const Key& key, const void* data, size_t length, ArchiveCallback callback) {
101101
eckit::Timer timer;
102102
timer.start();
103103

@@ -118,11 +118,12 @@ void FDB::archive(const Key& key, const void* data, size_t length) {
118118
keyInternal.unset("stepunits");
119119
}
120120

121-
internal_->archive(keyInternal, data, length);
121+
internal_->archive(keyInternal, data, length, callback);
122122
dirty_ = true;
123123

124124
timer.stop();
125125
stats_.addArchive(length, timer);
126+
return keyInternal; /* TODO: Not convinced this key is useful to return.*/
126127
}
127128

128129
bool FDB::sorted(const metkit::mars::MarsRequest &request) {

‎src/fdb5/api/FDB.h

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
#include "fdb5/api/helpers/WipeIterator.h"
3535
#include "fdb5/api/helpers/MoveIterator.h"
3636
#include "fdb5/config/Config.h"
37+
#include "fdb5/api/helpers/ArchiveCallback.h"
3738

3839
namespace eckit {
3940
namespace message {
@@ -69,13 +70,15 @@ class FDB {
6970

7071
// -------------- Primary API functions ----------------------------
7172

72-
void archive(eckit::message::Message msg);
73+
Key archive(eckit::message::Message msg, ArchiveCallback callback = nullptr);
7374
void archive(eckit::DataHandle& handle);
7475
void archive(const void* data, size_t length);
7576
// warning: not high-perf API - makes sure that all the requested fields are archived and there are no data exceeding the request
7677
void archive(const metkit::mars::MarsRequest& request, eckit::DataHandle& handle);
78+
7779
// disclaimer: this is a low-level API. The provided key and the corresponding data are not checked for consistency
78-
void archive(const Key& key, const void* data, size_t length);
80+
// Optional callback function is called upon receiving field location from the store.
81+
Key archive(const Key& key, const void* data, size_t length, ArchiveCallback callback = nullptr);
7982

8083
/// Flushes all buffers and closes all data handles into a consistent DB state
8184
/// @note always safe to call

‎src/fdb5/api/FDBFactory.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
#include "fdb5/api/helpers/PurgeIterator.h"
3838
#include "fdb5/api/helpers/StatsIterator.h"
3939
#include "fdb5/api/helpers/StatusIterator.h"
40+
#include "fdb5/api/helpers/ArchiveCallback.h"
4041

4142
namespace eckit {
4243
namespace message {
@@ -65,6 +66,9 @@ class FDBBase : private eckit::NonCopyable {
6566
// -------------- Primary API functions ----------------------------
6667

6768
virtual void archive(const Key& key, const void* data, size_t length) = 0;
69+
70+
/* TODO Using a seperate ArchiveCallback method for developing/experimentation convenience */
71+
virtual void archive(const Key& key, const void* data, size_t length, ArchiveCallback callback) {NOTIMP;};
6872

6973
virtual void flush() = 0;
7074

‎src/fdb5/api/LocalFDB.cc

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,14 +46,19 @@ using namespace eckit;
4646

4747

4848
namespace fdb5 {
49+
4950
void LocalFDB::archive(const Key& key, const void* data, size_t length) {
51+
archive(key, data, length, nullptr);
52+
}
53+
54+
void LocalFDB::archive(const Key& key, const void* data, size_t length, ArchiveCallback callback) {
5055

5156
if (!archiver_) {
5257
LOG_DEBUG_LIB(LibFdb5) << *this << ": Constructing new archiver" << std::endl;
5358
archiver_.reset(new Archiver(config_));
5459
}
5560

56-
archiver_->archive(key, data, length);
61+
archiver_->archive(key, data, length, callback);
5762
}
5863

5964
ListIterator LocalFDB::inspect(const metkit::mars::MarsRequest &request) {

‎src/fdb5/api/LocalFDB.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ class LocalFDB : public FDBBase {
3838
using FDBBase::stats;
3939

4040
void archive(const Key& key, const void* data, size_t length) override;
41+
void archive(const Key& key, const void* data, size_t length, ArchiveCallback callback) override;
4142

4243
ListIterator inspect(const metkit::mars::MarsRequest& request) override;
4344

‎src/fdb5/api/helpers/ArchiveCallback.h

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/*
2+
* (C) Copyright 1996- ECMWF.
3+
*
4+
* This software is licensed under the terms of the Apache Licence Version 2.0
5+
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6+
* In applying this licence, ECMWF does not waive the privileges and immunities
7+
* granted to it by virtue of its status as an intergovernmental organisation nor
8+
* does it submit to any jurisdiction.
9+
*/
10+
11+
/*
12+
* This software was developed as part of the EC H2020 funded project NextGenIO
13+
* (Project ID: 671951) www.nextgenio.eu
14+
*/
15+
16+
#pragma once
17+
18+
#include "fdb5/database/Key.h"
19+
#include "fdb5/database/FieldLocation.h"
20+
21+
namespace fdb5 {
22+
23+
using ArchiveCallback = std::function<void(const Key&, const FieldLocation&)>;
24+
25+
} // namespace fdb5

‎src/fdb5/database/ArchiveVisitor.cc

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,11 @@
1616

1717
namespace fdb5 {
1818

19-
ArchiveVisitor::ArchiveVisitor(Archiver &owner, const Key &field, const void *data, size_t size) :
19+
ArchiveVisitor::ArchiveVisitor(Archiver &owner, const Key &field, const void *data, size_t size, ArchiveCallback callback) :
2020
BaseArchiveVisitor(owner, field),
2121
data_(data),
22-
size_(size) {
22+
size_(size),
23+
callback_(callback){
2324
}
2425

2526
bool ArchiveVisitor::selectDatum(const Key &key, const Key &full) {
@@ -29,8 +30,7 @@ bool ArchiveVisitor::selectDatum(const Key &key, const Key &full) {
2930

3031
ASSERT(current());
3132

32-
current()->archive(key, data_, size_);
33-
33+
current()->archive(key, data_, size_, full, callback_);
3434

3535
return true;
3636
}

‎src/fdb5/database/ArchiveVisitor.h

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ class ArchiveVisitor : public BaseArchiveVisitor {
3030

3131
public: // methods
3232

33-
ArchiveVisitor(Archiver &owner, const Key &field, const void *data, size_t size);
33+
ArchiveVisitor(Archiver &owner, const Key &field, const void *data, size_t size, ArchiveCallback callback = nullptr);
3434

3535
protected: // methods
3636

@@ -43,6 +43,8 @@ class ArchiveVisitor : public BaseArchiveVisitor {
4343
const void *data_;
4444
size_t size_;
4545

46+
ArchiveCallback callback_;
47+
4648
};
4749

4850
//----------------------------------------------------------------------------------------------------------------------

‎src/fdb5/database/Archiver.cc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,8 @@ Archiver::~Archiver() {
3636
databases_.clear(); //< explicitly delete the DBs before schemas are destroyed
3737
}
3838

39-
void Archiver::archive(const Key &key, const void* data, size_t len) {
40-
ArchiveVisitor visitor(*this, key, data, len);
39+
void Archiver::archive(const Key &key, const void* data, size_t len, ArchiveCallback callback) {
40+
ArchiveVisitor visitor(*this, key, data, len, callback);
4141
archive(key, visitor);
4242
}
4343

‎src/fdb5/database/Archiver.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ class Archiver : public eckit::NonCopyable {
4747
virtual ~Archiver();
4848

4949
void archive(const Key &key, BaseArchiveVisitor& visitor);
50-
void archive(const Key &key, const void* data, size_t len);
50+
void archive(const Key &key, const void* data, size_t len, ArchiveCallback callback = nullptr);
5151

5252
/// Flushes all buffers and closes all data handles into a consistent DB state
5353
/// @note always safe to call

‎src/fdb5/database/DB.cc

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
#include "fdb5/database/DB.h"
1515
#include "fdb5/database/Field.h"
1616
#include "fdb5/toc/TocEngine.h"
17+
#include "fdb5/api/helpers/ArchiveCallback.h"
1718

1819
using eckit::Log;
1920

@@ -103,13 +104,19 @@ eckit::DataHandle *DB::retrieve(const Key& key) {
103104
return nullptr;
104105
}
105106

106-
void DB::archive(const Key& key, const void* data, eckit::Length length) {
107+
void DB::archive(const Key& key, const void* data, eckit::Length length, const Key& fullKey, ArchiveCallback callback) {
107108

108109
CatalogueWriter* cat = dynamic_cast<CatalogueWriter*>(catalogue_.get());
109110
ASSERT(cat);
110111

111112
const Index& idx = cat->currentIndex();
112-
cat->archive(key, store().archive(idx.key(), data, length));
113+
std::unique_ptr<FieldLocation> location(store().archive(idx.key(), data, length));
114+
115+
if (callback) {
116+
callback(fullKey, *location);
117+
}
118+
119+
cat->archive(key, std::move(location));
113120
}
114121

115122
bool DB::open() {

‎src/fdb5/database/DB.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include "fdb5/database/EntryVisitMechanism.h"
2424
#include "fdb5/database/Key.h"
2525
#include "fdb5/database/Store.h"
26+
#include "fdb5/api/helpers/ArchiveCallback.h"
2627

2728
namespace eckit {
2829
class DataHandle;
@@ -56,7 +57,7 @@ class DB final : public eckit::OwnedLock {
5657
bool axis(const std::string &keyword, eckit::StringSet &s) const;
5758
bool inspect(const Key& key, Field& field);
5859
eckit::DataHandle *retrieve(const Key &key);
59-
void archive(const Key &key, const void *data, eckit::Length length);
60+
void archive(const Key &key, const void *data, eckit::Length length, const Key &full, ArchiveCallback callback = nullptr);
6061

6162
bool open();
6263
void flush();

‎tests/fdb/api/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ list( APPEND api_tests
2727
select
2828
dist
2929
fdb_c
30+
archive_callback
3031
)
3132

3233
foreach( _test ${api_tests} )

‎tests/fdb/api/test_archive_callback.cc

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
2+
3+
#include "eckit/testing/Test.h"
4+
#include "fdb5/api/FDB.h"
5+
6+
namespace {
7+
bool testEqual(const fdb5::Key& key1, const fdb5::Key& key2) {
8+
if (key1.size() != key2.size()) {
9+
std::cout << "key1.size() != key2.size()" << std::endl;
10+
return false;
11+
}
12+
13+
// Then check that all items in key1 are in key2
14+
for (const auto& item : key1) {
15+
if (key2.find(item.first) == key2.end()) {
16+
return false;
17+
}
18+
19+
if (key2.value(item.first) != item.second) {
20+
return false;
21+
}
22+
}
23+
24+
return true;
25+
}
26+
27+
} // namespace anonymous
28+
29+
30+
namespace fdb5::test {
31+
32+
//----------------------------------------------------------------------------------------------------------------------
33+
CASE("Archive callback 2") {
34+
FDB fdb;
35+
36+
std::string data_str = "Raining cats and dogs";
37+
const void* data = static_cast<const void *>(data_str.c_str());
38+
size_t length = data_str.size();
39+
40+
Key key;
41+
key.set("class","od");
42+
key.set("expver","xxxx");
43+
key.set("type","fc");
44+
key.set("stream","oper");
45+
key.set("date","20101010");
46+
key.set("time","0000");
47+
key.set("domain","g");
48+
key.set("levtype","sfc");
49+
key.set("param","130");
50+
51+
std::map<fdb5::Key, eckit::URI> map;
52+
std::vector<Key> internalKeys;
53+
54+
key.set("step","1");
55+
internalKeys.push_back(fdb.archive(key, data, length, [&map, key] (const fdb5::Key& fullKey, const fdb5::FieldLocation& location) {
56+
map[key] = location.fullUri();
57+
}));
58+
59+
key.set("step","2");
60+
internalKeys.push_back(fdb.archive(key, data, length, [&map, key] (const fdb5::Key& fullKey, const fdb5::FieldLocation& location) {
61+
map[key] = location.fullUri();
62+
}));
63+
64+
key.set("step","3");
65+
internalKeys.push_back(fdb.archive(key, data, length, [&map, key] (const fdb5::Key& fullKey, const fdb5::FieldLocation& location) {
66+
map[key] = location.fullUri();
67+
}));
68+
69+
fdb.flush();
70+
71+
EXPECT(map.size() == 3);
72+
73+
// Print out the map
74+
for (const auto& [key, uri] : map) {
75+
std::cout << key << " -> " << uri << std::endl;
76+
}
77+
78+
// Note that the keys are not the same, but they are equivalent
79+
// so iterate over the map keys and check that there is a corresponding key in the internalKeys vector
80+
for (const auto& [key, uri] : map) {
81+
bool found = false;
82+
for (const auto& internalKey : internalKeys) {
83+
if (testEqual(key, internalKey)) {
84+
found = true;
85+
break;
86+
}
87+
}
88+
EXPECT(found);
89+
}
90+
91+
}
92+
//----------------------------------------------------------------------------------------------------------------------
93+
94+
} // namespace fdb5::test
95+
96+
int main(int argc, char** argv) {
97+
98+
eckit::Log::info() << ::getenv("FDB_HOME") << std::endl;
99+
100+
return ::eckit::testing::run_tests(argc, argv);
101+
}

0 commit comments

Comments
 (0)