Skip to content

Commit dc2cda7

Browse files
danovarodanovaro
authored
Merge pull request #68 from ecmwf/feature/remoteFDB-add-catalogue-exists
Adds functionality to the client-side exists() methods
2 parents 1336a19 + 5c18872 commit dc2cda7

18 files changed

+619
-349
lines changed

‎src/fdb5/remote/Connection.cc

Lines changed: 33 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,16 @@
33

44
#include "fdb5/LibFdb5.h"
55
#include "fdb5/remote/Connection.h"
6+
#include "fdb5/remote/Messages.h"
7+
#include <cstdint>
8+
#include <mutex>
9+
#include <string_view>
610

711
namespace fdb5::remote {
812

913
//----------------------------------------------------------------------------------------------------------------------
1014

11-
Connection::Connection() : single_(false) {}
12-
Connection::~Connection() {}
15+
Connection::Connection() : single_(false) { }
1316

1417
void Connection::teardown() {
1518

@@ -19,20 +22,20 @@ void Connection::teardown() {
1922
// all done - disconnecting
2023
Connection::write(Message::Exit, false, 0, 0);
2124
} catch(...) {
22-
// if connection is already down, no need to escalate
25+
// if connection is already down, no need to escalate
2326
}
2427
}
2528
try {
2629
// all done - disconnecting
2730
Connection::write(Message::Exit, true, 0, 0);
2831
} catch(...) {
29-
// if connection is already down, no need to escalate
32+
// if connection is already down, no need to escalate
3033
}
3134
}
3235

3336
//----------------------------------------------------------------------------------------------------------------------
3437

35-
void Connection::writeUnsafe(bool control, const void* data, size_t length) {
38+
void Connection::writeUnsafe(const bool control, const void* const data, const size_t length) const {
3639
long written = 0;
3740
if (control || single_) {
3841
written = controlSocket().write(data, length);
@@ -51,7 +54,7 @@ void Connection::writeUnsafe(bool control, const void* data, size_t length) {
5154
}
5255
}
5356

54-
void Connection::readUnsafe(bool control, void* data, size_t length) {
57+
void Connection::readUnsafe(bool control, void* data, size_t length) const {
5558
long read = 0;
5659
if (control || single_) {
5760
read = controlSocket().read(data, length);
@@ -70,14 +73,14 @@ void Connection::readUnsafe(bool control, void* data, size_t length) {
7073
}
7174
}
7275

73-
eckit::Buffer Connection::read(bool control, MessageHeader& hdr) {
76+
eckit::Buffer Connection::read(const bool control, MessageHeader& hdr) const {
7477
eckit::FixedString<4> tail;
7578

7679
std::lock_guard<std::mutex> lock((control || single_) ? readControlMutex_ : readDataMutex_);
7780
readUnsafe(control, &hdr, sizeof(hdr));
7881

79-
ASSERT(hdr.marker == StartMarker);
80-
ASSERT(hdr.version == CurrentVersion);
82+
ASSERT(hdr.marker == MessageHeader::StartMarker);
83+
ASSERT(hdr.version == MessageHeader::currentVersion);
8184
ASSERT(single_ || hdr.control() == control);
8285

8386
eckit::Buffer payload{hdr.payloadSize};
@@ -86,7 +89,7 @@ eckit::Buffer Connection::read(bool control, MessageHeader& hdr) {
8689
}
8790
// Ensure we have consumed exactly the correct amount from the socket.
8891
readUnsafe(control, &tail, sizeof(tail));
89-
ASSERT(tail == EndMarker);
92+
ASSERT(tail == MessageHeader::EndMarker);
9093

9194
if (hdr.message == Message::Error) {
9295

@@ -99,40 +102,43 @@ eckit::Buffer Connection::read(bool control, MessageHeader& hdr) {
99102
return payload;
100103
}
101104

102-
void Connection::write(remote::Message msg, bool control, uint32_t clientID, uint32_t requestID, const void* data, uint32_t length) {
103-
write(msg, control, clientID, requestID, std::vector<std::pair<const void*, uint32_t>>{{data, length}});
104-
}
105-
106-
void Connection::write(remote::Message msg, bool control, uint32_t clientID, uint32_t requestID, std::vector<std::pair<const void*, uint32_t>> data) {
105+
void Connection::write(const Message msg,
106+
const bool control,
107+
const uint32_t clientID,
108+
const uint32_t requestID,
109+
const PayloadList payloads) const {
107110

108111
uint32_t payloadLength = 0;
109-
for (auto d: data) {
110-
ASSERT(d.first);
111-
payloadLength += d.second;
112+
for (const auto& payload : payloads) {
113+
ASSERT(payload.data);
114+
payloadLength += payload.length;
112115
}
113116

114117
MessageHeader message{msg, control, clientID, requestID, payloadLength};
115118

116-
LOG_DEBUG_LIB(LibFdb5) << "Connection::write [message=" << msg << ",clientID=" << message.clientID() << ",control=" << control << ",requestID=" << requestID << ",data=" << data.size() << ",payload=" << payloadLength << "]" << std::endl;
119+
LOG_DEBUG_LIB(LibFdb5) << "Connection::write [message=" << msg << ",clientID=" << message.clientID()
120+
<< ",control=" << control << ",requestID=" << requestID << ",payloadsSize=" << payloads.size()
121+
<< ",payloadLength=" << payloadLength << "]" << std::endl;
117122

118123
std::lock_guard<std::mutex> lock((control || single_) ? controlMutex_ : dataMutex_);
124+
119125
writeUnsafe(control, &message, sizeof(message));
120-
for (auto d: data) {
121-
writeUnsafe(control, d.first, d.second);
122-
}
123-
writeUnsafe(control, &EndMarker, sizeof(EndMarker));
126+
127+
for (const auto& payload : payloads) { writeUnsafe(control, payload.data, payload.length); }
128+
129+
writeUnsafe(control, &MessageHeader::EndMarker, MessageHeader::markerBytes);
124130
}
125131

126-
void Connection::error(const std::string& msg, uint32_t clientID, uint32_t requestID) {
132+
void Connection::error(std::string_view msg, uint32_t clientID, uint32_t requestID) const {
127133
eckit::Log::error() << "[clientID=" << clientID << ",requestID=" << requestID << "] " << msg << std::endl;
128-
write(Message::Error, false, clientID, requestID, std::vector<std::pair<const void*, uint32_t>>{{msg.c_str(), msg.length()}});
134+
write(Message::Error, false, clientID, requestID, msg.data(), msg.length());
129135
}
130136

131-
eckit::Buffer Connection::readControl(MessageHeader& hdr) {
137+
eckit::Buffer Connection::readControl(MessageHeader& hdr) const {
132138
return read(true, hdr);
133139
}
134140

135-
eckit::Buffer Connection::readData(MessageHeader& hdr) {
141+
eckit::Buffer Connection::readData(MessageHeader& hdr) const {
136142
return read(false, hdr);
137143
}
138144

‎src/fdb5/remote/Connection.h

Lines changed: 35 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,19 @@
1010

1111
#pragma once
1212

13+
#include "eckit/serialisation/MemoryStream.h"
14+
#include "fdb5/remote/Messages.h"
15+
1316
#include "eckit/exception/Exceptions.h"
1417
#include "eckit/net/TCPSocket.h"
1518
#include "eckit/os/BackTrace.h"
1619

17-
#include "fdb5/remote/Messages.h"
20+
#include <cstddef>
21+
#include <cstdint>
22+
#include <iostream>
23+
#include <mutex>
24+
#include <string_view>
25+
#include <vector>
1826

1927
namespace eckit {
2028

@@ -40,41 +48,47 @@ class TCPException : public eckit::Exception {
4048

4149
class Connection : eckit::NonCopyable {
4250

51+
public: // types
52+
using PayloadList = std::vector<Payload>;
53+
4354
public: // methods
4455
Connection();
45-
virtual ~Connection();
4656

47-
void write(Message msg, bool control, uint32_t clientID, uint32_t requestID, const void* data, uint32_t length);
48-
void write(Message msg, bool control, uint32_t clientID, uint32_t requestID, std::vector<std::pair<const void*, uint32_t>> data = {});
57+
virtual ~Connection() = default;
58+
59+
void write(Message msg, bool control, uint32_t clientID, uint32_t requestID, PayloadList payloads = {}) const;
4960

50-
void error(const std::string& msg, uint32_t clientID, uint32_t requestID);
61+
void write(Message msg, bool control, uint32_t clientID, uint32_t requestID, const void* data, uint32_t length) const {
62+
write(msg, control, clientID, requestID, {{length, data}});
63+
}
5164

52-
eckit::Buffer readControl(MessageHeader& hdr);
53-
eckit::Buffer readData(MessageHeader& hdr);
65+
void error(std::string_view msg, uint32_t clientID, uint32_t requestID) const;
5466

55-
void teardown();
67+
eckit::Buffer readControl(MessageHeader& hdr) const;
5668

57-
private: // methods
69+
eckit::Buffer readData(MessageHeader& hdr) const;
5870

59-
eckit::Buffer read(bool control, MessageHeader& hdr);
71+
void teardown();
6072

61-
void writeUnsafe(bool control, const void* data, size_t length);
62-
void readUnsafe(bool control, void* data, size_t length);
73+
private: // methods
74+
eckit::Buffer read(bool control, MessageHeader& hdr) const;
6375

64-
virtual eckit::net::TCPSocket& controlSocket() = 0;
65-
virtual eckit::net::TCPSocket& dataSocket() = 0;
76+
void writeUnsafe(bool control, const void* data, size_t length) const;
6677

67-
protected: // members
78+
void readUnsafe(bool control, void* data, size_t length) const;
6879

69-
bool single_;
80+
virtual const eckit::net::TCPSocket& controlSocket() const = 0;
7081

71-
private: // members
82+
virtual const eckit::net::TCPSocket& dataSocket() const = 0;
7283

73-
std::mutex controlMutex_;
74-
std::mutex dataMutex_;
75-
std::mutex readControlMutex_;
76-
std::mutex readDataMutex_;
84+
protected: // members
85+
bool single_;
7786

87+
private: // members
88+
mutable std::mutex controlMutex_;
89+
mutable std::mutex dataMutex_;
90+
mutable std::mutex readControlMutex_;
91+
mutable std::mutex readDataMutex_;
7892
};
7993

8094
//----------------------------------------------------------------------------------------------------------------------

‎src/fdb5/remote/Messages.cc

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,6 @@
1515

1616
#include "fdb5/remote/Messages.h"
1717

18-
using namespace eckit;
19-
2018
namespace fdb5::remote {
2119

2220
//----------------------------------------------------------------------------------------------------------------------
@@ -47,6 +45,7 @@ std::ostream& operator<<(std::ostream& s, const Message& m) {
4745
case Message::Move: s << "Move"; break;
4846
case Message::Store: s << "Store"; break;
4947
case Message::Axes: s << "Axes"; break;
48+
case Message::Exists: s << "Exists"; break;
5049

5150
// Responses
5251
case Message::Received: s << "Received"; break;
@@ -62,7 +61,7 @@ std::ostream& operator<<(std::ostream& s, const Message& m) {
6261

6362
MessageHeader::MessageHeader(Message message, bool control, uint32_t clientID, uint32_t requestID, uint32_t payloadSize) :
6463
marker(StartMarker),
65-
version(CurrentVersion),
64+
version(currentVersion),
6665
message(message),
6766
clientID_((clientID<<1) + (control ? 1 : 0)),
6867
requestID(requestID),

‎src/fdb5/remote/Messages.h

Lines changed: 33 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,9 @@
1818

1919
#pragma once
2020

21-
#include <cstdint>
2221
#include <cmath>
22+
#include <cstddef>
23+
#include <cstdint>
2324

2425
#include "eckit/types/FixedString.h"
2526

@@ -31,10 +32,12 @@ namespace fdb5::remote {
3132

3233
//----------------------------------------------------------------------------------------------------------------------
3334

34-
const static eckit::FixedString<4> StartMarker {"SFDB"};
35-
const static eckit::FixedString<4> EndMarker {"EFDB"};
35+
struct Payload {
36+
Payload(std::size_t length, const void* data) : length {length}, data {data} { }
3637

37-
constexpr uint16_t CurrentVersion = 12;
38+
std::size_t length {0};
39+
const void* data {nullptr};
40+
};
3841

3942
enum class Message : uint16_t {
4043

@@ -63,6 +66,7 @@ enum class Message : uint16_t {
6366
Move,
6467
Store,
6568
Axes,
69+
Exists,
6670

6771
// Responses
6872
Received = 200,
@@ -76,21 +80,31 @@ enum class Message : uint16_t {
7680

7781
std::ostream& operator<<(std::ostream& s, const Message& m);
7882

83+
//----------------------------------------------------------------------------------------------------------------------
84+
7985
// Header used for all messages
8086
class MessageHeader {
8187

82-
public: // methods
88+
public: // types
89+
constexpr static uint16_t currentVersion = 12;
90+
91+
constexpr static const auto hashBytes = 16;
92+
93+
constexpr static const auto markerBytes = 4;
94+
95+
using MarkerType = eckit::FixedString<markerBytes>;
96+
97+
using HashType = eckit::FixedString<hashBytes>;
8398

84-
MessageHeader() :
85-
version(CurrentVersion),
86-
message(Message::None),
87-
clientID_(0),
88-
requestID(0),
89-
payloadSize(0) {}
99+
inline static const MarkerType StartMarker {"SFDB"};
90100

101+
inline static const MarkerType EndMarker {"EFDB"};
102+
103+
public: // methods
104+
MessageHeader() = default;
91105

92106
MessageHeader(Message message, bool control, uint32_t clientID, uint32_t requestID, uint32_t payloadSize);
93-
107+
94108
bool control() const {
95109
return ((clientID_ & 0x00000001) == 1);
96110
}
@@ -99,21 +113,13 @@ class MessageHeader {
99113
}
100114

101115
public:
102-
103-
eckit::FixedString<4> marker; // 4 bytes --> 4
104-
105-
uint16_t version; // 2 bytes --> 6
106-
107-
Message message; // 2 bytes --> 8
108-
109-
uint32_t clientID_; // 4 bytes --> 12
110-
111-
uint32_t requestID; // 4 bytes --> 16
112-
113-
uint32_t payloadSize; // 4 bytes --> 20
114-
115-
eckit::FixedString<16> hash; // 16 bytes --> 36
116-
116+
MarkerType marker; // 4 bytes --> 4
117+
uint16_t version {currentVersion}; // 2 bytes --> 6
118+
Message message {Message::None}; // 2 bytes --> 8
119+
uint32_t clientID_ {0}; // 4 bytes --> 12
120+
uint32_t requestID {0}; // 4 bytes --> 16
121+
uint32_t payloadSize {0}; // 4 bytes --> 20
122+
HashType hash; // 16 bytes --> 36
117123
};
118124

119125
//----------------------------------------------------------------------------------------------------------------------

0 commit comments

Comments
 (0)