Skip to content

Commit 7045679

Browse files
author
Neil Stephens
committed
Build in KafkaPort, added KafkaConsumerPort
1 parent 4aa47fe commit 7045679

File tree

8 files changed

+212
-73
lines changed

8 files changed

+212
-73
lines changed

‎Code/Ports/KafkaPort/KafkaClientCache.h

Lines changed: 48 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,35 @@
1+
/* opendatacon
2+
*
3+
* Copyright (c) 2014:
4+
*
5+
* DCrip3fJguWgVCLrZFfA7sIGgvx1Ou3fHfCxnrz4svAi
6+
* yxeOtDhDCXf1Z4ApgXvX5ahqQmzRfJ2DoX8S05SqHA==
7+
*
8+
* Licensed under the Apache License, Version 2.0 (the "License");
9+
* you may not use this file except in compliance with the License.
10+
* You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing, software
15+
* distributed under the License is distributed on an "AS IS" BASIS,
16+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
* See the License for the specific language governing permissions and
18+
* limitations under the License.
19+
*/
20+
/*
21+
* KafkaClientCache.h
22+
*
23+
* Created on: 19/07/2024
24+
* Author: Neil Stephens
25+
*/
26+
27+
#ifndef KAFKACLIENTCACHE_H
28+
#define KAFKACLIENTCACHE_H
29+
130
#include <kafka/KafkaClient.h>
231
#include <opendatacon/asio.h>
32+
#include <opendatacon/util.h>
333
#include <memory>
434
#include <unordered_map>
535
#include <string>
@@ -74,12 +104,21 @@ class KafkaClientCache
74104
return std::static_pointer_cast<ClientType>(client);
75105
}
76106

77-
auto new_client = std::make_shared<ClientType>(properties);
78-
clients[client_key] = new_client;
79-
poll_timers[client_key] = {std::numeric_limits<size_t>::max(),nullptr};
80-
MaxPollTime(client_key, MaxPollIntervalms);
81-
Clean();
82-
return new_client;
107+
try
108+
{
109+
auto new_client = std::make_shared<ClientType>(properties);
110+
clients[client_key] = new_client;
111+
poll_timers[client_key] = {std::numeric_limits<size_t>::max(),nullptr};
112+
MaxPollTime(client_key, MaxPollIntervalms);
113+
Clean();
114+
return new_client;
115+
}
116+
catch(const kafka::KafkaException& e)
117+
{
118+
if(auto log = odc::spdlog_get("KafkaPort"))
119+
log->error("{}: Failed to create KafkaClient: {}", client_key, e.what());
120+
return nullptr;
121+
}
83122
}
84123

85124
// Singleton pattern to manage the cache
@@ -115,4 +154,6 @@ class KafkaClientCache
115154
std::mutex mtx;
116155
std::unordered_map<std::string, std::weak_ptr<kafka::clients::KafkaClient>> clients;
117156
std::unordered_map<std::string, std::pair<size_t,std::shared_ptr<asio::steady_timer>>> poll_timers;
118-
};
157+
};
158+
159+
#endif // KAFKACLIENTCACHE_H

‎Code/Ports/KafkaPort/KafkaConsumerPort.cpp

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/* opendatacon
2+
*
3+
* Copyright (c) 2014:
4+
*
5+
* DCrip3fJguWgVCLrZFfA7sIGgvx1Ou3fHfCxnrz4svAi
6+
* yxeOtDhDCXf1Z4ApgXvX5ahqQmzRfJ2DoX8S05SqHA==
7+
*
8+
* Licensed under the Apache License, Version 2.0 (the "License");
9+
* you may not use this file except in compliance with the License.
10+
* You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing, software
15+
* distributed under the License is distributed on an "AS IS" BASIS,
16+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
* See the License for the specific language governing permissions and
18+
* limitations under the License.
19+
*/
20+
/*
21+
* KafkaConsumerPort.cpp
22+
*
23+
* Created on: 26/07/2024
24+
* Author: Neil Stephens
25+
*/
26+
27+
#include "KafkaConsumerPort.h"
28+
#include "KafkaPort.h"
29+
#include <opendatacon/asio.h>
30+
#include <opendatacon/spdlog.h>
31+
#include <kafka/KafkaConsumer.h>
32+
#include <chrono>
33+
#include <memory>
34+
35+
void KafkaConsumerPort::Build()
36+
{
37+
pKafkaConsumer = KafkaPort::Build<KCC::KafkaConsumer>("Consumer");
38+
//TODO: set up the consumer
39+
}
40+

‎Code/Ports/KafkaPort/KafkaConsumerPort.h

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
/* opendatacon
2+
*
3+
* Copyright (c) 2014:
4+
*
5+
* DCrip3fJguWgVCLrZFfA7sIGgvx1Ou3fHfCxnrz4svAi
6+
* yxeOtDhDCXf1Z4ApgXvX5ahqQmzRfJ2DoX8S05SqHA==
7+
*
8+
* Licensed under the Apache License, Version 2.0 (the "License");
9+
* you may not use this file except in compliance with the License.
10+
* You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing, software
15+
* distributed under the License is distributed on an "AS IS" BASIS,
16+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
* See the License for the specific language governing permissions and
18+
* limitations under the License.
19+
*/
20+
/*
21+
* KafkaConsumerPort.h
22+
*
23+
* Created on: 26/07/2024
24+
* Author: Neil Stephens
25+
*/
26+
27+
#ifndef KAFKACONSUMERPORT_H
28+
#define KAFKACONSUMERPORT_H
29+
30+
#include "KafkaPort.h"
31+
#include <kafka/KafkaConsumer.h>
32+
33+
using namespace odc;
34+
namespace KCC = kafka::clients::consumer;
35+
36+
class KafkaConsumerPort: public KafkaPort
37+
{
38+
public:
39+
KafkaConsumerPort(const std::string& Name, const std::string& Filename, const Json::Value& Overrides)
40+
:KafkaPort(Name,Filename,Overrides){};
41+
virtual ~KafkaConsumerPort(){};
42+
43+
virtual void Build() override;
44+
virtual void Event(std::shared_ptr<const EventInfo> event, const std::string& SenderName, SharedStatusCallback_t pStatusCallback) override {}
45+
private:
46+
std::shared_ptr<KCC::KafkaConsumer> pKafkaConsumer;
47+
};
48+
49+
#endif // KAFKACONSUMERPORT_H

‎Code/Ports/KafkaPort/KafkaPort.cpp

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
*/
2626

2727
#include "KafkaPort.h"
28-
#include "KafkaPortConf.h"
2928

3029
KafkaPort::KafkaPort(const std::string& Name, const std::string& Filename, const Json::Value& Overrides):
3130
DataPort(Name, Filename, Overrides)
@@ -60,7 +59,7 @@ void KafkaPort::ProcessElements(const Json::Value& JSONRoot)
6059
{
6160
pConf->NativeKafkaProperties.put(memberName, JSONRoot["NativeKafkaProperties"][memberName].asString());
6261
}
63-
else if(auto log = spdlog::get("KafkaPort"))
62+
else if(auto log = odc::spdlog_get("KafkaPort"))
6463
{
6564
log->error("NativeKafkaProperties member '{}' is not a simple value; ignoring.", memberName);
6665
}

‎Code/Ports/KafkaPort/KafkaPort.def

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,5 @@ LIBRARY KafkaPort
2121
EXPORTS
2222
new_KafkaProducerPort
2323
delete_KafkaProducerPort
24+
new_KafkaConsumerPort
25+
delete_KafkaConsumerPort

‎Code/Ports/KafkaPort/KafkaPort.h

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,9 @@
2828
#define KAFKAPORT_H
2929

3030
#include "KafkaClientCache.h"
31-
#include <atomic>
31+
#include "KafkaPortConf.h"
3232
#include <opendatacon/DataPort.h>
33+
#include <atomic>
3334

3435
using namespace odc;
3536

@@ -49,6 +50,55 @@ class KafkaPort: public DataPort
4950
protected:
5051
std::atomic_bool enabled {false};
5152
std::shared_ptr<KafkaClientCache> pKafkaClientCache = KafkaClientCache::Get();
53+
template <class KafkaClientType> std::shared_ptr<KafkaClientType> Build(std::string TypeString = "")
54+
{
55+
auto pConf = static_cast<KafkaPortConf*>(this->pConf.get());
56+
57+
if(!pConf->NativeKafkaProperties.contains("bootstrap.servers"))
58+
{
59+
pConf->NativeKafkaProperties.put("bootstrap.servers", "localhost:9092");
60+
if(auto log = odc::spdlog_get("KafkaPort"))
61+
log->error("{}: bootstrap.servers property not found, defaulting to localhost:9092", Name);
62+
}
63+
64+
if(pConf->NativeKafkaProperties.getProperty("enable.manual.events.poll") == "false")
65+
if(auto log = odc::spdlog_get("KafkaPort"))
66+
log->warn("{}: enable.manual.events.poll property is set to false, forcing to true", Name);
67+
pConf->NativeKafkaProperties.put("enable.manual.events.poll", "true");
68+
69+
pConf->NativeKafkaProperties.put("error_cb", [this](const kafka::Error& error)
70+
{
71+
if(auto log = odc::spdlog_get("KafkaPort"))
72+
log->error("{}: {}",Name,error.toString());
73+
});
74+
75+
pConf->NativeKafkaProperties.put("log_cb", [this](int level, const char* filename, int lineno, const char* msg)
76+
{
77+
auto spdlog_lvl = spdlog::level::level_enum(6-level);
78+
if(auto log = odc::spdlog_get("KafkaPort"))
79+
log->log(spdlog_lvl,"{} ({}:{}): {}",Name,filename,lineno,msg);
80+
});
81+
82+
pConf->NativeKafkaProperties.put("stats_cb", [this](const std::string& jsonString)
83+
{
84+
if(auto log = odc::spdlog_get("KafkaPort"))
85+
log->info("{}: Statistics: {}",Name,jsonString);
86+
});
87+
88+
if(pConf->ShareKafkaClient)
89+
{
90+
if(pConf->SharedKafkaClientKey == "")
91+
{
92+
auto bs_servers = pConf->NativeKafkaProperties.getProperty("bootstrap.servers").value();
93+
pConf->SharedKafkaClientKey = TypeString+":"+bs_servers;
94+
}
95+
96+
return pKafkaClientCache->GetClient<KafkaClientType>(
97+
pConf->SharedKafkaClientKey,
98+
pConf->NativeKafkaProperties);
99+
}
100+
return pKafkaClientCache->GetClient<KafkaClientType>(Name, pConf->NativeKafkaProperties);
101+
}
52102
};
53103

54104
#endif // KAFKAPORT_H

‎Code/Ports/KafkaPort/KafkaProducerPort.cpp

Lines changed: 9 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
*/
2626

2727
#include "KafkaProducerPort.h"
28-
#include "KafkaPortConf.h"
28+
#include "KafkaPort.h"
2929
#include <opendatacon/asio.h>
3030
#include <opendatacon/spdlog.h>
3131
#include <kafka/KafkaProducer.h>
@@ -34,85 +34,31 @@
3434

3535
void KafkaProducerPort::Build()
3636
{
37-
//create a kafka producer
38-
auto pConf = static_cast<KafkaPortConf*>(this->pConf.get());
39-
40-
if(!pConf->NativeKafkaProperties.contains("bootstrap.servers"))
41-
{
42-
pConf->NativeKafkaProperties.put("bootstrap.servers", "localhost:9092");
43-
if(auto log = spdlog::get("KafkaPort"))
44-
log->error("bootstrap.servers property not found, defaulting to localhost:9092");
45-
}
46-
47-
if(pConf->NativeKafkaProperties.getProperty("enable.manual.events.poll") == "false")
48-
if(auto log = spdlog::get("KafkaPort"))
49-
log->warn("enable.manual.events.poll property is set to false, forcing to true");
50-
pConf->NativeKafkaProperties.put("enable.manual.events.poll", "true");
51-
52-
pConf->NativeKafkaProperties.put("error_cb", [this](const kafka::Error& error)
53-
{
54-
if(auto log = spdlog::get("KafkaPort"))
55-
log->error("{}: {}",Name,error.toString());
56-
});
57-
58-
pConf->NativeKafkaProperties.put("log_cb", [this](int level, const char* filename, int lineno, const char* msg)
59-
{
60-
auto spdlog_lvl = spdlog::level::level_enum(6-level);
61-
if(auto log = spdlog::get("KafkaPort"))
62-
log->log(spdlog_lvl,"{} ({}:{}): {}",Name,filename,lineno,msg);
63-
});
64-
65-
pConf->NativeKafkaProperties.put("stats_cb", [this](const std::string& jsonString)
66-
{
67-
if(auto log = spdlog::get("KafkaPort"))
68-
log->info("{}: Statistics: {}",Name,jsonString);
69-
});
70-
71-
//TODO: consider also forcing enable.idempotence=true depending on the retry model
72-
// see https://github.com/confluentinc/librdkafka/blob/master/INTRODUCTION.md#idempotent-producer
73-
//TODO: consider also setting the acks property to "all" depending on the retry model
74-
75-
//FIXME: KafkaProducer ctor can throw - catch it, and at least log it
76-
if(pConf->ShareKafkaClient)
77-
{
78-
if(pConf->SharedKafkaClientKey == "")
79-
{
80-
auto bs_servers = pConf->NativeKafkaProperties.getProperty("bootstrap.servers").value();
81-
pConf->SharedKafkaClientKey.append("Producer:").append(bs_servers);
82-
}
83-
84-
pKafkaProducer = pKafkaClientCache->GetClient<KCP::KafkaProducer>(
85-
pConf->SharedKafkaClientKey,
86-
pConf->NativeKafkaProperties);
87-
}
88-
else
89-
{
90-
pKafkaProducer = pKafkaClientCache->GetClient<KCP::KafkaProducer>(
91-
"Producer: " + Name,
92-
pConf->NativeKafkaProperties);
93-
}
37+
pKafkaProducer = KafkaPort::Build<KCP::KafkaProducer>("Producer");
9438
}
9539

9640
void KafkaProducerPort::Event(std::shared_ptr<const EventInfo> event, const std::string& SenderName, SharedStatusCallback_t pStatusCallback)
9741
{
9842
if(!enabled) return;
43+
if(!pKafkaProducer) return;
9944

10045
//TODO: build the KCP::ProducerRecord from the EventInfo
10146
KCP::ProducerRecord record(kafka::Topic("example-topic"), kafka::NullKey, kafka::Value("test"));
10247

103-
auto deliveryCb = [](const KCP::RecordMetadata& metadata, const kafka::Error& error)
48+
auto deliveryCb = [this,event](const KCP::RecordMetadata& metadata, const kafka::Error& error)
10449
{
105-
auto log = spdlog::get("KafkaPort");
50+
auto log = odc::spdlog_get("KafkaPort");
10651
if (!error)
10752
{
10853
if(log && log->should_log(spdlog::level::trace))
109-
log->trace("Message delivered: {}", metadata.toString());
54+
log->trace("{}: Message delivered: {}", Name, metadata.toString());
11055
}
11156
else
11257
{
11358
if(log)
114-
log->error("Message failed to be delivered: {}", error.message());
115-
//TODO: retry?
59+
log->error("{}: Message failed to be delivered: {} {} {}: {}",
60+
ToString(event->GetEventType()),event->GetIndex(),
61+
event->GetPayloadString(), error.message());
11662
}
11763
};
11864

‎Code/Ports/KafkaPort/main.cpp

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
*/
2626

2727
#include "KafkaProducerPort.h"
28+
#include "KafkaConsumerPort.h"
2829

2930
extern "C" KafkaProducerPort* new_KafkaProducerPort(const std::string& Name, const std::string& File, const Json::Value& Overrides)
3031
{
@@ -36,3 +37,14 @@ extern "C" void delete_KafkaProducerPort(KafkaProducerPort* aKafkaProducerPort_p
3637
delete aKafkaProducerPort_ptr;
3738
return;
3839
}
40+
41+
extern "C" KafkaConsumerPort* new_KafkaConsumerPort(const std::string& Name, const std::string& File, const Json::Value& Overrides)
42+
{
43+
return new KafkaConsumerPort(Name,File,Overrides);
44+
}
45+
46+
extern "C" void delete_KafkaConsumerPort(KafkaConsumerPort* aKafkaConsumerPort_ptr)
47+
{
48+
delete aKafkaConsumerPort_ptr;
49+
return;
50+
}

0 commit comments

Comments
 (0)