Skip to content

Commit f25b0e4

Browse files
eboassoneboasson
committed
Multi-hop latency measurement example
This adds a simple tool for doing latency measurements across multiple hops. It assumes the clocks are synchronised to a high degree so that one-way latencies can be computed directly. It can operate with a number of different types, all very simple: struct Hop8 { uint32 seq; octet z[8 - 4]; }; and variants where the total size is 128, 1k, 8k and 128k bytes. Each process takes a stage, with the source publishing in partition Pj, the sink subscribing in partition Pj and the forwarders subscribing in Pj and publishing in Pk, where j is the stage argument and k = j+1. Each process additionally subscribes to "junk data" and optionally publishes samples at randomised intervals with a configurable average rate. Signed-off-by: Erik Boasson <eb@ilities.com>
1 parent 15949d4 commit f25b0e4

File tree

4 files changed

+418
-0
lines changed

4 files changed

+418
-0
lines changed

‎examples/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,3 +49,5 @@ install(
4949
COMPONENT dev)
5050

5151
add_subdirectory(roundtrip)
52+
53+
add_subdirectory(hop)

‎examples/hop/CMakeLists.txt‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#
2+
# Copyright(c) 2024 ZettaScale Technology and others
3+
#
4+
# This program and the accompanying materials are made available under the
5+
# terms of the Eclipse Public License v. 2.0 which is available at
6+
# http://www.eclipse.org/legal/epl-2.0, or the Eclipse Distribution License
7+
# v. 1.0 which is available at
8+
# http://www.eclipse.org/org/documents/edl-v10.php.
9+
#
10+
# SPDX-License-Identifier: EPL-2.0 OR BSD-3-Clause
11+
#
12+
project(helloworld LANGUAGES C CXX)
13+
cmake_minimum_required(VERSION 3.16)
14+
15+
set(CMAKE_CXX_STANDARD 17)
16+
17+
find_package(CycloneDDS REQUIRED)
18+
if(NOT TARGET CycloneDDS-CXX::ddscxx)
19+
find_package(CycloneDDS-CXX REQUIRED)
20+
endif()
21+
22+
idlcxx_generate(TARGET hop_type FILES hop_type.idl)
23+
add_executable(hop hop.cpp)
24+
target_link_libraries(hop CycloneDDS-CXX::ddscxx hop_type)

‎examples/hop/hop.cpp‎

Lines changed: 355 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,355 @@
1+
/*
2+
* Copyright(c) 2024 ZettaScale Technology and others
3+
*
4+
* This program and the accompanying materials are made available under the
5+
* terms of the Eclipse Public License v. 2.0 which is available at
6+
* http://www.eclipse.org/legal/epl-2.0, or the Eclipse Distribution License
7+
* v. 1.0 which is available at
8+
* http://www.eclipse.org/org/documents/edl-v10.php.
9+
*
10+
* SPDX-License-Identifier: EPL-2.0 OR BSD-3-Clause
11+
*/
12+
13+
#include <algorithm>
14+
#include <iostream>
15+
#include <fstream>
16+
#include <sstream>
17+
#include <iomanip>
18+
#include <chrono>
19+
#include <thread>
20+
#include <string>
21+
#include <cstdlib>
22+
#include <random>
23+
24+
#include <unistd.h>
25+
26+
#include "dds/dds.hpp"
27+
#include "hop_type.hpp"
28+
29+
using namespace org::eclipse::cyclonedds;
30+
using namespace std::chrono_literals;
31+
32+
static bool use_listener = true;
33+
static double junkrate = 0.0;
34+
35+
template<typename CLK>
36+
static dds::core::Time mkDDSTime (const std::chrono::time_point<CLK> x)
37+
{
38+
int64_t t = std::chrono::duration_cast<std::chrono::nanoseconds>(x.time_since_epoch()).count();
39+
return dds::core::Time(t / 1000000000, static_cast<uint32_t>(t % 1000000000));
40+
}
41+
42+
static volatile std::atomic<bool> interrupted = false;
43+
static void sigh(int sig)
44+
{
45+
static_cast<void>(sig);
46+
interrupted = true;
47+
}
48+
49+
static const dds::sub::status::DataState not_read()
50+
{
51+
return dds::sub::status::DataState(dds::sub::status::SampleState::not_read(),
52+
dds::sub::status::ViewState::any(),
53+
dds::sub::status::InstanceState::any());
54+
}
55+
56+
template<typename T>
57+
static dds::sub::DataReader<T> make_reader(dds::topic::Topic<T> tp, int stage)
58+
{
59+
dds::domain::DomainParticipant dp = tp.domain_participant();
60+
std::vector<std::string> spart{"P" + std::to_string(stage)};
61+
dds::sub::qos::SubscriberQos sqos = dp.default_subscriber_qos() << dds::core::policy::Partition(spart);
62+
dds::sub::Subscriber sub{dp, sqos};
63+
return dds::sub::DataReader<T>{sub, tp, tp.qos()};
64+
}
65+
66+
template<typename T>
67+
static dds::pub::DataWriter<T> make_writer(dds::topic::Topic<T> tp, int stage)
68+
{
69+
dds::domain::DomainParticipant dp = tp.domain_participant();
70+
std::vector<std::string> ppart{"P" + std::to_string(stage)};
71+
dds::pub::qos::PublisherQos pqos = dp.default_publisher_qos() << dds::core::policy::Partition(ppart);
72+
dds::pub::Publisher pub{dp, pqos};
73+
return dds::pub::DataWriter<T>{pub, tp, tp.qos()};
74+
}
75+
76+
// to be run on a separate thread
77+
template<typename T>
78+
static void junksource(dds::topic::Topic<T> tp)
79+
{
80+
std::random_device ran_dev;
81+
std::mt19937 prng(ran_dev());
82+
//std::uniform_int_distribution<> wrdist(0, tps.size() - 1);
83+
std::exponential_distribution<double> intvdist(junkrate);
84+
std::vector<dds::pub::DataWriter<T>> wrs;
85+
wrs.push_back(make_writer(tp, -1));
86+
T sample{};
87+
auto now = std::chrono::high_resolution_clock::now();
88+
while (!interrupted)
89+
{
90+
//wrs[wrdist(prng)] << sample;
91+
wrs[0] << sample;
92+
++sample.seq();
93+
auto delay = std::chrono::duration<double>(intvdist(prng));
94+
if (delay > 1s)
95+
delay = 1s;
96+
now += std::chrono::duration_cast<std::chrono::nanoseconds>(delay);
97+
std::this_thread::sleep_until(now);
98+
}
99+
std::cout << "wrote " << sample.seq() << " junk samples" << std::endl;
100+
}
101+
102+
template<typename T>
103+
static dds::sub::DataReader<T> make_junkreader(dds::topic::Topic<T> tp)
104+
{
105+
return make_reader(tp, -1);
106+
}
107+
108+
template<typename T>
109+
static void source(dds::topic::Topic<T> tp, int stage, const std::optional<std::string>)
110+
{
111+
auto wr = make_writer(tp, stage);
112+
signal(SIGINT, sigh);
113+
T sample{};
114+
auto now = std::chrono::high_resolution_clock::now();
115+
// give forwarders and sink time to start & discovery to run
116+
std::cout << "starting in 1s" << std::endl;
117+
now += 1s;
118+
std::this_thread::sleep_until(now);
119+
while (!interrupted)
120+
{
121+
wr.write(sample, mkDDSTime(now));
122+
++sample.seq();
123+
now += 10ms;
124+
std::this_thread::sleep_until(now);
125+
}
126+
std::cout << "wrote " << sample.seq() << " samples" << std::endl;
127+
}
128+
129+
template<typename T>
130+
static void run_reader(dds::sub::DataReaderListener<T> *list, dds::sub::DataReader<T> rd, std::function<void()> action)
131+
{
132+
if (use_listener)
133+
{
134+
rd.listener(list, dds::core::status::StatusMask::data_available());
135+
while (!interrupted)
136+
std::this_thread::sleep_for(103ms);
137+
}
138+
else
139+
{
140+
dds::core::cond::WaitSet ws;
141+
dds::sub::cond::ReadCondition rc{rd, not_read()};
142+
ws += rc;
143+
while (!interrupted)
144+
{
145+
ws.wait();
146+
action();
147+
}
148+
}
149+
}
150+
151+
template<typename T>
152+
class Forward : public dds::sub::NoOpDataReaderListener<T> {
153+
public:
154+
Forward() = delete;
155+
Forward(dds::sub::DataReader<T> rd, dds::pub::DataWriter<T> wr) : rd_{rd}, wr_{wr} { }
156+
157+
void run()
158+
{
159+
run_reader(this, rd_, [this](){action();});
160+
}
161+
162+
private:
163+
void action()
164+
{
165+
auto xs = rd_.take();
166+
for (const auto& x : xs) {
167+
if (x.info().valid()) {
168+
wr_.write (x.data(), x.info().timestamp());
169+
} else {
170+
interrupted = true;
171+
}
172+
};
173+
}
174+
175+
void on_data_available(dds::sub::DataReader<T>&)
176+
{
177+
action();
178+
}
179+
180+
dds::sub::DataReader<T> rd_;
181+
dds::pub::DataWriter<T> wr_;
182+
};
183+
184+
template<typename T>
185+
static void forward(dds::topic::Topic<T> tp, int stage, const std::optional<std::string>)
186+
{
187+
auto rd = make_reader(tp, stage);
188+
auto wr = make_writer(tp, stage + 1);
189+
Forward<T> x{rd, wr};
190+
x.run();
191+
}
192+
193+
template<typename T>
194+
class Sink : public dds::sub::NoOpDataReaderListener<T> {
195+
public:
196+
Sink() = delete;
197+
Sink(dds::sub::DataReader<T> rd, std::vector<double>& lats) : rd_{rd}, lats_{lats} { }
198+
199+
void run()
200+
{
201+
run_reader(this, rd_, [this](){action();});
202+
}
203+
204+
private:
205+
void action()
206+
{
207+
const auto now = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::high_resolution_clock::now().time_since_epoch()).count();
208+
auto xs = rd_.take();
209+
for (const auto& x : xs) {
210+
if (x.info().valid()) {
211+
const auto lat = now - (x.info().timestamp().sec() * 1000000000 + x.info().timestamp().nanosec());
212+
lats_.push_back(lat / 1e3);
213+
} else {
214+
interrupted = true;
215+
}
216+
};
217+
}
218+
219+
void on_data_available(dds::sub::DataReader<T>&)
220+
{
221+
action();
222+
}
223+
224+
dds::sub::DataReader<T> rd_;
225+
std::vector<double>& lats_;
226+
};
227+
228+
template<typename T>
229+
static void sink(dds::topic::Topic<T> tp, int stage, const std::optional<std::string> datafile)
230+
{
231+
// latencies in microseconds
232+
std::vector<double> lats;
233+
// read until source disappears
234+
// always create the "junk reader": it costs us nothing if no junk data is being published
235+
{
236+
auto rd = make_reader(tp, stage);
237+
Sink<T> x{rd, lats};
238+
x.run();
239+
}
240+
// destructors will have run, latencies are ours now
241+
if (datafile.has_value())
242+
{
243+
std::ofstream f;
244+
f.open(datafile.value());
245+
for (const auto l : lats)
246+
f << l << std::endl;
247+
f.close();
248+
}
249+
const size_t n = lats.size();
250+
if (n < 2) {
251+
std::cout << "insufficient data" << std::endl;
252+
} else {
253+
std::sort(lats.begin(), lats.end());
254+
std::cout << "received " << n << " samples; min " << lats[0] << " max-1 " << lats[n-2] << " max " << lats[n-1] << std::endl;
255+
}
256+
}
257+
258+
enum class Mode { Source, Forward, Sink };
259+
260+
template<typename T>
261+
static void run(const Mode mode, int stage, const std::optional<std::string> datafile)
262+
{
263+
dds::domain::DomainParticipant dp{0};
264+
auto tpqos = dp.default_topic_qos()
265+
<< dds::core::policy::Reliability::Reliable(dds::core::Duration::infinite())
266+
<< dds::core::policy::History::KeepLast(1);
267+
dds::topic::Topic<T> tp(dp, "Hop", tpqos);
268+
std::thread junkthr;
269+
if (junkrate > 0)
270+
junkthr = std::thread(junksource<T>, tp);
271+
auto junkrd = make_junkreader(tp);
272+
switch (mode)
273+
{
274+
case Mode::Source: source(tp, stage, datafile); break;
275+
case Mode::Forward: forward(tp, stage, datafile); break;
276+
case Mode::Sink: sink(tp, stage, datafile); break;
277+
}
278+
if (junkthr.joinable())
279+
junkthr.join();
280+
}
281+
282+
// type=128 n=1 bash -c 'bin/hop sink -ohop-result.$n.txt $n $type & i=0;while [[ i -lt n ]]; do bin/hop forward $i $type & i=$((i+1)) ; done ; bin/hop source 0 $type'
283+
// for n in {8..10} ; do n=$n type=128 bash -c 'bin/hop sink -ohop-result.$n.txt $n $type & i=0;while [[ i -lt n ]]; do bin/hop forward $i $type & i=$((i+1)) ; done ; bin/hop source 0 $type & p=$! ; sleep 10 ; kill -INT $p ; wait' ; done
284+
285+
[[noreturn]]
286+
static void usage()
287+
{
288+
std::cout
289+
<< "usage: hop {source|forward|sink} [OPTIONS] STAGE TYPE" << std::endl
290+
<< "OPTIONS:" << std::endl
291+
<< "-jRATE write junk at RATE Hz" << std::endl
292+
<< "-w: use waitset instead of listener (forward, sink)" << std::endl
293+
<< "-oFILE write latencies to FILE (sink)" << std::endl
294+
<< "TYPE: one of 8, 128, 1k, 8k, 128k" << std::endl;
295+
std::exit(1);
296+
}
297+
298+
int main (int argc, char **argv)
299+
{
300+
if (argc < 2)
301+
usage();
302+
const std::string modestr = std::string(argv[1]);
303+
Mode mode;
304+
if (modestr == "source") {
305+
mode = Mode::Source;
306+
} else if (modestr == "forward") {
307+
mode = Mode::Forward;
308+
} else if (modestr == "sink") {
309+
mode = Mode::Sink;
310+
} else {
311+
std::cout << "invalid mode, should be source, forward or sink" << std::endl;
312+
return 1;
313+
}
314+
315+
std::optional<std::string> datafile;
316+
optind = 2;
317+
int opt;
318+
while ((opt = getopt (argc, argv, "j:o:w")) != EOF)
319+
{
320+
switch (opt)
321+
{
322+
case 'j':
323+
junkrate = std::atof(optarg);
324+
break;
325+
case 'o':
326+
datafile = std::string(optarg);
327+
break;
328+
case 'w':
329+
use_listener = false;
330+
break;
331+
default:
332+
usage();
333+
}
334+
}
335+
336+
if (argc - optind != 2)
337+
usage();
338+
const int stage = std::atoi(argv[optind]);
339+
const std::string typestr = std::string(argv[optind + 1]);
340+
if (typestr == "8") {
341+
run<Hop8>(mode, stage, datafile);
342+
} else if (typestr == "128") {
343+
run<Hop128>(mode, stage, datafile);
344+
} else if (typestr == "1k") {
345+
run<Hop1k>(mode, stage, datafile);
346+
} else if (typestr == "8k") {
347+
run<Hop8k>(mode, stage, datafile);
348+
} else if (typestr == "128k") {
349+
run<Hop128k>(mode, stage, datafile);
350+
} else {
351+
std::cout << "invalid type, should be 8, 128, 1k, 8k, 128k" << std::endl;
352+
return 1;
353+
}
354+
return 0;
355+
}

0 commit comments

Comments
 (0)