From 79c05bba9012ebedc6b1139b0702b5a868f7e7c5 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Tue, 17 Sep 2024 16:25:48 -0400 Subject: [PATCH 01/73] Implemented setting Segmenter and Reassembler flags values from INI file and added this to e2sar_perf; provided sample INI files at the root level --- bin/e2sar_perf.cpp | 51 +++++++++++++++++++++++++++++----- include/e2sarDPReassembler.hpp | 5 ++++ include/e2sarDPSegmenter.hpp | 5 ++++ meson.build | 2 +- reassembler_config.ini | 39 ++++++++++++++++++++++++++ segmenter_config.ini | 26 +++++++++++++++++ src/e2sarDPReassembler.cpp | 40 ++++++++++++++++++++++++++ src/e2sarDPSegmenter.cpp | 39 ++++++++++++++++++++++++++ 8 files changed, 199 insertions(+), 8 deletions(-) create mode 100644 reassembler_config.ini create mode 100644 segmenter_config.ini diff --git a/bin/e2sar_perf.cpp b/bin/e2sar_perf.cpp index 54a8d5a2..a27a5629 100644 --- a/bin/e2sar_perf.cpp +++ b/bin/e2sar_perf.cpp @@ -134,6 +134,10 @@ result sendEvents(Segmenter &s, EventNum_t startEventNum, size_t numEvents, evtBufferPool->purge_memory(); std::cout << "Completed, " << stats.get<0>() << " frames sent, " << stats.get<1>() << " errors" << std::endl; + if (stats.get<1>() != 0) + { + std::cout << "Last error encountered: " << strerror(stats.get<2>()) << std::endl; + } return 0; } @@ -232,6 +236,8 @@ int main(int argc, char **argv) float rateGbps; int sockBufSize; int durationSec; + bool withCP; + std::string iniFile; // parameters opts("send,s", "send traffic"); @@ -248,7 +254,10 @@ int main(int argc, char **argv) opts("rate", po::value(&rateGbps)->default_value(1.0), "send rate in Gbps (defaults to 1.0)"); opts("period,p", po::value(&reportThreadSleepMs)->default_value(1000), "receive side reporting thread sleep period in ms (defaults to 1000) [r]"); opts("bufsize,b", po::value(&sockBufSize)->default_value(1024*1024*3), "send or receive socket buffer size (default to 3MB)"); - opts("duration,d", po::value(&durationSec)->default_value(0), "duration for receiver to run for (defaults to 0 - until Ctrl-C is presses)"); + opts("duration,d", po::value(&durationSec)->default_value(0), "duration for receiver to run for (defaults to 0 - until Ctrl-C is pressed)"); + opts("withcp,c", po::bool_switch()->default_value(false), "enable control plane interactions"); + opts("ini,i", po::value(&iniFile)->default_value(""), "INI file to initialize SegmenterFlags [s]] or ReassemblerFlags [r]." + " Values found in the file override --withcp, --mtu and --bufsize"); po::variables_map vm; @@ -284,6 +293,11 @@ int main(int argc, char **argv) return 0; } + withCP = vm["withcp"].as(); + std::cout << "Control plane will be " << (withCP ? "ON" : "OFF") << std::endl; + std::cout << (withCP ? "*** Make sure the LB has been reserved and the URI reflects the reserved instance information." : + "*** Make sure the URI reflects proper data address, other parts are ignored.") << std::endl; + // make sure the token is interpreted as the correct type, depending on the call EjfatURI::TokenType tt{EjfatURI::TokenType::instance}; @@ -300,9 +314,20 @@ int main(int argc, char **argv) auto uri = uri_r.value(); if (vm.count("send")) { Segmenter::SegmenterFlags sflags; - sflags.useCP = false; // turn off CP sync - sflags.mtu = mtu; - sflags.sndSocketBufSize = sockBufSize; + if (!vm["ini"].as().empty()) + { + auto sflagsRes = Segmenter::SegmenterFlags::getFromINI(vm["ini"].as()); + if (sflagsRes.has_error()) + { + std::cerr << "Unable to parse SegmenterFlags INI file " << vm["ini"].as() << std::endl; + return -1; + } + sflags = sflagsRes.value(); + } else { + sflags.useCP = withCP; + sflags.mtu = mtu; + sflags.sndSocketBufSize = sockBufSize; + } try { Segmenter seg(uri, dataId, eventSourceId, sflags); @@ -319,9 +344,21 @@ int main(int argc, char **argv) } else if (vm.count("recv")) { Reassembler::ReassemblerFlags rflags; - rflags.useCP = false; // turn off CP gRPC - rflags.withLBHeader = true; // no LB - rflags.rcvSocketBufSize = sockBufSize; + if (!vm["ini"].as().empty()) + { + auto rflagsRes = Reassembler::ReassemblerFlags::getFromINI(vm["ini"].as()); + if (rflagsRes.has_error()) + { + std::cerr << "Unable to parse ReassemblerFlags INI file " << vm["ini"].as() << std::endl; + return -1; + } + rflags = rflagsRes.value(); + } else + { + rflags.useCP = withCP; + rflags.withLBHeader = not withCP; + rflags.rcvSocketBufSize = sockBufSize; + } try { Reassembler reas(uri, numThreads, rflags); reasPtr = &reas; diff --git a/include/e2sarDPReassembler.hpp b/include/e2sarDPReassembler.hpp index 34af96ad..f66c64ba 100644 --- a/include/e2sarDPReassembler.hpp +++ b/include/e2sarDPReassembler.hpp @@ -354,6 +354,11 @@ namespace e2sar period_ms{100}, validateCert{true}, Ki{0.}, Kp{0.}, Kd{0.}, setPoint{0.}, epoch_ms{1000}, portRange{-1}, withLBHeader{false}, eventTimeout_ms{500}, rcvSocketBufSize{1024*1024*3} {} + /** + * Initialize flags from an INI file + * @param iniFile - path to the INI file + */ + static result getFromINI(const std::string &iniFile); }; /** * Create a reassembler object to run receive on a specific set of CPU cores diff --git a/include/e2sarDPSegmenter.hpp b/include/e2sarDPSegmenter.hpp index 91b5e407..e327a050 100644 --- a/include/e2sarDPSegmenter.hpp +++ b/include/e2sarDPSegmenter.hpp @@ -286,6 +286,11 @@ namespace e2sar SegmenterFlags(): dpV6{false}, zeroCopy{false}, connectedSocket{true}, useCP{true}, syncPeriodMs{1000}, syncPeriods{2}, mtu{1500}, numSendSockets{4}, sndSocketBufSize{1024*1024*3} {} + /** + * Initialize flags from an INI file + * @param iniFile - path to the INI file + */ + static result getFromINI(const std::string &iniFile); }; /** * Initialize segmenter state. Call openAndStart() to begin operation. diff --git a/meson.build b/meson.build index c4f25c5e..a817d36d 100644 --- a/meson.build +++ b/meson.build @@ -1,5 +1,5 @@ project('E2SAR', 'cpp', - version: '0.1.1', default_options : ['cpp_std=c++17']) + version: '0.1.2', default_options : ['cpp_std=c++17']) compiler = meson.get_compiler('cpp') diff --git a/reassembler_config.ini b/reassembler_config.ini new file mode 100644 index 00000000..90a84615 --- /dev/null +++ b/reassembler_config.ini @@ -0,0 +1,39 @@ +[genera] +; whether to use the control plane (gRPC sendState, registerWorker) +useCP = true +; validate control plane TLS certificate in gRPC communications +validateCert = true + +[control-plane] +; use IPv6 address if cp node specified by name and has IPv4 and IPv6 resolution +cpV6 = false + +[data-plane] +; prefer the IPv6 address/port in the URI data address. +; Reassembler will bind to IPv6 instead of IPv4 address +dpV6 = false +; 2^portRange (0<=portRange<=14) listening ports will be open starting from dataPort. +; If -1, then the number of ports matches either the number of CPU cores or the number of threads. Normally +; this value is calculated based on the number of cores or threads requested, but +; it can be overridden here. Use with caution. +portRange = -1 +; expect LB header to be included (mainly for testing when withCP==false, +; as normally LB strips it off in normal operation) +withLBHeader = false +; how long (in ms) we allow events to remain in assembly before we give up +eventTimeoutMS = 500 +; socket buffer size for receiving set via SO_RCVBUF setsockopt. Note +; that this requires systemwide max set via sysctl (net.core.rmem_max) to be higher. +rcvSocketBufSize = 3145728 +; period of one epoch in milliseconds +epochMS = 1000 +; period of the send state thread in milliseconds +periodMS = 100 + +[pid] +; setPoint queue occupied percentage to which to drive the PID controller +setPoint = 0.0 +; PID gains (integral, proportional and derivative) +Ki = 0.0 +Kp = 0.0 +Kd = 0.0 \ No newline at end of file diff --git a/segmenter_config.ini b/segmenter_config.ini new file mode 100644 index 00000000..cf5b297d --- /dev/null +++ b/segmenter_config.ini @@ -0,0 +1,26 @@ +[general] +; enable control plane to send Sync packets +useCP = true + +[control-plane] +; sync thread period in milliseconds +syncPeriodMS = 1000 +; number of sync periods to use for averaging reported send rate +syncPeriods = 2 + +[data-plane] +; prefer V6 dataplane if the URI specifies both data=&data= addresses +dpV6 = false +; use zeroCopy send optimization +zeroCopy = false +; use connected sockets +connectedSocket = true +; size of the MTU to attempt to fit the segmented data in (must accommodate IP, UDP +; and LBRE headers) +mtu = 1500 +; number of sockets/source ports we will be sending data from. +; The more, the more randomness the LAG will see in delivering to different FPGA ports +numSendSockets = 4 +; socket buffer size for sending set via SO_SNDBUF setsockopt. +; Note that this requires systemwide max set via sysctl (net.core.wmem_max) to be higher +sndSocketBufSize = 3145728 \ No newline at end of file diff --git a/src/e2sarDPReassembler.cpp b/src/e2sarDPReassembler.cpp index 5abe6538..81de4cdf 100644 --- a/src/e2sarDPReassembler.cpp +++ b/src/e2sarDPReassembler.cpp @@ -1,5 +1,8 @@ #include #include +#include +#include +#include #include "portable_endian.h" @@ -523,4 +526,41 @@ namespace e2sar delete eventItem; return 0; } + + result Reassembler::ReassemblerFlags::getFromINI(const std::string &iniFile) + { + boost::property_tree::ptree paramTree; + Reassembler::ReassemblerFlags rFlags; + + try { + boost::property_tree::ini_parser::read_ini(iniFile, paramTree); + } catch(boost::property_tree::ini_parser_error &ie) { + return E2SARErrorInfo{E2SARErrorc::ParameterNotAvailable, + "Unable to parse the reassembler flags configuration file "s + iniFile}; + } + + // general + rFlags.dpV6 = paramTree.get("general.useCP", rFlags.dpV6); + rFlags.validateCert = paramTree.get("general.validateCert", rFlags.validateCert); + + // control plane + rFlags.cpV6 = paramTree.get("control-plane.cpV6", rFlags.cpV6); + + // data plane + rFlags.dpV6 = paramTree.get("data-plane.dpV6", rFlags.dpV6); + rFlags.portRange = paramTree.get("data-plane.portRange", rFlags.portRange); + rFlags.withLBHeader = paramTree.get("data-plane.withLBHeader", rFlags.withLBHeader); + rFlags.eventTimeout_ms = paramTree.get("data-plane.eventTimeoutMS", rFlags.eventTimeout_ms); + rFlags.rcvSocketBufSize = paramTree.get("data-plane.rcvSocketBufSize", rFlags.rcvSocketBufSize); + rFlags.epoch_ms = paramTree.get("data-plane.epochMS", rFlags.epoch_ms); + rFlags.period_ms = paramTree.get("data-plane.periodMS", rFlags.period_ms); + + // PID parameters + rFlags.setPoint = paramTree.get("pid.setPoint", rFlags.setPoint); + rFlags.Ki = paramTree.get("pid.Ki", rFlags.Ki); + rFlags.Kp = paramTree.get("pid.Kp", rFlags.Kp); + rFlags.Kd = paramTree.get("pid.Kd", rFlags.Kd); + + return rFlags; + } } diff --git a/src/e2sarDPSegmenter.cpp b/src/e2sarDPSegmenter.cpp index 4ac9f913..83c39ee8 100644 --- a/src/e2sarDPSegmenter.cpp +++ b/src/e2sarDPSegmenter.cpp @@ -1,5 +1,9 @@ #include #include +#include +#include +#include +#include #include "portable_endian.h" @@ -594,4 +598,39 @@ namespace e2sar sendThreadCond.notify_one(); return 0; } + + result Segmenter::SegmenterFlags::getFromINI(const std::string &iniFile) + { + boost::property_tree::ptree paramTree; + Segmenter::SegmenterFlags sFlags; + + try { + boost::property_tree::ini_parser::read_ini(iniFile, paramTree); + } catch(boost::property_tree::ini_parser_error &ie) { + return E2SARErrorInfo{E2SARErrorc::ParameterNotAvailable, + "Unable to parse the segmenter flags configuration file "s + iniFile}; + } + + // general + sFlags.dpV6 = paramTree.get("general.useCP", sFlags.dpV6); + + // control plane + sFlags.syncPeriods = paramTree.get("control-plane.syncPeriods", + sFlags.syncPeriods); + sFlags.syncPeriodMs = paramTree.get("control-plane.syncPeriodMS", + sFlags.syncPeriodMs); + + // data plane + sFlags.dpV6 = paramTree.get("data-plane.dpV6", sFlags.dpV6); + sFlags.zeroCopy = paramTree.get("data-plane.zeroCopy", sFlags.zeroCopy); + sFlags.connectedSocket = paramTree.get("data-plane.connectedSocket", + sFlags.connectedSocket); + sFlags.mtu = paramTree.get("data-plane.mtu", sFlags.mtu); + sFlags.numSendSockets = paramTree.get("data-plane.numSendSockets", + sFlags.numSendSockets); + sFlags.sndSocketBufSize = paramTree.get("data-plane.sndSocketBufSize", + sFlags.sndSocketBufSize); + + return sFlags; + } } From d88f62fb69ba4f95a70136a420e4ace2693fbfd3 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Wed, 18 Sep 2024 14:00:31 -0400 Subject: [PATCH 02/73] Adding figure for lb live tester notebook; removing commented out setting of IP address --- scripts/notebooks/EJFAT/figs/live-lb.png | Bin 0 -> 81246 bytes scripts/notebooks/JIRIAF/FABRIC_JIRIAF.ipynb | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 scripts/notebooks/EJFAT/figs/live-lb.png diff --git a/scripts/notebooks/EJFAT/figs/live-lb.png b/scripts/notebooks/EJFAT/figs/live-lb.png new file mode 100644 index 0000000000000000000000000000000000000000..2d51b73525aed82810d7d422e663eb43b50feb4e GIT binary patch literal 81246 zcmZ^L2{_bk+rAlt8S5BXVqz?leX9r!#xBY($}W{wTQUe^Un+Z2$yyJQeaqTdvKJLm z#Mq*Uv6Jn8kEiE--|ze0|8dN5G&5%AcQ4m{UFUV4=dWlJBRv*IUPc-k8kW=g+GaE~ zFf0uX^cI{Ryc4PSnt_G}LvvbN!-8Nx|BQif$Z|Ns0ggppm~Ge)g8#gt@s^QGt;xPr z@+|C-lzviCh>+QQ-e#7NVhJ%@?5&IrG|V0*f>gxO8N3G<;%Cng}wfEHJW#N z(>yc2#IN+&EI&JT##ICl{Vxxs6Ld?=Ks!|_QW%25{GW#jOw53y`+wgD??f>|Vo;_{ z?=?;nVfx^KQG~l(xzhdD*{3233YPg+#WGi~UM-cEe;;!2XX+=!;U>xYF*m;p{rL*% z8L8HL?z#WQ!t&anLtVp%0xQ|W3fGd+YIm&-lEv7O!#i6m(;-YU@ed`f403cISop(_ zyfA!pr>@F(<$KxfA0KhMmW3BARc_B>Lg`r&k6r45q2pMnqd9n>?$PK(w|+G)ium&e z6s2@~HvYuTuNIgy^~+t7AM-6G&sW`eT))4!eavUU_Q$8(IA#Gg+oNZnh^PoCer;U+ zX!x-)z`tf9M$-1;$laB9W?7naKL(34YUv~9_yaecs(nvf{iv=(A}5 zSj9VE%AwJAP1%2ap6GRj_MdePZWY3t3YD%W6$t*l(7{9)my|W8(55;?{$d*(CtRT+ zl6S7~$#3@k_m5;V<{HA8xFzDEmL}T`2aD{!wJ8Peyta60rt`JxrlettR05Qau_GgB z@3CGozOJXI2e zHO`QEWa}AS%b&$V%fM<8VC(;@QpOaDkiN$)yTBW?zqk8yY;A6kJ4&ST+Q7uZzc=eJ zSi{p*6eVVJcVnzU^Vd*m&fVosQ5n5%P=}A~erXM|+*!k%^ z(;VB9`e@YQ!-w-L#G2K5E@^svuyJd^2JNmrx-r&h*Y!xYxcTIc=$2P{j(JD_(b+hu>=>s_tS}gVlLk$&G-9b ziB2p|=xgP5QJs5S57qZJwbK25iqpaEkpuabE8+&2ot$d}`DRr&&JTS4bZ2o?8YGc( z&UD&LLFp)@J|@}GUjNDIe;}J!+ID20}RHOZk<8u+)of_Y>?J{Cu(Z%yRUJ2-67=kclmkY=)OFUrw(A zk@GNJ)xYc1V!bfsN#?%~4jb6?&hkN9@jo-25+pM6@SVJ65iL7@7}CqY`_Br{Ahct^ z66+^?SNbpM91pz(tB*l#um4H{L8otI6#KjR6UTpi`Cp*`o>&XUKY5bxd2XCZIfwC( z7mkmp_&ywO!H0PAC(0|C-4YH7&a24nL4XyuHJcwpwcXuVvsVLGC#3{S)Fzy}J zAoJW7*oPPHkH*nbMmKl%UfsJwwC zn#A55t~x#0_P{ck{!BE=@R7{Je9MCT5%zx_^M4Y|-=)qyg!E>APf{o5P{I?1>)&Hg z5oR9Orn?@RHTZc)+*Hy?3;3_I5`%WgmilkpnwHMeModPU|H;KD_WwEee`cnnOQS4x z^2QKG+NP?96I*rTiznl>vV=t*c_?!}^}kOB@3{MW&FoEJfcUvr)h>nS?;-$}>{}V9Ldc0PT*~&f zNvwdn%#~-}DXTvR3@;HYuYseVgWH-)N__nIN9}Xl8qbe!f`V#R-k+K#1=E7vTO)o8 z*j*c3&e-44J6CL9*x2&9$gV(}wcL&{9)67tSLiw2<@ruwu!F-GCkT=Rs}lde%jlm- z#GgQzBu5?HRXXC|m#MM5+%4N({AN#L?dO13VYWsneF*KI_n@7q+ne30xwYcP-v&o| z#t4%vHG9A3v@Wbk7S{{W(lf`udwSCF!i5X^0ff5Qjs#)PQQsNyU~om5vg?B~Ta*?y z%kP6@LHpm#d`G6c9~sVlE_`A1Ol5iXlQC0RutVUM4*0^6fyMrp=F?@}GUjKA;{Vc7>u2lJp{d1dvy{a3p&kA@&;8%K%_rA8*Qyv3= zN74xv5D2nxm4~z5|Bewd45TF^ld}}HE4}H`)uVm|XHw-f0wTw+?tQtEQOG)_gwFYV z!@wZM&-fmXQoYqHYk3hBqiB>;tNJE6bh(K^pyQ#H{mbv`pKaVaC*#y*l8&BDeCpPy zOup-IcS+~)F_);G$H(>F<5tdC`NKJ^90I)g6X;n2w|||)>f`ozRy_B2H|ich+#Uc# z*74$zy+n74bP^Tzu&M8mKhtN!zG^5fo%p2#al1zafBy-3{@JYx5aLl>TT1*s8w=mM zBuVJl;1ZTFxBA~d3zoq$caPL~O@rcNQV{ltCX6wF7`XAx$1=5DAZRCxb!^byzmV~f zm_=Uv{78-Ytv9=Ora6~D)$iWl+fe^8Qd91Dru?Ah_V><~yNxw^ZcRU8s^>>WEK%`Q zAmzg1Xv2%EpZ>bW&xWHAshEcRLH^c=(#s2#`CZ#IPBg!~#Tc7~+)IKMYG(Qsf|;GL z6hESMXdhd7Q@5Z)}AbyO_;kn6j<8UH(R%wzDT&8#t}hZtCYMY0LpL9i&*3wKU#FTRnjygryR-V!M( zKTd86H7mTJz{x9rV@S@*8j542f=p_)SLtangp4u*gj}q0SIcB`p>8pGbE%n0RxDWa z9;!neQVN-%x1ak`p38U-KigugzWW>}VDL}^n{+RpZJdd6-*vRU=EU{SXJl$;g&r&3d{S^MJg{v4M{b6MEd4Kb z;vD+~&8Km`jt|5Z}Y_=}Qr2T#E8>BpxNBa?I@0DlJST4*elp;(=2v=s4)+6d(vG3A0D^4 zKJ0d>HHtl?sw0VJk(MAv#@kolvO5VEbBKPPs2_tc)r`C}t9v1t)JxGMo}_8!AZI~Z zDv{8;hr1FdFmWb{cuVp1`7b$L9?j*nnmU+J9-3mNUb7de0=UCfx4ub$hRXEf^%blE z9(KOMHDORN!Jk%8QqFCh5B_BnBzemhtQ5UwqyK}YT91JoSz3QeUhO3MKKoy$6|SZV z=3zFZ_;v_F#FU?-276l#RzKF@zkx#tZY+)!r~edQge~Ln$%G#-9`y&?Kjp%b8;Few zq0mbJ>ji5k>X#wd7^mVWm%-khz&xVEu~m$#OeN#@FNf+eEE{BMyJECD5Y1-fXXLA5 zp^UN!FK@PrrK1L~Y-`NktMA{nEN!tpdgxa|0c|sVy)dQ=mfsjGhqPGuH{ARuH%f72 zim`XsaZGxBvJ{m&+iSo{Fk>@@5ZGtkAAnkv-=esT$DlQy7L5eEGmBge6O+F)Z(Rd= zEZdtn!w|K#esc-8koUWk?@!$fGSmW+_2_YEM5GSiQ%a!#|cdX<1o~YDYJPG8A}UJQG%)dX`D0 z<3X@?I>>KPrvw5wv#=W2zOS2V|2t}wP7|}kl5TUo0Al0@to{g>ti2TSB)O9Uy3V}N3Qkk#D5pLKQOs=7NhA`N*fr%{I*`*% zOQ2=Rt4HR=LK2#uiUr5Bg$0s^AurBcAIP7USiD*(lE=k^3B(A%x?+Wrq2rR2O(HTR z=&6?-AqLeRd&K;OA$wcMPfU75Jj!B~Xbt($oB@_WXle-`^ndVXlqEn}&fZrh9PK;i zh%LB8>02YOO~XxHw?Q{M9V1y}BFLd1^O7ZB)^bC42q!|ovZ&pQDeL#;EF_sBHFq=0V*p(;hS@sY_Rv;KPMz53Fn2G zmOATne@j#GHb4k%+Be?yJy}fO{fvjjw33z+XJ&40>1_DSO*mC;=%2>%U+FH9nigEX zTrD~-0tHP?vd3iGR{c5ACFYl2_S*k*40z~iBhnpHsBvB#S~)wBZsOE~gK9`pgHogb zu#R7rlQ7K=3pi3P`cHp<0Q{q%kYR0v2)gI}`*RD6m%5ToqZ!zrI%ML5#iZ_OQ2{W1 zf}ONzHKo?Xtr_mP8E_9){6DucN)K8Uh004MUV}%Cvx)Mn^Fut}Q{fa+8j1{AGOQ;o zO(dqL^A}!qw?#$qLZ(EU#{}y4h0E!Xy45#_j}BM6S@jhB4WIA#kkR1>d9UKuHx);0F1I~FBZ5t!D1G_V?ri~QLA5w4W(J)zeFK(( zjX#-5w%$Y$bWRz8zhBzkU3#mrF+K;4Vmfl_j>VO|mby@(zv~Y& zQ$G4NsF9hkl)mS)3>2zul4&(gFPnV1a6#@5?WLWJw+3zV1wdF>foAb206O<(Ta|eZ z@)YSRl!{mrv4$TnJ~*jkhTH_JJe^-q#Q2lMZ=Br<+KepL-xO#xo#_O~I)?~)oZ ztI+l90vyG{d&0zlay092exX%8xI`PV(@u0rxvRpxe-dg%jmt3@Wt&IS}y{y0RG}s+EVf6 z*Sz=;&@y&&JKPmiif>_}@ghFLk2MBN*UY4wPo`$_aidtvZ$i+FbC{j+Hh^j5ot=gW0#@BGTg$vUQHT#gJ?Z9}+# zUO5Rc16}i9%BNqIzxTYZcDaY=Eo)B?pdX3I;5oUtNP$3W`LL3FRqKk}u#=|Hi?7!pU|8BOCs!*jfO)nb2AU-^W=^YJ*BMG2Yd1y`S zBi@s#)jyKxA@=6`lk5FSUSVD*ZjD%ni;`zR+;zS?IW~{N%Dn|`pxK408!vx@OsxJ~ zFOL71Yaa$XKk(|jc_B1}p;>EhXT^Ac71{(!)vM7^yu5QOd>TvRM5_sB{S3c?e`z1a zgGZD6&puVOtl18f=Ig67fUn|*tkH#?pWt07PgjMkX7 zI={ByjfszjvJU~bv)$112oMj?DQUzK8Vy()<}>l|IvrS18p$y45fC%ijIaLPNG=&; zq)eT^)q4L_x6S3yeXD-C=pZqqVTrFc_3gQ5Gs*K(L#Ghh5k%H8QZRpW*qtA5z@ene zqL&ky(Djbe?h!N~wLmgNcbMy@5)J!#2v69I z>CpKJt{zhz2|stiff0|RlcpzhiwDOu!{HQLq8>+E+YDzquYNNkRG-?yNiNX{-|mryNBS1vSEk(|3;JeRRlR&Mkji9k@gB12QfBNQt3lczRm4mv6fe?x5IfC)-m1`ruXA<=f(xPaVIgjS;X0TqXn(`$`o z{-xs75HXk!#!qNs zI)LQP{7s2d^cnGptYzJA^ZrT}q%np>p^EF54fst-Jx8)wr7u);2pgo~e9fX+2y9MF zVlTEn=m{VsY)IwMagmN~BCQ(pXdY7B{dHt6eZ5+bvNUdkmtPpE>-g0Rw%^7BAL!$X z5rfFn_$uSNSzDgM(or3C!IX!99;B zOn+ku%pSGJ+*h4qq#_hN;G>)P1kb|(_ESPnk{{SIDD}4#3K$QuPq+`-`P4+C9zT98 zfz#qsJ85O>5w0>Tw6BRVg+|^LNlXJBXFpJ&*gU7aH74}juo~FAlqHe& zS%zu9zp97bBWXdl#3^)fY{rIX&w69|BQry6SsYCOo4o~ERa<*RRul@JgMU#Or@p7u z`x?#?{3uLW6vpezfkirhtn+dE@wxD6!I``TWWcQV?{0C3Q{Js_wYQYj5~mwyRH#}U z#uUs+zyH~EEtpecEZDN=!^_X@O29Ui&`sHp#`z-e4E`8l)Rf1ioK-zeGNpj1Pp1trY869;y`d30Ph-x5IT;x0)zYxr8_NmC{2=n!*2;{J_q4ddE9+9tXvqsA%V_hN zb<#FZONJPvNWYAAk^N61*`YRc`~DrA2OXXcVwCo@4*nhriNc7HAXqXdy^p!22wKxa=6chCro ze0=N0T>A95cV#Rf zkoAGqd&qw(m}s3nb8&%TRET2np`Oe#-fa$T=2-uF{OZex-X$yt2n#Tii>dFIbY>O>i!YQPlASyphaDOdAz5JaKW+|HlhJrQX}qg{dM4u@HpsP)yRZFZyHv_av$z`G$Zk*9F1} zfu|xtwvI6t4|JX`fsn-x1+&`QSH(@UwcVeUC>$&g@h0`#5s60*R_8V~b;$(9ERJ8y-rfG-l?bR88qkYgdI-$R zX&@rL0CXU`?4uvE%|pJ;sG-t}kyI$tl_LGLNz*R3(N#@6eWbK2i;=om#C)*2tVM{#H zjnmM4!d4%H2?oGTe33r}1u-p!P)9ce7{IWbVqXq~C2eYW&Mwc)E>OQ53p&w5*a?q8 zr7K_)2%=yUMt$N}s8}|5AG`{w)|8^27{j`z8f&mc%@X2O2g>I*m_kV$*8gDh@=)L- zmOJQ?P-^JcVW|Y!Q^lK!^Yb?}ydMJfGoqqHGo0}>Kmad*%3}7?4Eqx7jsC8Z2fGeJ zG_TB<+UZfi++d3!`G5#Of>EJdggIOeAN>2^5>;wnp6;e&xFh;1aQ8Qr%ocZOtmpW~ zFd(WEo#|dGe=t)`FjHxsF9*&AZ7P_UU4&e5e?lGBYd)ZG9Hq8^APU+)&~btqqH$A?Vo<^Gr2y*HOTZTq+mFlis4xCyTnPZjlYPIs4Ri=DqxHVAzWM}INPBBI zx4oFSxVUMko~h8aS!FqJkWnD5CbT~7eE|-n2p5Qq=*oZh{uuRZJ$1d6)UQ#o52Es!#~Yp)l*gO!E<@+HF`Vpn)Kr&T ztzB_HTf3se%NFdC%Tcgrk9{((gFVw1qb}REQjwy|XdMt(&cCkI7e{1k*Dq#vOm!w{$!s5f|5Lzk zHp^XxZSF!0%4vM^?ZHVyfzb&%P3j!H@TVTa(R7iO`_#g@@CJwrrsrW)ZgwW~-~$R^ zRM`>fvJEjrQOisZm$!O}niiSdT<%J`~|QBHI02 zj9rv(`pOe9sUWr*=)txyP~+C=k31gT098~oG3A`Knb~q=>uPIslILunX=kUh(DsHvimE?rM87f4&LR^Z zb4ViD8U2bWPj+|l(->1&&v6ubZ^!FdVaxO9&pYV2)1(J&yL6@BlpjAh zIoew#ZN zFJ^%1+nTWVY@xzLa8Wb#{N0WW^^8xS+$)0WbPy{ppFSCOl{f{km>zvQ#|!qhwsJm= zx=r;U^EB~=QlFy2z)r!wbimF7OOo@)=i+6qAIfl&?|^7L?&+XoX8z3knfjgiu<^&G zI6@wLL-DE+pYo96Qtab#I`IKAA)Q8j{J9-+?1rZWG?LmL%THjL|YwvPFa zku6r?M{QSoQNT3!#go+loYpDoY29KWO;gjuAB$FW3Cb;7dvUav*NH>f{!t_vwKiuJ zGy4(P1O$B{gltDX9-S2)Sz`uN!f{X9Hwx4`JtDdkryAj|hx< z8r|!Mo={iPIR&g_>bED)(^AhT7Dj4}6>p9F zs2sLt&Lxs*QWDBskXTZgU7g~`(u=3UA7*PmUyzAC5cYFHl_h`sUk9*7%HZzWzy=_H z%6E%iX@qu}y|TWhZ55Y~K7W22U2NoJrKF=h&J+5~_ww0bC6252EH)#2?RK{}l0g$> zmK?Qu*$YRFIXP-E%^D^HO|p&S>4S*5Y5AO@I||}Y_KaLxF@X9!dUC@{NqU-pzS3!cqRX3A>k?3?Bzbdm8$@U~(G6m4(0l!yG z&-C2+<~`U6M9dcerFa5euLJNh3)OY(!n!=u1L-Ha3XWI}QmZK7ZJ%9{0Y<=@+p~Rw z#tx2S&rfquJrh*RnOb{H8^7~0r4cqBIXpcZ>rQ(i=fZbIfxDF8`nAtCWzKIIse0T9 zAUl@(wAfAuL+PY^*19B%JAjK!dV3=y)>sXgv-%tD-J9rQ<|yMWyX%p?ZS!Ahj5!u- z=VTrqyA*>1LXd9}lMmawq)lK@pz?8#3W{nCfP=6Aw^9e(crMhsiqVVE$R-RXgjDS>+2=7d|;EEM_yN7`cl?IRMnHTAPu&@vdG1= zM#6Q?VJ@ydq1H?*KgFyuEqK#WdB@ZE|B1hduU_i5rl96#sW)f~=5$DmhV#L}z> zsLB_>2wODhPPzG|?1c&-mxj+Z1g^Y0*$oJv*{ZZ&we6m$XY*oI-u_xK3&RtiQD41X zCAU%B)o-sz%yj;?*AdQX6{!`tTgLKcvo+ls{?DuLf;tT|bB{$JDX+1sSpSOMN z9*UBpv3c6hulD*Ye0CUD6UNhN^zxDm7>s8p_K}#!uj;WtrVcZ}4h#c7@62cWINHA) zHM-{0 z!oADcx!NT@Ov6-o1^`U9kiqJ&VXKW}gcetx&o@ zPKL>b0A`~bbhQIcMJ}C*5@^~G8s|YL&|jQ0HXEOa%l)}2e8kvYRTWEq94uN4Hz#GW zN2qOYEEb2+5Ri_zFfH=SllEGtdT1v#m`Of3r*@}R_5H25>)gOTp2hmom9N7+O4_O@ z&1%pW*AUK4KeDNs!Mu&aRz%Te!324Mu4SK2Tl$7px?bKLzi~(GL@_o^wN*pFRxk~% za$Db5lq5V3gab1z!o^*X4^M-$^$mw5%_QjYn`TMCy+{Jcy>%pmJwcG1`35w1Ylj3j z&@kDNIB-f{Uynv!cZciVf=en3lHY}TYq(09J&!67m}kJ9a^Lr5#8a(1b0jUAjZ^7l ze@2r#^I7T7xv=g-?N{i}P`>nW=_eCsX_g=xv3uwzi=1KpE-o%&y#mMi+%i~#vSHS? zz_YeIaZgsl?GVobM%lGHC?PD0a*2`Acexf%#TRU^? z_1!TeD4er((cQse^Ndih(JMY3#QrImY)adM0^a@hzm#epiW;Y%&=jlYNiY4-aE1Ra z7WBXgHFBXaoOzo*hDByEgDHcXR)d^j^a0#FVGw)9Fo_%M?Bv|Zhum!d^Vj+j_9 z13wy7)cE__^!wP{4y8V0AZqX4-@JqS_EwNHL<1co_&< z*Bu#g1={sGE%*;IZbJ_%4DKaDoy)$%VsF3F~ zE4>oI*$#;UEXt1kPs3!wugbxx@koJl&dej1$2%cv-%B7t3>VO7;CHcqt0RzIe^#Vl z6$Y9$-j}o@>x>>dl>1G+O1GPDj&WayTdCZo-Sz^$0Z9!826ciW>}v}!^-Dms76n;L zUJn9yeNLFDwAyUi;d2dkq+FlH(Vr#vw;8rHWExh@GHTdQ7z0Dl9*)#a7irT&E3>au zG8q{9r8Lry|3WgpgOzo0h^>L!l>@{BJkmtKZA%deAqP2Sw8LD^x!bH6Sni`_$PTbXW4xMQHsn(4`6p=eay$Q0| zRL~=%oOa${q_CW zp%h+-u}=q(DF9}7>VSCAzH_Ha#daLS&`S?z0*Y{0oDz?q%#luUb|GT!tZQ5-JUv;s z>MpQW7KxU9$~Cq!&RnWEWbES6_cn_WK`Us4IiH|>m)5w9T_PkjC9pPDx?PZAqP16j zf8I|hCii8kz{Hk^T$6&53)S&(P3Cgwm5BH1L|RmOvyH-37m)~wW4fu7ruCc@+;0O_ zbI69JmxMlg|0Y-qhokNBRap&Qe{#LZ11r(`Nvi+5E}R9%jcy}%Ep}^u(ZWd&Zij}d zYh#9hRka_28ikRIbr8m%si8pB5@yXpicr3F@n^GWBY2uEF!vv`gz)_SY&;U$d7S9a z&EODa`W54c2^LxS^>d(LM&pNJa-ToFX(13IWZVRC;gMDhJ+Z3&93YQVHNFj)+DW)h zhz}9moU!FpVjPUS{wk1b2V1KxaJiIi)Piz3M6~~A^|!ww`a)1JErNyyj5Ncrq0YRa z+^Vt@8%%WNf|CbA0z#~#$0U(_QZ7OD(Xxp4`)td^OP{C_#G18>xwgFHLbe z)3AEE+ePNm!kE!w4Oo@dCRzqzwCh^d;9dA2z%z16H^`Zh*9277?INXcz*hQ`<}a4W z+f!8}^K0lEl=Q6;HqXy~_HNqHr{3_m?E_L0Ll=cpg)1)kc zNCV2pi<9xtV0zE5vqkzN7 zfdmYh7#bspwBI@WJvx;0^K~@}Ss|AOetnl<~Im;b3cE3=CX35rhQ)1&dFWWf6NeOwSzV!1rrytupCgK7F-*I0Y$R&Ofq!jy(oTfOzeqMkofntF0JQi^QCSj=Jk+3y`(`olvVch}D^ z48czAKC-aHw8ySjaJ`vW`|&yy&-NWbeR$M4*j2dIvQ z?FYvh&j5zTQTN8d-4}N_qMnFlu?hT^V;k2Hz2AR+eTg3i!*Ra-aziCMbwc6SSJ)@vCNh+J~q3ms0$jlNc5RM&cguojk|ju+qf$rM2`8Qijaw z!o>?l(J(e@(2y-afRlu2p+_`rOEN~gYU#@(t?z*|UaXb(!BT>9zF$)yOqHOhL4)5z z1gIMHO_X0z>md{4M&#h(rHSxQz+$OYw`yAvnOvh8W8{2q8+|uH9a^J?v2|@}bPf_k zjr&9x-3G;v!mN3V=an%Z&dNO_RjcuCz|j!SgCoWrgSxZNeb_#zl*=fKjzN+$yxGQg zVPHwffW#-y)3P`zW0%hQ9aZ%}m`YT;mMd9;mNHBbsbC6)c@H3ZwN9qH&V_lZ1Y^Hm z=MEAP)3$`7CCJNhIAS3nxP&mC-+Yz8@T@#;#+{Phkv~ z>)DE#9T+ayWm|{}vYepqnit8o;tGV7LXjhiGS=&24xMMm$`zMhY|qb!|K>+jMArU} zIG;?+B!-Hmug;|jDfJ5!UK7Jnr{EE6=DaD)*C; zyX&BT0bprai}v=S z?Sor3ZC?Jw-~$&+r#IESr%Zsd$wHzO2xW`!+}%S58Bh@CZ%V6v{#~)6{*(oh26|lj ztN+_U-jHuvtD&lIA#6b<&FSo1fR&m>0qqwO`m(n6eBSQlT9+B{^4qJ^I& z62fzL@BGRFUi_88EMsIB`7N}iA*L6j$x9xGw%7y1PloaCR?{4kyTTPY>qXqCtd6R# z(hrQ@rq7TqY2BqQWnCdU#e&-&6dmD<^yuH%b@4cAT*V&yr>7 z+l4g}2c0kG?qM7cj-&RgYNY$`d)So5&j~%*+MVnon18$gCtAdDhE6|6Sc3D-!@Shn zS}LPu4Dd9I1J_Fgiv(gLf05DBKs0e;KlGF;V-p`x^*DHvow6~Fahs_xWZ)1byq=-d zg%U%8&^b`1mYBapy{Pe&iPpmatv15EbuobVmDq=}OVZP8hpsXyet6*)8r;ono&)FlnkjVxyviXzwe#Naai^z<&>Oi+wFf!+m$8B(EZi zZMT$*7m}cT?`Yo9kTmS~W0@YbPdOeKLV>+XE=@?r?Ra-mm4w0b7o3WLRw?qRzP~Ok z3>QVfKWM7`<^C#YpZh*e@l?x?X767G#jzIqd%vo&y55;D?YF^A&PBQeTW{er?sW<6 zvv1?u3(a#QYVQnJ9Wj`S^Y;Jw@|NyE;Wb^(3Wep5hHgrVcK@O9nTBlO?{eS~eSmz< z6M2R;NRGPq788v148uGY^fc_#2Uguu5ir2!AKzkk_X?v>4<&Q?YHU8{RHcHZ|1iS0 z_c&_i)`{hJn3*$veWSeKo^p8!#pxKt_!BN{UvC&$GbgH6gWzkBTxCUKN9i$_`TB9u;J zfN0bro;K3d`g1r#%(R}s-f}9tVtnnKQvSWsH~V>}O)~=3%B%CfZ%$lGwmILC->}?u zO80FB=eeibH?DxexROcoy2r+(s@KoyAi|K%UcHZWE+kX+H(;ETaM1;x2;Fw31LN0s zs$F7B+F$|EJWLbmmmDZtxmeVHxs<#v?L?3~6bFAB$`IH2wQ~5r z%Nt`p{+l`jslvMVPww@8kxDxH0RFpi?&I{;$-aPNuWGJm(M$*|_^%G;wBxIHPVZ?) z1IO)QY+hZ>yYY^M9Nn4;(}-%PBD}|3;r>HI8RfA#9N@D~x6I|#E$v%chIYfwzX?pT zsRQ2V)9+6eKP0TMikpcHzzcX>;DX8@>o&@N)&j!4GgG7C!8MB)BfqzHH(o5S4+}6# zoV$8`c-T$+wrOC=#C9AHYE8md4+7DK%|J>61lyQ`&G(5%_(^%Q1+Fo2o+)E)MlftoU$mgoLD+9l|4A7iz>#5v9hI_|N%4ZeZ-pws?6$ zD_%_c<;&de%afGdUiG^BY0UnXC+QfU^Slu);2pgr9vc$A@panz}k!aZxJl}c{pwszRwD+6ldsr03$X3I7FyZvY6RJ71QS& zoIZ53+D`Q5*R&bNQ;$xsCN$qZY7le`fNRO zNz@=&+U9FV!BT3iTj1{>R=)eY#w`gC3Qe-^752Pg6gZtdz*7fvEDZ5$= zaK~qlKFg0;rF&e`iO1h7X_=X&?Oq)Ipi{DvDokvlx2;%Oqf+i}X~01~y>(1If!T*$ z$$GZzQ6mmLF;FgUt60Bx0wQhj{P~nsv{ZMB_XcsOd?phm5{vdI&JJsRG~J~@$*tr>x$p|q>w*ST+w73A={-}!(O1J0KKgR+HFE+r> zom?kd4n%^|xcGptn&;vJpB3EFYI2@kT`g`&+sgZwLy&ZmmBTaCF`t$2L)aTl9YzN^ zgz5?jX@1+5AW{;vA2`WYQ1~%ZBUGPtZpa$AP1^*T9}WoAoj{dnl5F4UFtMeN{pxuV zhpGy0rH?ST{-Rt30j64TgKH&9q$i!NJubr}i}#ez!Uhwg6B=un-RdbQ%uKHE(Gk0y z;E)CEJ-TM^3_b!)k&`!=wzhhQov{1ly3FzPR0sZ569(3)j*D^A5i()UmrWRI z&X-$XG~8Mr_RW7|nvy+~dc%X~y_2eD>9 zShPFRe1dWT7-^W5vy)ny?bwHt&dmRXwN+SaUt$fLpFe{QJ5p?frG zwdr?{Ys$eqe;&i9DMcgsqnYtt3XOG zzr}wAepBC2GT)Qib$fg9VZ;o{W%5VOB52&MguKskU!9fDS?nyk{LW8Uwuxf$S8frw zi!?j`J9`Ycp<{NksP0z$=fd7|-gk{X$w z!~e!O!$~))EAXT-!fnvibfPb-<<|88AmFnYM*~IV+wwMWx8yi$QHy-Z17M&UHYpRX zL);jyI_n)RYV%bl6<`=A^rD-__DDY8=;z5zG8!`h>lr_E!B&om<+s8qV2aQT7HvW< z0f*ikN}vKByvUpzwOJTnL0e>3cP@OiDcoWBLry!3ywEVAcA<_pjMH(@Zt!!*Q}<{a zbF^c^YxWvcg`kFrCfiOl%Jm*B{>zuTfuz7)mq0SfKEldoc0#`Ofgb-4w@b>tUjb9n z=h_jkri=%Oqet47SGk4bcLznn7E(WcCkviYt*Wjmavi9IZ4nrO+Q`L_- zex=v#5~aZ?e$M&xs~X++4-D_8@!bUXSC36KY^Zsxy?7hOtB}uoqprXM8jVt!dOI&Z zZk*hA&!+lTQo>Mm?U=mg7g!7`%VbjFz&*kbqG}x6dCZEE6PVyL^6I5oAeY2??`qB2 zZd@$ym>6=LA7Y0lu;!A?ouXJaKQ@Q%ec7?I%7s_|ju7q}y801`uM~-5|Ls$4zZrYv z|IziHQBiG8x3ILWHVWF^)~xLm0?t@L_s8R$kJnM*UsbKQ z_zfB7*Y7hbZ}D;~+o!GA+lJbafen!l1FW!lgFk~e;*UUtRsVD&Eomq^lkjMXor?=?e?aP{Pb`iuGFLEH%f5{_M8lhSKX+XOsl z#NpG0JM1z~s(5xKKNPV2ykX0|w}0-y{;~2ZlTxBwJmD&qzLJgZekZ{-JV^anq4$NJ z-fes$iM{m50uE=#pD25sYh_ogh)*=&8f_T5Wq}~GZxTk3QFR*=1Y3QDu(*|_Y;c??zbO2i@lx{b_D#x{sVZKC+4L2Cz2q#x*A7;pNoCxR zNtr1@>YG}oT+a4Y*4S>(!{v)ppFfB2=lDU1uAW@hsD@6ELfN$fn_}!9(9=$m-HTty z9zV%&ccR$*X@k`}Oi#3@xVA>yZTM9AP?cD+hQzJkFJEPLk7HQGH{PxKM&7d{rxPBBKYP;_7ML^iizOzP^v&vlwO=B%roKDq zqk5=b`>1SYbCK8c4b43!Z22uUZS4q9?;3ml3f^tN}nV#pT|0cFHCxsutT6oGESS%s5%&$(dsi^y7m%N=)RL ztR2z)jZpwr#Cj`o@NI(S%$QG)bnC4wclQ0xR3_v|_Bu1D0X@|am>MtfU%AP>(sFeU z_sIol0+k;u4K&jP^F|&tx9|LFtiM<3{DJ6R`}oE0Ur+Pb`KZ}s3yarFwsyV`1M4U_ zFv|n@L#k=UibqXTFn1aMRr0qGDq>H9iae2EZkpS>rqVmzbm7!&ziFf+pSQ?` z+d)}=5rq=y!_Tx~^`4ITk+H_OvD+;kMy6ar5lfunp@oGab8My8tG8bjg9?VWafEi! zQRsgYtuL$uHMPo6{`vgZlfh2R1KfNIf>JIod1NX}ygE2$(@yMgG;6<9j{Xir)lrHQ?alBG=bx*@6_Cvb3O|9>~!skY3=k&0Xiu{7M8TgNeqOi^B)m8LkYvOXHnhek+hJy)wu?r-FUb-0st3 zWIovY2_Z^v4e`HAf)uY52yD8Ae+-`tkht_KRJN+66_{(w9{i;hIL<+Lq8+~pygC4{ zKA-k1gbcj>J>QLX@ZlC=t-F8n4_v7ErWu1&Lo8Ln_<{451dN!`=D&$L2cOKw?@~?R zb=#u!Idy+WdENpdP2KriEet6%P3!Q5KVQ}#Rf&XU$r}S_b0$h*k19LM37l^^tu8#~ zbc%qlq|WlN(uJ6vX`EG`-ld8+yjE|sceM7$F|Fj1ib4F%IFef@4n2FZGL&XOurQ9T z_!}^QzrK6D+`uWV5fjpMu}a9IKtMhBjKpNN9XuXrr99hNIC0#Sf^;ffApLXCJJtGH zp&OK5xmQ15<(AW~dz7DQZp#d+ToT^8G1BX1OQyx&CSF)V-5X{3?!bjVxrPqa?tfg* z?pN~4h9hEu$p^g$D)7KlZxXxkCBS-Zcv^R^1t=;A$qIMD1h)9&Zyuq~yjlwcUU}Y* znv{&A)-`RGsL&f;1AVR(g4g2v&$pCYHKK5!taiAg^(gfDGYb5|tb}{Ijn%x-PI`Oi zWayctxMq4xGqP$jPktDhe)uuNe7R!TeG*9K;jRI`L}fLlc8F*c>DBmG(@ zR~~A#g;0p0eT|f*i%vbxMN+x->BtKmFK;|Z9gTZk2}OkebUEO&VCxYlr{!&O=uDd% z8aA)2U|Bo4+)m_wypt2%Pc2_K4$RFK!NUxqt~Fu|GGl3d4L7G>aB1B}&o&)=2QN3x zI%@x!NuwFslxKl&Sj0P1PA!oO4G8MCB?c)?_klWAIN6(Vnm^(;lR;{c=ar)#G0?#J z9?MfyHWEAM7q8qq*>TCM%lrB`mXk-I@9aZui7=fwDDQZ>GxZTA zfDB%iYz+)mHD$Ol&ZPd1y-zHIX)bg)go&MLaalH@@3wNZK$bCSd{I_3HJ21Co?Ix* zQ$k{V5WMnbZt$s@&OJE~Q9;31mW?H5DNn3x!>Cw?glEsIm%Vv)>p~KnQW@xk@ZYIM zAN_^Vd3vU_kXGc3`0UFY=ePPy8auaFf|<=5I~>-Q%HvrabfOn(k7|6dyzqx`;idxr z6V)u|ogaPUcP^G)9=OJ&^4o$xBI;bu|B&5zBw$!a~=lCm&=^KR%@gvV6ihH)IW7?*5KcJHPe^q0(aZ zvx_0xg`rr$dN4RvF`sO;hA6yZ&?T?!b$$DqD<2OE`mK8|4?g8C3Qkq^J+q#p{0+}0 zsvcoRG7w)Y$`?osxc+7Kw%@mr$E{J&VqZhu2XY-QV2wkXqflA4*Xn(%wz1Nv6tn-2 zkjwaQ*SYBr^_v%cDUa?PI82MV3T!^;>g6xH_{u~5r*AtO+gxt+?n5*=TjkJPZIy1K zIo;t;!u;ANZ>|qA82r{MY6X%?WJZ!L?NM`a0#;Rp$EB*QQkotWfj*ZDAUwt2HJ8H^ zx%n{~GF=h9+1%R7wr}qhUJYxFzww)@vbX{0G(S)F(yG8x?1r0b{9GQj>z0XUTB?qv46f=H+o@xh?O_C_CyaorIz z*89`3X72`Qr=!aq{|G>POYGyHN*|(fNi^oe{7AsFg|zYb>vrLz7$86n*}C=oOsSpF zqp)e(R|%J_gyGCS^B`r+t*i{-$5xOf7*|6SPnl&d+Y7Vu6DbWnn@Q6VhluV2yyerL zZuEqcH=Y1M5vGsZfAB|zuiLI7f}oZ8eT%r?Ucc6M1xU}L=7Ye`)caUGDIou7TXH8@ z(r@V^x0ts2z;gVFyNtCGhnZ~UV$CTB;PhbTdCY+g z>Px#@UtquSL$#U-#9pn_C1$K1>bGO%UfGRSi7-h!8)omX4gVb3k0wj!1kV3TC<@;k zMZmNAjhx`BrzcZ@Q7dqLqsDTw=I^f6T&B;tjym6reu@c^i-fa|;fEdOGCZbyS1Z5t zi7*mu=NEE%yjiY%YMn+snEAE9|K^?yi}lL5pVc9kzIp36UDi$kN)357n|%6Xo-G5~ zbji{E+ptfi^uw3^e}>kw%O`O6!HC%5Zt`sbUw+fbQ=-n^MGWE+Gl% z{QVozDw!D#ahzuS;9g6KGZNIb#ae2GAchrv#4`1l7IQ~$#uY)KUHo}Jk5fsxMD%i& zJzH2Vv23~Fudl~=odqTSNy!81NRucWqH{-e54fs)%9s25A`(tD%WQ&N$H19M^%=tG zsGCa1tB|tHTL*v2-rT(F@A`a^(dN+kdetocAf6$0CEq$zkUhAZa>lsZN6_1duz)s< zrh0Kr5Bj7q|`xBj3ZQ!_1X|RsF);W z&ji_4=L_IuHT?D(7u1m~OIp$fD~N&`?;(lXqQfQZ+qOt=o`)cFEkBoqea=v5cw!lS~gAxQBhK6vfZ>zJ|( z3-hC~zkgnNmbWUo`0@Cr+6q(L^fR&Je2W`T7C$rw+jtx$g@$k#Ug)cE*{|7OYY7^{ ze43Hj^cav`t@}%{-$vY9XX!_6#l_D0d*N9A6$_t9=!DX`H-mUbC{}=@lxr0M8<41% z$vhK-H0(WOp$T^ZEnhuKeB@C!?^v5BCpmew`wQ-T9hj?75Y)f8{Ac?_0bcfUi<$o{ z@8bA2#hgRPEs3KY&;=>3%zI7Yg|(ass{YWe)I$v_-h+R_OoguI8>E2rR?g1R&9(O0 zX&iKWSA1aSjb?D_XeF<_IU=TfDWz1@k!K>PMYpN8O(qU1!PAo1TS4i8$4@V~%b5iK zf%Y^Ww>T{%vHeeb<5|RI`(2SQWkh;JwdMH2`ob_25~y6#Ftk-ZK$H9X8#pP9je1;6 zo6Cs;#Hi5yI?u)q2YiWtA8YrzHzw$aNqwM+e5sB3TeX20PpYqLmdj=fL2CFe->Hso>NmY$$3l&0a3#I4SSZXd9s=X+0)2!a%=LhDox6d zvBtk4U+{JZK0X!paSq+bt7uw4v$0&y@fH}2k6Y0^=b?kbAa>=z+^u6TJ&oSp76@(Y z&!pC(Vh&hJKh7-GEJ5DtK~h8PkXb|OaKvl>@oSS;1jL4#m=_JT~6;Y z+u3gCyx**H%Wh|(OYIL!pvH3D*Mp!{sI|Vm65XQ3+zS;djpnxKJ#<+FK2LLVdvq2A z;mvw=OG-E}2Hn>4)D3({7#2#p>-fwpKMv2cnxe=|Picz!T6WLsmEb`rKcs@pS@^h$ zvZDT7<*WuX%RaR-JvS^92xUL`x&V2u=DEsFm9ZAj*jkmOpyo@hU2iOnDaYH-I?1FS z;%Mj|$gj3fd8r>(b;X~|dB5VPB1hI$^F{?bw0=!?(QmtKrKG@(FET?Odi^D#+l!Fp znAs1D?H!5GGa2Tu(}h?;xm64KwE$umbrMI7kMOFdik$r}ug|QHij!1!ep*IM-&l>ccUpNTb^7NQ zi&=oZ@9-HhGiFmCFc*Ax;K*)5y3|6A2A}5#Xk{Cy93H-I{aEFOk#+U?mqD=&e_vh~ zI9T*~;t#t;w><)vycJuh@x4X7`)%|BCy*ge&z+T6Sb3STBGdW^h_(^=Z(l=4wb8&K z4bKGeAN4Eq!)?+-aZouii21fxtP4Uuj8g|YD;Ks_g3NxN_Xu_&%HbBCH)4>%72Q=y}?OXV3lr2R>1ek<%}aXef{ zW(eAgdMpZG3#DR-55jf#`U`al8er5!jC3SrwpvLpu&we8)43EB6kZ*p|3n9!390of zFRdH|a!P`78Zt}UolL+;NL_Cc)jV4!i!mAElY{FzY1DSkpeGr7`#^s3fwtaeE+D)- zSg7&R?`V=ny zxw6c;-XBq$*uzQ;FA6;^(#8l(jm2&rdJcL76s;} zwcNMq-ahZbzNM!&<+>Z_>Ar^iTF?@W6T;{?18>Un@g`sYWf1S!cYo?Bjd!4NgtTJa zfCT&3!*`QCqGj*g`gO#L{Z=l?!Q7RR3!aZNEV}e9C__C$lcD5S>v;M$)*(5cV`Y)Zrb!pdT!_wsb*srgolmelE#*QS5uM$=Cj z-lZ=RjgBq87@JEhX&q8M0}CIpL+O|B=5t`Yl{(Uc@NXW?@dPw?EJH%)&_>Zmd=bfq(uYW;833ZDP;NPE?59M?yB z8kvoKg1s5zqkli}RhMxqjD$Ke*xNMP!fBg`GKj+II=fUsFO7gYijMcwz#c-wyV)S9 z&$GGf^AnEQ4}ke_DUK-=96&k3c{X!4O|AA+&(AYzO?*69qz_$v`u4R;DO)5$=;fm$ zNUKb16cH;Xs89SuKZDi)6$CD{a{~$K9BqcT;W7+q7*jw2;UNj+Ty#NxD~p9EOl9WS z52(^15&+!tLXrA>u(F>nRYPyX^VTT9{i-SDoZqWa;>mUZg*fmhUtxnSf2FbZ9Au(H z$Sq_yLd#b*rl6w`3Si7G&?KR;iVS1ue{LnWoHW3T-r+%-&W9!!5dmcIG^0dLbB!1Q z%OdL~F+vp6sT2%~KBJ8a-bbRGE)!%M)bL1%Pt0J8H#pxAP%3@7U-vLWmCZFN`oX~2 z4jl9hhF3U$;G)lOSKvkT9ZKsHp+({31a5Volod!5KLTmNQl&L`Si+m-OeRZV~4d`;W&+w_ddEOpr`@poO%~cNWBkKF=u2M_GicJ zLLrB8TUvVF<37(l0<6G@D^K>>)=$ttp-?^?Jd>vFdbD z>=_0kCEp-um%VTI{soY&Uzcp^dmSKILlj56zv}7fnZB8V z7MlLXOCLZ5hZCgvXfgOLxz?0it?0H-<_h^}Zdi}sEOnr-b4LkEjG9Gn0FF6HMxeRd zyJ&9t{kQQrQuhN__`PP?l%*(S)BSU>bS!Ns3X=NLXtfF96hTK5in4aL=ZX{%8I`%F z5DdtiP}2EnaEIz5hk{^lV0-BQ3L=|jKveRfT&+MXH1g_CDYzm|i4-2su-Brdi~K-} zWq6IENTE`J{!4Cw|PwHU#IJM26oj$IZd^^+> z`yrzj;L|;tK6IDSo*Vv|Y#R2Hd9p+3JM#8&}vvftxe_-@sJ- zSLA&V-WU=B-`tFo+EbfD#ACSpH4D#G>QmzLjkCO|Zzmj&!a;m~dyHrozyn*oK_TfH? z2-}^dCQT&6u$H%h3zbR6S?*JQJo`ccXzBRcu4;=`(lzD{l$D)j&vl4q=kz?8gNSz(ihpx0p;UuFEStI2u;=-jDf2NH!v#^2}O z=C?+H8rG=XNz^7S$nEIFZv#7GSr0X-U-bCJ?r-y7Gx5A}qIYsh4+Dl3kW(Z8zH*q{ zLF8Zelx3zpW?`z!9mZ0>Z|ADtfhCrKd>+~Vm_nz&RfwdSv1d zk%NJFn{O2YrZt|nfzX7mrz=~VN;O@g6CdjRsg+Vk?PQ84T3Bgsz)2%m#_B;tr7jeW z93vI6%0XKtcy3uY@1Aw=C%%ZB&7-`$h>u=L7v?GiB!J2x2%NDbGvjx2Jz3k)hFQPU zxuf%uedq@&2Ws$O0mqP>RZS9>6}KZgGZo_*3#tmWqN30a`R`;5ifm}sYSJ^!b?G~} z-od?@VkFz zw?pI}D3t)_5)$=K{}ibSQU2YofQb;i+R&(9E93h9ekh*Dc>|$qikWQ88}G}tOBFR4 zWp7r8^kq-O>wn@6N(_Ro0RNe;*h(CEDSedmUCydVoAGL?xmkn^XYapjPWTp7C-lb#)n5 zGr)k6|D2DP`tSktFgk$2+Oz8Fm#dnnWP`ZJoH3W*Wh#U{DmNZp9zW#J62XQtq1r1p z{{M*z>h4SbCOQ6j`68#3M)P@<=MNBNA zE4LMzyzdY|CV6J08`?94Bys^epe$=ytE2)YB)w#9Ov44|H5~#>OUX0IVgEp0m}R{~ zC%juK4K4BpX+wKwrYm@u=F^!=pivfr&eb6>X_0}uBrIVEQ%~Hdo>>+bp~+D4A$n*g z1=+)8A;z$Hr;x9eL=)FDCuY-d1_U6gqm0=H{3BLJ+}dLl!vv%-xxHwZ z2NP;f!#lN0t&U>asZ=M0FjAO?(Vs_wsll#He4x2a^bciK;X(N1d3LaqlsmE?4qcnq4qf5qpb4xTcbdQpe9s1O&f_s)Iw1ohWEvN z(FV_e+E2~2UIU|F?)J~bG{Mve2T(-D<@_wL7>&{Ah(yI1m~s)gWRX<`y$fr>Cp0e{)%l6$vJE zGG0I3b2lU~h$!sh-%BgXQ(p_e{0$!AaqGKV)pBpTyV-iSf-V#{ z9DegXb4_(YbyFed`=6Wf1vm;EHq?~{Q!e1OI;AeG4SJMHN4a%^xO6*qbO@ulXq9Yf zm#|-;^SypyVIbF!Qa9!OEEfKV0;kJEFxBwUKoix_8XAPc+*ag>O6q?e&GI-Y2`{Wo z#il6D^N``C`>aMv$Vjf%_ij2%YrJX*=eI;6IpJtw9k@XPYMcOk4wluc&oEITm@Tr} z5RnW)|Iio?yli}n_1Uv7$P6U@AD0dAs9HSllWR*u67zLmi)2-<3#GKIB!j&6q>k=4 z8st63Xc7bxug@2lBF|K&K`_ik+Yv1+ELii)$|SFs9kjtq{-x=7EI<*fdgFK6v7o7M zZk%j0JIDB&kHhr0=~tG6T54$GeKr)HS6T)xK_u{?n+f_$Ag6Fmr+;AJwf19B&OIGN z2quZhqrH#o%S!_OBne|g!kJFIY+jv|48|o0Oc$@6yn_67(=jxU{+2Cm|3PHAI`)MU zqVetV9MyJ2UteEVGDJS-)Wc|G?O$;vh?zDUd`>a@oxb~LTQ76I8=n_@j|;_x^@bjfqmbia}eX3{|;h(D8sLre`V4&E*o- zj^Q|=s<@xAEc>^5iZ1b5r{HVHsiv2CH8rUq)Z#zVPOhq`xcB>aUX{YA^HC0Wafqa; zhZh}+m4GGfOv4miZmF)sYmm~R;B6s*d?rC;X(ffu@BOgLhm|(XeemGbZ5$}zkoQ~J z*tj1cfBG~gX1$}*{OWR64v`Wa^+5AbT5s=Cz}V8L{}5=XIK{EMQGF!LJ*d3NzzfqM zf_z3?q(DF8_PUym#qAJy6=~RAQS&v=2H}RBJXjvn+`PQD!+0YjqtZk*l3I*fO2UDh zkPAdg~B>&xUYW(%~0(-Yr z{H=fikRStS-+|3Nh(SoH^od-3Sa;N|$5Gjo=?(H8=3Nl`S^t@niV`iXtg!Do2>iumT z#ZSmeS?;aSHE}iGFYIR07f?j^IE)s29}BIVho2CnVLm54Y>Gi)Fyg*WIFE7 z&Q57pXjmeWM@CfUL1Cubz=Yp~eb|r3IVbbujal7WPYXs8`xI}Pr^9=2{M{0s2U*rs z0o)_X$5rr=NnywWQ9V@L`x3i&-i_t)E1(3=4AwIjeg4mS zYNL~^$F9k{>wl6`V~n!8B3I8$VNdlwte#6W_NulzYik?IB$d}s#hy8m zXPIYp-qO+owp4n>5XF9Di3@*!JJ|&%3naSy7BWHJM3Eo7sgHKuo>vAf?F)M$#B)Z| z^XJ*x$uu$@Y_BL;$`*0D^G3D7f7QP~HMzm-B+jEhOXXWncPHh?6&fCSyK;h`k%7UL zcg>d}$F_}k8L!`*r7_hl+kVJ#L!sy!o_GC(Q$Gs)?ILcmNk7O+zZD1O}HU4ajGZ#A2nn>#ZY(ikACv2HlE_~D^jJJwwSnm{`A449dj z3mMh3Y=p(cs>6t0US7r{5A=5o?w2>1fsD&aR}!!ANbL6>Vh@kLeVED_{@2}r4h+R z@zu-7F&VmyhX*Tl>R}hwo&CaFQ=c{P2icg|(rkl+CgY zZC=lL8bywGyB3inBKXRkdZw##6mKyworcAjSd05B1>w_nf83yhknE8Jj=8SiI-Ey&=2vwXpIhF~f;? zDPyaD%y!_w0hOg2T3RZjMPC#Bq@Ertj>^mXSmk89Qq?#6J3Y1GQ7N>D%9$0`Z~jSUaWT-&IxuP4WjxpTi(pOouY>YhUu}1V{)-&0U)R z)2x|Gx}dG`4VEJcAL<^B|M~N$+p^d7F3_?ZwL7#;Bxn5=Lbh8Vznc3Yn_S$%!-s9g z9(0+v9A#H_ejNqUGOy|0;Bqi{p7La}nj{?4%_qd5%wV4l&16;YW8A#5&X-(NzxcwL zSDzF-Q4k5&+U0)lW*R11Y7l89R)2LpZFQ3HObh<|G*44`YNUM6IK%|)cKK2RqyP2A zbUINOJ0b|;<`tMKaPksNxgPD`ZggJ@QcG}l36vo7D74XwS`Xdj&Xq370^d9Iz^nx# z8#Ln{6R+})7FSYH~y}5eUcNKSZ*5-2dc3{PM$5wI=vLk zZ%hHQ(bahpButxq+2hIc^Nzx&up^vEqz2ItD@Ch~#@yB`C9XaN9-#JI$_5rl&B0@A ziv8DgI%q_FW*Ebhc=SmgDNJSnYnwVLGpxp84BVN>`00%5FNv^;|5Dmpr>q924ru(_ zzmPX|wkm}&dKGhOCrIh?q0Yn7B_AOBS zqkKaJmK${YzBV58FW18J)={3Ujz*#wW>}X;6h_r>?I@U5x_Y3=^rnra>$(|XFDj!t zZ_cr<+jK0q6?Ea;H1&PlY!&cV0gRmo;l9|G_BToxJPZ!(l=+z1&kr(!X%=L}U|S)w zd_cK(YT|&lKLooCQkT}B4Gj%r^Kk;AqH&BkUpW@^4nZFgEpY?|tl>qQdWTOAS3xIe zq2isPWt-^==}JI6N>oUw%#JwJQVU-0n(R~MJ+GY!tMHEL^I_x%P;|d{kC~oXFRsw$1|eYsFViPwy_ zux^6-B$HBh6wX$mO$@sqSE>yWFdH&Ql*B&y4Y3>n^_N?%DyS9XVJ#obWy%&t1OdfX zeRP>NtO+{ninFsg4BS(*))*}Gt$RKN{L^&cA3K!aj=4k4(i+v;=kM5)?Kb}|E2H#U zn0W$`Oo+WZ0-gj*kGKfQq%V=kHx9sHbs8ZTm*?Z<bIF8Ri5O zUT9Rk!cKEzdn1-c#pZW!>ODcTJbMQ1L1EKfuJ3KhFD99P8QTl%@g>byJd^oUol zKsN7@40k+Ycft4}(6Rg(Hw06bt@jVtJ_lQNdN@#U62F=(FukvQYPO$Am1FkgVjPSL z?tu)hA3Gni`2ohXxPqs>q?HYmj5juKmID#^j07@i!^4@j42G1pJ;Bme`1xausqvy9rVDR(c4L@x2p{rg z1hE(W=~tqliud|_ETcr=`Yf&fdtR=DK$IyBFF_Xs@R5wNHRR=zgP0V|WA0F%61`gk zI+WjG3|-r}@0X|f=hSln!9Fi?ydl*f1+y|#-~`v*1@F<^c(rpAJ69|ddHTTwoD}eJttuYfw9+IEU5o=-IIAs%fR6Dh@FlLHO|{~ROQG8W&d1fo?c?w zYJ>$cnKYv3a#d{IS{x)iSFA0L(2IymId*mU{R}*v5>wjW;mPI~koJ*JW#7*VX?NFk7?%$fgsabZ~gYmGo?xlA^5}u6OW;YPu zgoj94o%wjB%*nj~U@N55*?utGXUTyp;V6ZMG?v>la&rV&#cnkR*QOm6G7#y%EiOzV8(vRHu&049+D;Xy0+#u;{qyuQ+3D~$iKo9h|4a)rUXDR* zayAXT0PFr^R=yW@*HgO(8+U!nL3mFJXisN0L84Xl`^(K2Jq^pg#ZoXm$ zFo07o(}am+*ZIcn@s%#{>{+LozKs(`*Dr?u;SiYf>HuxRYO$w*J=lpGe08`b@zg|dK*iCj5@U+Q^ua!MP z^HTGXCw8XwzRd`9z?Ay*o|_xxRUBm(@k2FU52**8-P~e*L%4TV<1Uu+(@>nCX=2NE zVxXkMIX}G2^P=@q7*E<6sOT4p!ywL*7{b%1PfNo{a_v<0e*$zC6*4=VBL2$-NMyEa zWo@1WlqdgvY=6RMA>^p&V1&=Xq{-AB#vV9}^8t!a#;ALJx3$-6ItdKN&eWl9WXW4T zky9Usg`u|PY`Wybma(*Ho5V|3?Pxvjhm#H;GG$cjYD2x zL?`gOSydMpwj%erPe$P;Vt!S)LnG=(C9cc-%v(Q8l9{ zX&Bo*xeQb8-0`JzS-O@a!KGc&vQi0Q#^?&y*#$J#JAkR}R``m6EWbUN3j9-x1Z>p} znwBi?Pet-3cjo0XR&K0ZuL^G9DV2)XX#y0d3x+hg2dFn^MtMQUL&hMytjk)*U4c+1 zv}ln>;9ILM|Hkd}qq3huez7STMeWZ5slvBQ2J;M!TUxiANK$s@Y_}jf$gH?OtHs9d zsaLVtu%I#z6eKB+fEJcQ9dy0I1jBWHeZozi<+|ENg$IwzH@>dy5}M9HRZO->B1%6B zH-5Hh>!JT;En3PFtWM_CcIX1_wCfJh$Xe5$nC2O1JCwel#GPe*jMPTZ4KUfdn93oc z!HHoD?o5k^)S+l;ynYZSPWa(@a<{IY%7O56M-96FIJ7yZ+~mI&mM^8&A8Eh%K^XvX zADAgR^ggoq=Fevm$3m>!YhMR_wVEuO*x!&=cI$7fM^FpWAVtL}-joT)4+1)r)YA=Z zLSn>X5YJDsxjq?ioadVU*!7hcvW6t?LDk*0_|fZUqb36vvNqT|Un|XkXr+6#wKB`Z zR-4=igLVULrWZ#xb;C)2N79XkpU*uiC$Irp0kM*c{5SNJZ$DQt4IB4fKWOdKEi*q* ze6up}wXltC;ES*4S8mgdCkm`h-8y!!y7AAq(@VOR=r4smzpu|@R=s2V(d=3_+|swP z?CBVE);@Nc?(Xi7Rw3`>1emEXZ@oFslDrKw~2>23JsuBg6a1=c#$(!ZT*I}M=TKvr)4`l^8Y_axCI`F;B zXKt+ao0$w}9*NqA!HQ&_jVcY8|GOs72*^<_knXw=hw_z4EpXP1Al)jgPta}n`)8%v zms4X3Z;*Xt{rswu1_65n%kRw6Of2(;HZ8|;z|$7{o!(qgO-)L_HHM#hWA92}f5 zF)z;TMRi=BlHg$O(T(%1zSlf9yc&C9=M%$6>O1Fy8V+M=Uap3VrFq=^o)N(y?$?eL zz+9Qy3^1GO1nK_oF`i1>h_j_uz+;cC#x(U&yb8;toLHsqe5~w!d}piuL=`9Z5_{=0 z!=V<-m+N$+C%*8V<+Y{79DCDggbvM+_|{rJV{qYjlF?ZDZ50sO{#YKGCE^stjg0H7 zUCfP6TDGeM6KYXe-;P7uzm1OBOQyzUo3le%iwpr>W0L1Mbe=B%b2Zm z>n8GRP4yW6Tr~T!D8||A7Z*ALx#ewdmEe+4ikdvHzK38BlaAhOn7cL_GT6qix<<)T zKkrJ1yeacSmOcrGYmyk&KSpmg+1))eyU{GyF{&O%O*Nktp75w&{d@`zr5u3cX@DV=s!=$exH}Lk z5+g$l&V0TWqeGZ{%JXWr9{(Zhq=r+}24-8)R8Y}}r~+^xHnap%7u*K!o$24PX2#+} z^t}^W;(>HyzkZ1PcoR8gTvr39KXtq3HR@87#QqhOGViL0+I`P?B4u0;PwTBmDLjTP z(IKf7pcCG6n3A<6U8L=9mh%@Qk<|6I7(yv8zq@CzmL)?JDQO2H>?&J9Qc^%{A1#tJ zjO?Cj9K>r;+Ne?XTpV0_v2w#>tkT}9{pnu&cA_s z*@K)*GC=7!zk~B}+-p!#J#vTEb6Gd#J_qUA)C2WN`(NBJhmjB}K2246R6--nbhk}Q zCOYH-hVYVRBeiQady1)uJM@p_xyySAS(F+dlq5QMx-qF*e4Y%W*SYK{Pt4$x8V8%%E2uxAv0oUFD4U`u6LLv-WE`YhVq&cs?Nrp^|x-uhg%Odu&>e* z9_{a;ASK$*)g_B|giC77KI)7!NI=nfeI=>zC_msxHZ=6 z&-3#5q=L3QuLEp5YIn9-;US2+dsMSii%M|kQ;+$Lx_z{wFtC7WS8BS1&%D!Yvwk(Z zzrVT(9B6CKrcuOX5RB!{*77Xy$Z1JFv~CtVRr=-2<*!T{0#Spjgy~J5EkkJ{<%mCs z#SEJH~Vn%nc`oLivf@=>qDyZ((4;6%!rK!3dFRbjNlmL0y&3 zDS&30Em6R5uF9dn2+C_FXe<}@o&u-Pht;?VheEU>uAMdswG`|~9Ne}_hXsZ88e)hG3pK%>O zt$#=e$rz!zO!mEEF&(~w86eRrvdFroQ5*D^bFcSl?L%%tNCB*eM@j+x^ zyZZGuyW}9cGH048fkEKxwWz>9Wq}^fbX+3bTeZa9#DeR#SPJJu& zyruQa!1+#i?;X96*6M&0*wc$L%1^D!7F+KoKO=jJJCV4PTJ-+qpahgHXM&e$nAjNP z-i)6XOn+4RFx-8bdlOPB9{d!AzMZAVlH4FcyQHJz@!Hay?kgC!>u7;_@Ys-xOmy{flu#ox1_S9Wi#AP4|8E1QKVun zZ9uPhi!#jeja5ExeU6hyke20iGgDKX(o>hDqyV`CksyYc6AzXzla7JGY9-3dL`6o-?aPx> z*{=r%1{@(re|aT?nFnRdUjhUpkdjz@|Da~!fE`*-NXj;IY?@9NrvvGH7CF}<0!f^a z+&NRN4T`m+iy)=l_R`bGSc7h_&Wqa1(=o%VBQRL5(4Umk!aekucfLVMsq7vWzUX;w zwME)mjt1=_re+!WI=SS#i{wzX+pGVmAfN(VK#_u;0dV)q ztJ)&+6S!8d?eU?fbAg809O-JJGQ>$rAT^5NYrUgFQtR&vIQ+CiZwXSzZGu#>6!|F!cuUqH6#;$fNqal zsUNmYT|ROT2Zo2;F@y{-M@PpEF$rc0_M1y0-rfm=9V`-%DijW6JPYI1pS@l2g?u>l z+i10w`X`vUS1Ovkb9W2}N&a{02pHzj`~I}kFAV=0HI)qU-vVI42czxM`gP$@GsI0v zQ@2JT1#3D5a1>wJ62%6;8vg9L!)?mdN3^g>p#+lbSdGNdS71i2H4z-~dccTMz9;ta-K9ZyL5nDS66rDS9o~%=@^XFQYDdGXvj&&EP;aN zm#PUI)QoTcY^bRL{56JMWn@q~JKz(!Mp*$P08r@<-;4qIe=fusXb8A&FUw*r!YTA# zInva^f^KjH%oNHJ4DB0p%r!)^HI3yBTW(~SUpX+ZYuxXordsVmlrLA@uUvU{7RctD ziN)n$*NoZM_qIP+z$y)7N;jFQkpqx0gFchVt=m61*xu=w4=7jHau=RgQuU7cMIr{F z4e&LN!=To}b}$B6^GjnjUZJ^aHO%ZZP|PJ)X3&I(eT-LAx4iVv8tUM?5mOUC1CIrw z3VGmX(g8nngm!QY7=ZBD7^ST7X%W+0r5{&4%qdrE&%)F(>rI^)MqAj}UrYza##~kS z8o1s9Vb=(Xz~+%>{~*vbJTenJvMLK!Mh9p&+i1^v?F9LoI{}5_WwTJQg|4=3I`n-s z8Bf8;BlAHH4RpfGJ~~T7rQAhEwhFaj4-MDSh8lx|3xk!9>8%R?*Uo~FvcL-NBvOB3 z;%+)3XV(hR%%{%GJ)|0m_e<+`Y=BtN7@sh*z<|n-^Qf9;w!xKGRP;w&Ajco}>WyaO#x9yl&40V?I_x(4_~-iY0v((behR5#7uYSij~r=@g4tGcCA_;& z^m9ok+SH$~icvH(W4Wtbhw+0QPtEd~NnPU1wY~ro=aVx7=hGcwaOyj)5@z62ab!`vzN78!)-Uy0Dp-s4{SHP-%31z|&W9>jcMTa(^iiu!D15m8dXj%F{oCoz9KxKgvWh~b9QklwQB`N@r*m+ zmhuG7R$o7rIdP*nN#Mizu(}h5X!%4Z7-ZyiUI+^MT#iHWhnAgLan4x24w+Codiry= zb{z?vw#+ww{)-8l7U1J;1%>oO$t(0P{t1G0qU*1}3kX&H^P|tm!QrtuFZk#Of}E(f zY-#lxAdI~`eE4u7Ban^*I4^U1PBl>o8zOKzJHlm1Bbb)_*R^@8gNcX=JUa?kuAuac zVcR2~Su=?Uti~-)HU^J4ME=qy<89f`|2$5InvGRUSY=jl!VJFm;m#(s3vwGu%g79$ zm{JK7=Ugc_cf41G5c7acAqoaE&iiB=IWn|E!ezopKF9NNKYg1e$B?6o# z$(D8?@+u~QJM`bNQGWvpv5qiT{4*^R`8q+1jY6KC`DX>X3S89i`f_*^5jw8Q~?Fa+PTDnM_13MVUQx-H63X=x%N6O zM$|O(twYC-npu|jrh+fY0B$PF`sPKbPF1qgu%-(A*BV_B9P4CpBqne`6*YL)B`^Usx7!cd$5pvC2-*bw0JqK*K}Q(i_2EFWc8~wJ5eF4Mgov`j z*MA_MY-X&$8{2y|`aj$I|B&_F;Z*nG|C|n;gX)~4Y&y0gd+%|K5J`nXRuQtZ6FOvP zl}%_$MjC{&S7yn`9+8oe)#Q7B)bsuRuIqPQ{qtOp^BM2Y{l4G#c)eb?mjSH%Y>ZbJ zY@7WU+=m#Z-B#fz0Is@JbITSP+>wVBd|dYab}gT3D)j`*l7pUqJt}?oM4|Pb{nq<4 zOTp*dEv)@YgKgf0w?Tf62?a=o4iSX}{1W+KC(tPMVD?Vrw0^;m;#9S3!||5ipQI6Z ze@k&Vp76%3EE03?_w(teTGdPO*m>WQv*`j@Jm@;I$#&DwPp8NKJ6QhZlP7LNb*Gyb zRgRrvH#&cw)#tE6CiG_(PMN)y&4wM-5g|%H9$|>a#{Kr{duVtV@O;b?eC-xpm&bgM z2SUTj=4#?P!5Y=~pYLxu2+x|v_1FVW+1Dg1=qC4l{^Z2gf76F-UQ|gIpKF8$b=OJW z?@1JX_PdSn@^9Zf1a$N)f` z?ckgHNAbTlzeYPXfFu8!r+$YSe3p2<(S0bFTjG7S$9X3T@7}FZmT9_mcsVCLyo47) z2hx^uSM9gXd-O z*VWwf^Yb?SjNk;s@&NsLU);;6A1wU~i-^TnLsaG?wN%FAjMdVfGx`z`{au%qZ+Hrf zuq1HI_BKjFf`VmrXo>7`!g<0M6+W^1Pwi*Qk98P*x~x z;J?4bQ$*WTanCO-*!H_8v)MGER-Zn1BQexK9MHg2(58$r;UQ~ zUMx>I7`#{=%m)_OKzZH&wwJdc(wIItW6KBc*UNz^L%5I5*0r95>!<3a$NkUSQuMX`+W9J@|)SgfGa=O=g!8T z^xho4b)OVDEMXEmhyQO^gliwNK}VaXy~RSVHcHe{Kp8?&*Sofz!$Yw{Qvt zT%%{BcuQ_(c9HuKTQj^+M0^teO2JD#G=cH?a12hZTx%JIx}Uxy+<6#!x=rr~A@uC1 z@>~1XPeeQ;l4bs1MEw0`4al5;4P3c$#XR3s|I0;_!Zr?cHSX*VeC1ZyDs7S0Hzd@#h zlGT3|h;9xr_jFcx?JNQ`5MTIlSDE!yL=L!CaRUfz_%icr;wv!C={v6+7gB2=^mi`T z0-Ik{&t?@myYU+{wrE{=0s6?M&3q@)?Ep)8=uKRKHnNIH5CUZxZl*I47;;Jtq`$C? z78VwTNxPBtqryTu*XBK=5)ktpu8Eh-nF=wx`;BdtB^&Rn(^985o@Spd1|X!Ut8Q?* zU4m0?;%kZjzqYqDJoM%K53yW07o=j@r%;zs9>bAD-mj)!I=0PmJ(94^kt6Wjr8$mW z+I$M?$fZl)hKaq=Y?9Ab=!rWg0cTv84>bTOwnNAMZBa@QWFZc~WAo#Qc=k~|0K!V& zA_=k>O2#~G2T2oln8GT;*bcR`t+RykO|WD>bkzYHuE6EMR#F4&gCccd!2)z4?C2V z7ZpV_^a_UVd|IFXDERp`^w*cu46pzE5Kfh~_3qbRmG2=)L94@Fd5wAPzR0!5_1`*G zb^_O0xNi+O6_roY?DmcW=Z$mZ=iWf|DHX!pdlx4j?2K-`=@0x~e|X7FtOA1M`oZbN z7{3vZ;;L`Do-mm&9pKzIXP^^%p>RLLT!hf?xVQIhk#<)*0Ps41E%%|Kx;~&KccoDz zsAt@xEqeXkg>4VdKO6lSxf(YgnD>wEZ2LgVu%F5DeO7Ind%#xZ*cH%n;`E#dWAr{} z3Ap2n{?O`7g;Aw#?HMv%F9myA96OnKSk!A4EVtx6r($K&f!um;gB@yw{@TFhlv-M5 zoAQQ9NExA_p##en3ZSTIb!+7ppb##0Ds7n$DgXQ&vweMh&*2Equ?IUje-`aKV0~(n zsa|gW3}gLXIl3YIORhE9^c9H5N!WL?t*z7_$S|pzJd_o8f={^R5-^m+#ng{-A3{aU zPR9*Y2ly;{WS@_~Ljs_4A7ep}N5K$Mv1u0fCJD=Q&8_PHumGGPc;>*Xy8s_~BJ$#07|gU>vdN1l2E=c9S&&)81m(#>V1tzp{>;%|R{ z78%m?3qa=ErWIfN>eO`X^vG4HD3opSM;TlPi&#{zP^f$V#B$^4jSRPERQ^XZhbkWw z6toGywBG3SdGi4&$NgS^s3O%+Q)F6N2e8RkA13kd2Ph{s7iKbCU#=Zookr^9Q|r(g z{`9uKb`X%+CiPv~DnENtO;9$$8Nb$MyIyTn%EkN6Q|)x6`prtxQZaGI_-Ys z3$lT=1(~-$V*TdyF8C)pkRwPL@&w-01&qP^VQXqei^QzeAF8qy6XC3dI*C!d3yhx7 z*V%sYl@Y};sm&MjQp^PZ9{0(^da%$K@(`_)S6BYT5+|e*O}TUgn2Tn##ZiHWVG^gr zCck>XutHA6-*}Xiq&ljJeRa37ap>!#;cHtX-i*m%oY5DnxHCCzO1N{F)<$nnyt^Y% z%N5vtSFKO4?wF-nVA*#&lf#UgoiXg^%)gkq1yl5ixddyn#GsNUgDhd9v(^Mj*nrJdnUnid?)*u5sDLq58bRxxrcyEXh`4`a@vuW zq`?ra6G?0n0UsR-eas%V1H`pVIjrSz#QHTY9P;n3SZJV-@X`Af*fZ2kHd zjl<6@DHrL=;_H)wjZp)G7x(rZLz^8ejR+VI*}KRUhf4shl;Jm`eoDMJynrW^XuV}p z%CB2KFRI==cV2pu;)Jt&smG0#kmujI}$hBFp+s zq1@LR=4k(U6&SqS&N{jhUTy=%H(3<`u88@p;^9=!7g>269gveuA27NQ;9GumyiBF2#rGW7*SRZk13{_$?wPIIKTn-=Q zc3F{x3*YM)+YzptLN?}1WK;PFOhexE%$T3$OdF^ReFWLwrYGWe#H39y2Yvto!Bvxy zbfPNGrQ-9QR3i1>S+j;Qk&0=;yX;}+2$hK3h?DI`F@7~;es|oIHjFqXw6mDiC*b$) zzX~VKYRkx7l3bjlT#6sWEgrr)LEx2V<@}>$ftpsBtWaIzw)dT0P4mw7o4fP$SDsq% zG7A~&+HzpoWo||!idt&p6cQiQ=@CH~^?bK{ZivWH@=e3QNqXFdMdhPUF#NQHF0q-IxQbOQ}9Qy74f6i?!A^SSLE{L7=}*iSsfdhY(>t+B6{gy(O{h#HsYpag4C4>OPElhJble7x1&g| zU7l6LLz7ru!Zh26X}+@zsQge*hjU8MUMe? zBGfXs1H{!un_oMB-tO2}8xg9dKhg6dy6!s} zKUi4zNES#~D*SNmD}LOw!)FGf+$rISidK6Az@&551Wq!4{bMkx6n%o~ZvdzI95Zc^ zOd&&ij;{Zz2j7ZEJt`{YI+}*zw~$R_m;KEID*m0=lffrfpQTi{k9I{9jw;a7dOyw* zs102MP*-$C^6Qm-3@6J!lr2OOC%jc67wqw|ufl(M*oEBiL0^1UZj~ER>SnzqnO0Fk z%^)o#U@Ybxyn4{8XOgbx2D?QArS}>=fSynjVO00uy`4xD^k4ImHA1;kw$%JFt*M4Y98@kvW zz|#{D>I7eMIF&P|MATbOouFiyc{BF>U1Ppcf%fx?LYS%5CC6_*>P`;PKK363bw#hQ zcw2E)y4~p{MaC&gpPIPiyYI7KQkehFHsAs3qnrb?SA}fi{MKH{y3#c8d1T9mbhFCW zZD8nr`%_QAVX`{E2dQ>Xm+>RKfR$>pXl+>!Grux#K>zWFfg`(EaV{Jk>J5(9&3zkijh7iI z7B3}^1zY%ZO52CDp;EG}8}>33L@+jROkT=K{Swa;D3m#nz<(VTJ=$RBE41H~iGJ;t zeVEL3iuxv-s{m2t&pKV}!e3cJ4khZARNhkX&s*hvCzZKUYm+nbtLV*hP99=8@j*$s zyKvm~UqfSyT*FSSBu;60CJX50P;6W=x+DqagVHj?f4@ktV)GkmSpm!Y+8kaQ-^Y;9vE8Z7DAfM?z z+?0^z;$4+>RXUvP>$Wf2jY+hhu#_tivJ_RmsH|mn6SJAy70^esMW$&u?J?QhJk1SdbqgP- z4`GI@^z_>zyK9f|`kuxszfX`e;_7)4)Ry>FH&Zq zm^n(H657r9qUSo=_Y${0@wLK~Ley;G-})|*1s%w#TTXQ>+Rb~_tm+%rGK1T6q?L6` z=#3G3hTm0~iNvoP0^|NwM=>G9^&reX7s2%Dmhx z60}Oo+7@5X!@;l}$$yo2m(y-FxBkh^*IB&uyygzZ0UQ2}FG&ZeEWeFsrP4O|5niMm zQZY)t>!N72LRQPPnHYm(no#+=)X-VsYi%^T=ZKNHUSHRyq6lV99mjn-T$+>X8#CwF z4F#6ln)DB|WQ{_*@Oex*$N9gxcfb+0@5c9;Ll~Uw$WB(iGzffjbIbR1`3LS6s+wrLLbs*o(nq1d!Cb9mEA2xj&i3G+r15H_ zfXO|-<#pQMhmI(6UWg4dp(CkCjXP#45HK}Zv7eKClFGxVs+od z_13y?&~F8vt!b};?}-{_>&MmHYO561nRpD&iqNr<<95WO!s`-y*bP}qLNT7KLPCGj zTw$9~CNl;Zw$M$cOAy|2nL{}%(|z<*yHAQSsy0H_k6>H(iMhY-PQ{18yh3iH&{eyj zfUtP|-nq}9Qd2f~|E&)xc)6p&I3;z>_rj`@0DXu){dho>kJz#n)kM-<_WRp&n}7L7 zz&^yUgbAo^WoAf={BdE-87sqGY99O=QE6fHiaYvzWwer=PY(MP@5_;5K61Rns5a|$ z4s6?>=~$~$t!fMnhB<1e42ythT`v0C*s*=r1noo;*W`BSsDI~jyp!;>2c z(Vptc0~Q-g&&wp*QCU>{bXDpac>)s+v5o=_W5IY9+E`+ctzs%ogQB za`PX5N%ZnKVbemFO%OJ3;w9MCHn-v;X-&N!V5c~9_Dm(jyx7t?>0Kjt^iJ3V)`iZd zc(-6X8179Ja!d+qz4uehE-3yTWkOq)<>b=U=G;VuMpoqn{bbegJi4Y20q?0V%BeMH z&8wT0~2KGJt6dzdk*l;&`hXxwj5Ur+Dqb649pY=**cJ; zvi;d?yy}U0QG7#uvE7yB0)d1``RcDw5uN8h+wX^6`7!M|t#l}4nY97KKKQ62@e0bB zu4H2T&zs4?fseQW=U2v`cYAdhyF0%t9 z#@2KE3+ZSc`r<@qM_m|Tb|uXR1=br1%i?RNg2%XRh{-&{$s@-Zd8&EZxq93V7+&;= zWVi5ciVUzfNl#Htd4vdVZI#eS@%@|VLRZ>K4(Ip3vnI=EHw%TWx^u|}feO_J~GGk&X0 z#S_$hF>CqNVm!?Dc>-z!d#x$Rxo*!d=1m+=iW+DzitnX0QS`wj;WAggajr+~G}G4$ z3IQNbc;X4Il8!SiC%-RM`CExuha8it@v;Vc>eZW?a$E=fOy!?#lCVp)PUO;BIQ70~ z=;`P=pmUrE{aj~P0vOtsrX%;WetWATUrJb}wQpyw3dZGPmO})!X{I(vle!+!IF1|r z)Ejt@Fd4cWS`xpqMnw8zTMr+3->2rwVsME2a)=%fIPfNL`^TcuLz4lA;=>k)cvlMN zU9v`BLObDN_#BG}CE2P@s%l2@?I+qF1T}n%BqbWF$Y*})%o{;GpS(h((1))l{r=YN zD!9j4jp~j$n3?*H%&DoRQZld8Cw1b3)Qa{fvWQxyoAxR!hn?2MlJ+vp3dy^fTy)ph zp{Sgrs52cP5=w~HDJkga(XeL<($A`M*ZjS_B&&pC%;C zRO=+7)C7sJ};8h8GX3;&5-3(G& z8_sqn`&p_`3KA1FjFonkom#80@4FWCT}dQ4g2i{`o@2Xl3U^(=fzZ97tU2^Etd^&y z9}}C&5@@Oh^`_XagX+9k;mc1XTr&u_W*erW?DwlDO3a9`84rqiTkCXYG~Ny|IWT$F z<~wyB$!B3*90UsHg+5%xe2u_kJS{a&^27jYu7a;d0bPgoq@Sv!v*x;;|Iu^g?8jZ2 zBUkq|)EhkS>ko&-rPms7$Chh&r~DAtHz#>NtbRQFu2Y_tq(~(j7at!@V%$F6hHaBt zJeM>UYJR|?W1`#a@S0Tk^JryGCgOgv@T{zXZnFzj*P8>v!k+i13P#}F)^8;2pnGs# zT*I!_>Sv@s-88{xA9Ufs7d5Cy|IA8lHrH6R(RVPmf4FzWY+5anF?Q=fj7_I+9ImJ*&Hjw9z$5-3o?u*@XvmkC)%|4AT)(;Io>Lh}@9%B^ zTAfqZ>wwzQ(xsb{?z6+^18lz455Lfty006IzUM3T++H}+zToqHPxfq@2z9f4c3+9%M6UnZc`jPd>e;YX1SX1`J`qf1j&B z;+Hl=f#o}gYN}fvGru#ZJ~6ZPZ~L)M19~OCP2!14oHNN;u0aGt>Q)(?_jv2o$)Kv{ zss}^T9`-ym=gDs4?6q&j@ov!Sa1eg3O`(?LwvVx_>uTn`hRCzJ0+0NrWKa{vG~x|=#0wq>kkukBvTw5GU)YnhBY}7y)Jym zs?{ijo;+BbAej^foPPpz6S^~Xw1P`SX2SbhdO{zbd7v>X2_*Ov4v~dM~fZ1l!D%v}}huBZuin@`wP9w74Loa?R*n3U=c9gr44H_$2t<{OH;-0Qc zr)hMs1RPZQ)Tm`KUJ~0++?!N3S{-m+td9N3F&nONdlXq1xlKeuKGs^Dzy7;lZ?81+ zHaZ9uwpVuUXbx^Xj77}`sG8x7^LLq#ACCMyBQVq%V|xtG)1tlhO}tl-py9?UIgM-FQlAUVs|ny&->sCEo2XjwAiW;nHaH zckV;e5!wc&&{D>A_NXj5Hor9d@ieBeYe$nSRgYde?-lxXC92nUs#BQ6)t zhTo0!A#eF!_81vwPiQNB`DQVh<&YoBSzpNP8BbHl79|<0F88llQHu~3Q>PeO&w6^X zXQP7Tmc&QYRJ~Nqf!i(_*(xG~h_+xojAU*3>6bCjCf|{WXP|#}iuX|G0!{_b18@ZV z0cECj8k1u=X>cV`TR5hvo)aaQoCz)LW4l&`$T z7A6<4>HGn@o;5z~TgJ`+I_h&msnxOjG%aI$!2Aloi{o`}!sqpRMD>fKsujnRo}O;Q z&dDr1iobvVe(h4`IT6%x3?eKUbY0#yn_K_jp+k`=f3YLM&}qhWTWMB21_G;8RUrA| z!)0h#g;OwOLQy&)Aum z@b9EMRgx3zx$GS-bsxX-L&F+Bdc4$U*>d4BQWG%cKfZL6Nln};4MVz3as1vYn}@Vb z118|;Kd@!Z7%bZNfWY3@J9xLe2B)(d$1_@k`}`#eJB+PK?>HsyeJ(ko*pq(NOgmsx zx{`)Eb>xbQ0`1Xn4?(t^jfCx`y< zG2lE`>3zj~N{%nhZC}=V`gXB1PbBn$?GIa$I&}oD0g&R-OTz7cy;&o$N%1J-u#3^E zKBP#_W7_TQO;4|=4Cxzsa^;^OC7X@}#F+Iv^&bDc>F^}_WUSo2jH&IH_Z=bzr5!^3 z^!#!wtPCz{4dvI7YLKNUsyJz4V($|SfL`_mCYQP0jupnl{v#{?3tvbN0V;8TQXM#U|Pi_!Iijf^Tu7Wh*W%wM82&lOT0)`->@^Uou)9H+>51Iqk zC2ws$l@vL+&xxKpTS9_mhu&ni$+EtEsxhe}H~ZSxWjJN8>2O^LAw`ZGhWz~LFQ~Pm z`4u6U-kkBB9>1074}Kf;|71X)E?TNIFHimW{XdFUju`-bu!934VzGPglPS3E?Gr;C zRNDR1h509xXikjfse4vE<~(bE?N*-ja>Nn-Fdx8y4VI8zazzqqIetDp|LT)l?1Kjn zOtqmW4E_DfmoJwnJnV~B-H(mGK~;>kpY}{w@NwCH{_d_K6=)A}gW86K`p-%c%sp;h z^B~_eX20K!Fj8Wv<>yO9da@JSLeIjz0)Y7XOXiv6kRq+({Gca#h}+T8aZ+CN9|qgX zTlXnLh$21lE!vMS)7@(+Q_l0-@X<3Oe^(NLwwPnkA*VGX`_HbI{Ye{q`;pCJ)801y z!tllRGujLO3_~YlX9M=RzS0E`$`bpf4^BK^+qa&x27p-y_h&i<%5tur+HTKtp}0f5 zhRSTI_3RmJh@^iyoA9sPvx9h?>vX-L2Ql}K>MHt}RUC`e-+-#LD-EyxW@5e4uhh!D znHS@HJoF!tBHKMoF7qRNu0pef@+7?lcM1ER(lbz3e(BlH!Tr)(S>kU$pm=q?qI4L& zlHiYBQ0+Zm0e@9C!LoOM6_aD%6B7|Zt>ly)6FUUAY!_DpipnPdSy`wWrgbsysV-6o z3Hc9W^zKN=T@XLjj+11x@JhHS@p=2{S0oVqFh}3o2Cm}z0R1hYdyQ~Vyi|DA*x0ze z9DGgyMajFe{=u;M^OZF5i&1G}wpywI`ImN8PX3EM^7g77yos9R|i$ogFZDtp<5Qb*+U{|`$VrHaEu zL9<7uC|$88FmUFscQr49@BT#f8U`u+mJ2M8r7v=a0K|09yS6sxa+q4S=l7t)3thCj zt`*Rr!>4?*ICY5zEq_Bj4?S>KE`Cp5O2Ssqq0e-Q7D9YG%TL%y3+%NiMgM!Q7B>}0 zap5>odMUzE09}vXNEO{*C-)y`8*H1)r|^ui41yt}RvckDj%;a(#l;3L9RP$jv79+| z%BEf_oQ`=iC%k}{3ulCaHt4-U(3dORg3h}nPfhZ_4<8QSa@g(cu@MMD@-ir{228^> zTXy1KJ%2uMG8RtX@+N_y>)Xv*$8aGazYhI(gbDD6r6-u=r?uL)1}3}d2c9|!N;UV8d?52!tIZB13V zjw!=#M9#6#pmp<%-zhmIW%i94qTG1*9U4=qo;pxwZ@4&$2pS+g!wR91UGNZBJPSPJ zWB2!G{P^HXU3>1%@6krS@J(mv=U)P0Sohn|2KwZ$4GI^tFU1g^*xmE!^o)$Bq9NU1 zoB<5=ATKZP{O9?9npe~C@(!2Kq40dFbV)38-0;L65=^j*r2pFQq0L?UjneGuV0y#= z1N%aNA}6T7IQCZM>~CiKOlttDCdJO52HZAqKhxnlItQ-1$%`0q$=cq(^AkZ|FW%C) zdiBM=ep&ZPQXzm*#952PtZIT6WmKi$H+HvGH8gI3AO?pb(-ZsJP%3H>fq(J+4Dxl^ z^lm5rw~jXxk<0B9q&?Jtd81k_eeI{01V|7so+(U(&*EWl7t}du=n*c$N96oX2Z_m< zs$&HwcYBKnB+gV@+V0PMfq>ZjcgnTvH&Z@m)?KIBoAJva&Gp}YBO;#R_>ky}^1&)s z(hcO;?HqbUlaJt=y_@{tkdZdehu+RvHP8dx*Uuzwr{jI}%wss_{CGjz+elD4s@lc9GqP~r!iD3j!U=%DwLYmM!NQ9x z@VB(wVk!mcobpiqzsRxMgKekNVQrT&Y!E2Z(EpB3P?&-CzBs}Nu{i&ch`0_6Vijtv zr4cV*J;@^-@T?p5vN1Dfa}e%6W?tSsUiVc$n^1ALNInt4{Z;q@ za8*xygm8*CzdSYHZaf&WA9oO94cz<|f$Vcej(h_mZ%1i2nW>!!gr;VCNm3!=L47 zjhA1?uH>@rIM9yhG=j#I`tg1*BywNsmbe741ZmJRVzWZDJeZ7(jC!i;uD*g0;O;Kt z&|klNSX_~HB~{?qv3*`|dwr2rYhi>{`@vQ0Ur8de8%VAk(bu(g}0Qe684L6D)9G%rO=s#3pqO ziV1>sUPZ;==3|J?`fNiRUj57OieUA7q%2>{8=~LidCiSyrF%`S^X$)OM>-E|hX&0Q z|GAWILAA$N*fNjQ_Q=aR9oE8&ovn)*22*`{A-Hs@AtI8V-@96M1Z1#qj;MgGr5AOG zMDi6C{4qQq^?v<7*NDBry>iIQ=_J6)98{Zo_M7k}PC)J5&6mAtWeZQ*Bc}GA=!v`& z<29qxr9AT^&-&8wRh|#S`FHg$>L!072wP-Q!>U-^(Aoy>*}3!QZDkUx;@tKf6Mfxu z3iJy=^9J+_Xy}&)dq6@-P#;t&h`LO@Cde(nFu*TI>Kz%)Mq~LVpDq%01FxQno{&{b!`LCNq4$xrq$DUQ!GYs0bNcMr^vfSky~8J+JMmf< zJaZv>rAR#$+{Rmuv_HSLUe%#@5c(zj`%mYwZZA3$TJ&%7jgJL(_a>*q#{sFWN36)o z?LWX7>b5VXFis(2Z?l4W+PcE*LqYhhB~J+iLy#$y0hYdzhMw)P&|kFU7m&KacEqdV zARm#G{ZiKuWpanf(rWq$K4OlsMN2o?;rhV2bLR?u8sO}n6rNm@1=^Ns+$)x2P@=ew zf>TNqPOC^Ma<>Q>%iGeiGw_HC{K7VP>bkM1{JuZ8RE>?3IJvh$LQ|vGt}9z<22i&0 z=R^84Ym%ubs;E&= zP%ybI>M|@VsB@5k;inhRM11_*AC>updeQfVUypKrU$ScFejMV9ii*mk)l`Br85iOWKEAsbLkFnm{R4`0YF!bAU#PDcDM!}m zSF1{&&TF`SGq$7{VG_qqJTl5}2VU<3*=5yEXQ^}OP>s7FkgEvXl~U>{_DE5qlie?D z8*Y5CeX)x+eUAh!N-nPj4bcOBUk1w_GD}?2&py{(?b!P5EpW?#I?@m6;xkR+d>&ek zTh{DG{?Dp#CNYSvoUKhN;LW*k+QH=4U5ab^i$36QC7`hJY+3x@cHV*fIyVhq(RLkx zq^ag~Kiy3RlAOFo<`Lmcpb8#1=p!ZVi_R0$1;MBxw^&32w+6&r2i^8Pn?RyL5-|mh zQPJeN+N1ztDgUti0Jq&k$r~&hlJ1S1CI&D_ui;f|K0*L790as_4k%A$Bqk;XCz_Pq zAQ7YTNXY5QjoYa@Yi7p$y`-4fq4WZnuH$wnRvDM)UHtab&GG;j4TBAZ)E{Gm6?rB1 zq4a_^m9dMa>$X7(F0`xUm-q*@2SUk{I(rmGQ2i^9K*j^VNFs23rt3*`+eg;hJ&>m} zC?4*((Zox98vIi5B8UVo=77fh2iJfQyJ zqt>p2rvbw>t1VCmzf6LJs!mwft({*X&XQHZyKrB>fRIqTNZ?0dNY~sd(`t}>!e!7xDcO%?7CH6tZ3GWRMA=l z&+agC65((lF&lU=+MHbpJ5a5|BnhZ>Vm!3tHR_xq@CZj=Wv}Yf-2~?#b|rAC<)l6E zET4~F*tMbvf*lHsK7U<;}2b5O^lA7+*aL|GNr(w#fW) z!ILMp?7dj^33jIq$N|B4fW?3~OA?Wc4i(n@E1`yX>3rkAlH?3^cw>akl4pQm(jyf{ zegCnRjGFVdVZ}(n;NCHG&t3!R9;bv-W8lScx!JooI7I9l1{dbMgsI}$O)fwQR`6`0 zD3B22uDZ_mKE4F32kWmzKX<7Scfcc`KqKE0!x=)1B9I-3TD}&k%q$2kW;ewl(xC5% zTyQ~Xh6@FX4SAyT^Z6K32Hf1?)?0tbnsKL4YV^E1HjzZNQ+fznuFR$ZpywL$y7PCC zn3adW1ksXPk*eZd`O8yMWkTx8Q;!M^!7vi|L5JH0k?@;Ru9gNJU+|iT4rvAnQ6Yxdr$|{f zK$L0+I-?B}qPZ-a_E*Ht8c~h>r_dQuvxE6=kz6BzI)E*Pxv*_C1fVv2lts2iFy{BI z8o*|EK*7;gQI_Bbnc?Iv2qzbTuS;q?j)T96sEJ1Ovg0LC%NP%!6N!Ng_dhS@i{Rr$ z5d`w!3n!Ds10bF&4d`dwuDTh93JfWiN$cvap7JF2yHPTbWJ$n(3;oT*=9m`Ad;%|FBq zV19=FlkA=k(*%)y_?R9Du;m-6Ak*l&_yE$MBYO?!F;@|4k0E@OmARTnX2|ur*-=4w zFeyyG1DMoRdw}Qea!&reRWBAik=ZS*Yv?!>Dr7z!KTp z&(w%HpeHbZ=_fwEFJaJ2^g#@)tjFT^u7pzp*U1YPT*HttYmGmv-6quQk7(^a=D(<2 z>;+H*fg3>t75q?ylJda`PwnDVi~e&~u9*`5>6h-l?;1J|M3=*`+gH6oyfh8!$@Xu= z{|aNkx*BomS)BpKJ4hEKsS0aG{wpMwFA*L98`PTh-|8;niKq8lPXbTkWg^H{u#h7m zt6Mx%fFxlg;tg|YXJ!KvzkC1X8KSRsl8 zsoh7kD<+E1!Bf6QGTN>0b1ulfU*QdLjwS%*1)^+FD=FPx`xZ^CP00E8D8mHVT{*>J zd^@?DQy)}JXI^q5Sub2h2QI@HGUu`DK2Rafkpwo)n@pu3h#^b&PiS3DH6M{WjZk&K z#*gi_1`X=g%~F$M%d@FWioR|RzY60Z_d(~tW*)zl>kli95Mf%v1!@_=pSm!{>4JGU zcxtz3nkMZgnZMP7$@LgS`0S+`=m3<~k*fxNizzz-J+r12*2Up0@-P%6q#RFEpZhCjff!}*>=rnNeRblk&1QiASTg^wNq`I+bBze3%2IQ6+F#3ty}P`4t13G%CQ!V#gB-Gc{;=yiKi z1`lM!+0AYgO1kW;jZ0ai!=siF(NZm{+@HG}AeUHT+q}Qbt~Js00yQ0zl@i~I|KE!r zMqa93;L*O_m$F99MG+ujyaa_8_ovKq9`_FMxc&L3Nr1jp1J1JpX)!s;9v8v$>4YBH zJwpFqwmVlm6$;P*h>08a)qL!Svy$iezq%7TqXv9U`}_NPwpk$kP+CU|E|HvIAiw&r$;@GAp8}M;{E9K&)#hQap}|%(!DgCdl4JXK+!&% z3b6>Cwd+S-hzW>@Oa|HL6zlF~qS5kzX8ulJ)YILK6HHrMkTcsO@{jnW>Wl5|?M+2m zIDNHHR}s`-UmTRE$~xNx<{t=^KeFpht$n+zw2pnwubV0BUiY`v7M3IfxzWEeizH@6 zsRZWpn%dvo$9gF&5~x$8J;DC)?fyT%B5s7tDS`?}QIYUjymlR@7SJS*dftA5)DGa; zRdGm$Ph?6%`5KoCfnE6_@*AuRsiCC=X^xs;}Gs7AHdOhkw7=A~2)0k}< zz&DnS;Xdx}P_FR+)PD(jVHQ+$rFUfvDl3O-K!m&A=&>&j)K`abb%ef^Lv=DfOT(8} z(x+uj8~Cs2dihY#ZR;atk|T*ncb_MaL}6qDrb3MzzHD;u!=2Va8d&@38Dn@@`1kDu zg{%CskCvZbg&breYa^iKv4b!_hCSXLqh1|l<~O`H#X8nv_7->=CK=Fg&Ok)f%og~m zR5k}}XMQde%Yf0B`s+CUspz`axvR%W$F$tJ2td^*7t3HqA``bq64Apqc9;jsPYRD^ zN7cTpUG2B9N4SV)SAPbOk*(Nm_2da`qaI>STo>FxW@;nu?kpnV4TBb=sL_7`_DFTn zRHq2!eY_7_2LUH%*+VYq?F8s%hVANyT7t%F20}f{mj4E~e>V&gk}v0wN^b3Wo*|^t zlRh~)Sr|o#;rCNV=-$eI<_uZMA;6zb2mSJ9H^oJeu@qcGfZ7P3hT4_J8OE(53az<_ z(Cg(+x!FvnaG>sNY?6ya0w-u^GRA&C_@lJ*W{Mp|0>e@|kX>%C@@sHl*-z$2hii~+ z7{0CjB`)b}sqv3AMAE8E@eEQB!%W|%5H7FMX9vmLzWtqd`OR%12=Kd-aTLib;owjS zX<#Ek=?M}4-?(01*q-0JCL9;XH31--GRRfO5j9_zv3sgw0HWE zM!*G*|EQ<&M?*e7F=?*&2}3VOp-{m8ap>Chtf5x^-crLH<M`@(su%+WZCG4qx%rDmPj< zZ$X+(1R?xTQy$AGxdqFAO;H6D^uaH+Ch{V6kWyW_dHR&|!$SYU1Fr5fNNXx@1Zho8 zd4H-rmz`7OLwmK31v_Q6r98OBJ3&VlQr!MNHB<1U=viGoP$Z3wA=0W>!Z{)2zySZ_ z2uZ}J5uq;vOy4s@KfO^Y8%;h+6d+oT--jb=zXk?7qt1NlF>UF+!oqeM*5@fNxm~>M zuEYEr=TXe3tsh<2NBw2Hm*Nm~XAse?1gjoFZSzs8-`utlYY$Y6s)w1^HU}Tvzu#DB zUh&@0Sp7N- zqHCZ-&pI2@Jj30Zc*7-v##aFmE{wYxw%i#l@@9aBRyYSey3aLWXvle+YCF6XXWz3+ zA`0Aiav?U75OENCEYJIgeWzk)5IM*npbmSbx&=>fDmY;@u6|_s?M~1d@IPhvbnl&F z{2Xsezy}BJgShL+J}S#YmOT_|Q=zoNCD-&A(m!MiKx2YZu&^cq#~2fMkZjW}?2~?T zjg%W`Kb`FRed1;Z#C+|yuK%|69K4uvzt`L1;6B8{Yr^#CneB{QM_Kk&{d=vO6e5}! zLKRVVinOna85Z>4CBga&8NOM zT!50bN+Pqu*Z?QR;X6*LK{%6!kwsM1paS~TzlDcY*6pqU()WYtI)oxJf1vssOQi`d zqww?Dr}Q~dsuRx~>~pb4cS%enqB>fK=J=z#Wgt(F?o&u#)?)Y)XyTt_p15ZMmMwo@ z8F6;cakeo;jF&1fAUx$epbK2JKrScLZo$5U5}1F|~ag8YIuF+UF-2IqWlK@5-Z#GZe+j}|)0sva1TBOEDV&Wq=E zllmNDuz1uZE%3)~xe5vHjnTLUIraBuC+Uw&aRxV2axx&SVZ*w5^{kUoNXW!S_Z9?p zpO(kBV{Y5!0&gwk6$K{&QLX*Dns1rUZeVx!Sk^~P)Cq&`8U@ZX^<`lz{#EpZuW<@X zC6)weF&7^)$OO2(>`3(1K7+*-4*E8w>1Nddfa>_f#}O%8OlNkBXe}Y%N?u3VPJnlI+P(Z@XIe5^wBUR68 zVohY1)B{)KZr>n_2S)t!i;wBYeMQy9VW^5%B2n!Tn|9i2zFTZ>+Pv=H5NsXm_H9pP zdgnmcmaxMI5jh{9ZA8vAk_*pbs-JW-Qd4|lFNmm2b003XB=`n%AI1H2dw1+rs&#$% zAnrf|jlm09ph3zX;%(tqd#{J4NYU|x=E{t5&d=lxVV^wG(X02lb!Ch9k`8`TbGs+7 zPzFNKt}{bp+b<8veRp;3j3F5!vra0p>Opi(Dm}8gx9OE|g1BSAs2pzP&m7Gq4_9QY z1r;9=%q9daKmWAbDpez@{uWNFTvhf&^bRwARNcGBgRwZd8v;X5MJG7x>bDj-TXz-E z{Ha+_t|ClJe*e_}hC$Ee_!RbqYW#TfvG-<*p6h&;TudAcCg$#8emlR$TzAI8T;1p% zNyN1`WYlG~)HW?XeKqPROI6Ps$Wx5M3gYF>9W?Ec7wUe{%n{HY#YHytjkU{+IJQK> zZOQQKcF({lwX8nl>GygfCs;x!m;8WW}gKf@moD?S`Vt+^VO92*>>_>@M4!obtIAZqo5r*a(`<2JSEe%f65Y?U^XNw{?Yy zXlQ)p%2SKfq_J9dr?i1vPy`r6r;&9ZI4zx|sA5?0aPAT}GI{n4yFt>=bNQ(oQLX=< zrnuUb-ROG&SGGRdT>mQdf*l0|&CFND>lyw?B1 zbNhbtwVfe_10@v&@;f;Pbu<#=VYN|cq}nE^nOuN*WLgvcq%zOxyhUko`7fXT7bZeZ z%N7kwMFL!0MA}COLvXrWyz2}Ya{PAJ?DxzRh5B{{pLuA*&nkBPQXQ?>%Jx*AZLeDU zm3_g_cAO;x{WB{#kMy=Uekv&#t*b_1a;gDiE>jkJ?{q`JCmtdtR_E>(8k18$!f`;_ z<2UZSYd|u^^-7GI?@I07n?c9laI5dTLWr-iq9-;)K6?$(U9<{W<;ACiyytjluMr$V zDO!_!%O-@i8_^@5(}PVy^&WcyM9C%hal&a@F`I{3-bX1Z9||D1*}<%&>8q<(ebnL0 z40?v8{N>tPeHdTYfyl5&os5H#ABr=UCC61!H}r^+FL^G1UQ2XzgETJs^IXQ2C_)Ee z?dNN>^7q;U3WYbN|HA@=Bl6@SO!dxwj}RazjKs)2a6If;>|#}>qPhRv{bLD@@!YlD zh?+yq;TjjIHlk5LTB3G`WZQA0oixJ9K`l8R!hZa#mhWlJZn1RTJQkCZp)4OI+s>Tf zSI}pUeUhWO$km$AB5CoQZm&KccgEm z@&s2OKC!bE!H=y&51-0EeW}Moe_iFWwm`@>uj(7Z{!b+eIQNXp@J8L=K3QY$>DTv) z@(H>-CZDTIZN7ig5%pME;^BecrCd)$2CtpJoMP^A-1pOmYz9QEqB&7N%cYz*8&rcr zSh^x8f7f;lwaEXz$FFpxuY#{qLW{3y_5cVB2n@%!GM7%l_yNDh!>fpdRg>+hSvjen zd}SNIAS`hMNvEa@Yhy_U@{es~k0;-4Kgh2jd0KzoZLR)hgC?Ve-RJPC=z+!ss8uu9 zOX|b7odo9NrT$^eV@U#@X)Jq1W>N+E_BO(L_izAyQ_RakF+VP+Ki-U#`QhmDo7TMrt7WsVgY3~F^; z@~z<=!ZAMD`JMbwk=N~Y9Hn05182QF^yFUN2aeTRtp)aqQ_RO!Hy?E3%0Zx>!}yL9 zh+@P{uyh=Km&Kf5m=KJHeSjISZrwVBj`W_)f2W&NfXC3YDc*0INU#Yo^C7PZbB92h z(a?2DUGAH-W@Od9Dq+SDIAGZ!5U5b!TEubTCMRqcR+os9NVC(X*;aPi|gRPSKF4R>*1xmaI3SXZy@p-h1 zHIrYx*?78YNw1~b!R80*#65EL0yqJ0DCNSO=?ie^W z8BfMYbYeuV&FSUKKx{G??t@;t*VU}F<|lrQpIBWolovd3({Bqed}P7QzoC(+k9)#I^u z6d9*0$9X4};!Av!>?ox%Dr(`F{^fr-Amq=J$yZbOj+Rh)i z|LjmS&k1k6Fb~nXxNiz^U9^*5aRQtBy7z-y9*HL33}iURPmlJU2M{{GD5P1>(T<F5(W|b}I)o z>dc%9>0`kD6XGB)q9c4(%jKNrY3a!bozubF^A1LNS08WJwu{Gfq)CMMUFRrRp+fQfnim*AOnJcY??c6h#sr zaf>I(x^v=*I}Rtj2C0EY;^D>5V}*#Cj6LxsgP3(e+(k7d{=2L%unr4tqJ zTo2CWGa^nT)1az$us`wq>P6kJoNoBtq0e->YXo?jYG@8HP%BelYt3KeqXyMLKA>SL zl+wzNB}1Ur)O z1Mvowj7Ya=*@wHPce-ORVQ-kU+-_qBu$^ck90%RgkOq|vp3iqNIv&6QJ8#}vOIGh= z@J|_%I6bUXV-ce#eHmmLRS2!W6D?^=_>4k>W^Glw7u-IeFdkP8YOHt&VfXT*FipNo zBO~sbgsJz`p>6g4Q_nvs`9Ub~fB)2a9t?b`8~eI7=j#5|dcW)T~Mq!)w{v^k~10 zr!J4;n}i)IPiic}CK7F$HSsFf*!wH8w@G781l5yEx26t!ktN2@K^KN%Qp5v+_D)EF z$d5)FrUaw*V@_~c;$jvQ@y3M37e86z+ITy#m63*l8>vP(9W9U#h!V@^;!J8v9^Ls7 zbi9Tb>P(OwJVhtoy~h*rfG8hT#Dqf^rfHD(Ibvp1!>J6Oy*s0tsiezg4Xg&;=QC^1 zE2mn71g=Y|>=uuj)AOPE_S$Fz>E6nhrZy!V{(RX%6Z`ksvD=H&e7%FJ&PMG@biUQ` z;%DalfV^GH=ypRUrmP3(HX3pF?o^ljc{@s1zxJigT~c!97afoHjo<7Kazqf%<0f$o z<~OD+NcqPe9ly<`j@eVT+?8z~-zM(?UOATq#mM(sb;MChlhY)t)q>(EGZe|YNklu=J-b)DCiAN>^{f?q(wgAj=OQJO3r+YnrY6y= zCaGYOQB)~mGkGB)Zp#Wp+v6jq);HIufMP~V!X1PF{ydCS0$4jNo}8A7QNF#Pt5TFY zuzK7(H%zxe&yW68i#Si3Dn@oMztkkv0HM8q#D?@ka2pbTN*nV;lIZd~M-hXXChqXq zq4n=KSF68I7nPzz;>T%srna-S2~!{iWcWEC8VN5mQ8}@;eEXl zAsF(8``-mJ1@5(d014TxAz}rhWBTFuN7qx6cW2Ev1qxD zXpF9io~{mMW8=Q}P#MW`-=txD!;Ma~Yt7EDd|461oiITbpoI@@tC>D^KpkH=fo6C( zScQCxI(&=gm+GMlZ5>30kq~4TbGPv>;bOQ7SFV@Rsdl@zpI>v*d6%>eHf51ZsaUUEtLc$AVVglv$vuhq{FO}6JuT6(F{0K zm{M5NMHVA~A1G<~K+dmOL09BUH8|Bexud8fuql!HVsBsNkAtA6A_WFWF;}-d&6g5c zl@K^wwoA5DEq*8io`Qj$f!*tRo_PqyK>i{fjWj*~W*z%p?OhbcDMAJK3V5p_bt2!< z9!~j>&iVFM({*tB?h?N~2H*SHSF%L*YY=R8m_Atg#Jch;U`o zdGGSqs;stYjXN4MjVX(kN; zniCo4&bPi$v+c^A7ld`W$}km{Izf+w8Yd&eXygP}V8+88kz8pw=sb>HdEH)F$N+oK7F2JjtzFqAo*sv;N64Y^^uT4?3B5UT>K${{3?fMLpwo! zE;6FAgGOFsH45--n}3JA5`S;J7fa0P;ir5b-%klcz+hk+S~QP=pARG2ybBim_v__~ zX=g)!^JD7I^sw(vq4u>1QK=DM3)w7l8pUhlZPbcf1 z!gYevnLKGd6=FV4IEgv2aW9V$(kCa+H^1vFaKJP8Ehp^$H5fNQXhaBIBu zB=#iv#*ab}O8vaZrm9x1>I(iyUH%7cIA5fQCA#vv|3XQGrb*dnsDmYX&LwO6%ug!; z)_YL~{yKqYD+D5aPI{HNhNaRMXP;|4<3vTP4y=?5O6bUcZ3IjGog2n;t~$%t<9>+; zW{Kl&Cw-Q00}?^W>A~C<$ONQ=MdVdmkTHq_-JZ$ZOEvQfjC`bi zzFC#qfg;BlIB!HXy)SK-N|Ph0sOTVi(K)%|z3 zU!!2&d@d-lD7%YTJ+$4jU zIWi;D1|t+F9qNodTx!F(o*}a z&vZ>k8$PtF;-7Zn6VHV=kb2u@r9ohkc!DD|GZ)E9o|0zu9#XZ1xf<9>B6_1|WY>E7 z^Dmg}o5&oTD!Uo52-#|mG|`)KpL=FO;NBBrO`lpr4pAcVi~R7D^kh@`HHqGdSr%19 zAyxgtgl!~S{)0}Sa3=Ns5?}P}e}4nJRB)bMW@o~yqjuCv{^IWc?!2zdxPr9SV3U6{T{2+dV?nSsWD&Xw1Z+;{8esSQZNg zZC+R+4rGZeQg3T}1HvX85^c7Nb>|mSUDC2f^>b5U(&cnNIl>tX&<)0bQa0i6lKcPe zNh|9FUtY3jf)Xj4zkw52kOCf+U-o%zRsj&4!YMa79_T6#OcAZxKzfIJ*_ln8{ zlXD)K!ux*m!R*Sowt!<5EPfaA$5fY>K%j2xpnXcD>cTHCI7|V}Dl~?~7^m#kN2XS1 zL_sn=hGn7u3B$=~P)(3NZcX`~CumfC^1*8uoIXs8dHa`oX?KWAC>ZL$`-~4^Y2{a5 z0`e-fnXQH-wKW&n4unVE?C|rl;*bZZQ&QCfohAeBG@8-#INw?n?-c`hYgqjsXa-Dd z@BGhg(a4h^!`+{w9JI#29LGDP4X@dEl3Ag!mwikk{iWbU%LUmz%9w*1QtcF-ThHqeX);+CFDy5@`D1&9I_^+It$LbaW# zvPgnGRtJVp(pYAoNBol$xY|Qewk##Ucg;7Skgn+OBg^S+n-+t)8N9>;CO(7*=HFA_ zTp|yu#oD$r%?E#2_WuoQbFd*7bX#^qO=>F|nK7b8AGqdT83t>l@xJP>3#Vk#QnlM$ z9RB^_%8Y=Z9s#$W^%@L725cpdYvFH`)Dtg!kl~pCra*Ns8L-P80BNO^7a$W22?X;# zocLlU9lqr?N^m{;0g6xAI78f4t)ETe&ZNS-`@!%KGR}Q87`G`nN>qoK9`XwF+OvL_ z6xLcv)_*{+1O<{Z+v*kR*^qzFjyQ_+PH(`feiQHn;XPDfwL2_*>?EYS>OLUTMWvm? zO+-#51?Qm6>c`5;8bKw!)iZxOIa<8xlY#srvd{~URG8ph-t-lSp_WrWQHgHiSxeDz z2bOT7xUa62Z{It((zXFM3@;N@w5ff8QRqk;>cm8+ZE(FAlN~*%Dlbcz=F9_hwYV4(DyRf1et}W!c zzuo#`l>Qa$T_0J;8U5Ta1GNodD;nH|3APu12txZ|df zS7z3_-R4ImbQ$@-RNQ2_!=fDckdgm(fBJC0J>m!T!jC-V!L$ZwpD4io*RA>fUJP|Y@25E|5qfB?4))bfRGBpfs(zA{UOyXCR92ek5e*w#DLeX!+lgx|GSF7@< zF8CdF|A5<~D(0jwb~i6Yv!&k8RU;c@g{fJjI)&V72BeJA)8+2Fv>2BB+3kJY#$0tB z-{%2~8=j^Af}{m;@}LyMj8J2D+>3dsZIR`w{JltU*zm=)8SfQ4;zRmQdG1?vq+Otp z1b4^ESdIRr=R3xx|7(L|v~!_l2f!GvyN`qR9y@)$Jq|#U`L$`AM!gMLRPhNIs>2590D+|1BuklJ|6r4vfK_brL_>-$hB8Risz7I zVf5%>hw`ENE^yy6OzwSnpful{b9}fk4fvct_SstvJ&K|T=lu44*fo-w9@&!ZxLefH zpa_$r%`u4Uz95(Ox75^i6Uc@m5uMb#iq*em2Yn58p(M-lB5re*S>j6$U@#y}Xb=@T z1YLCN>GuQP>04or7J`lh(tMt=K|D9T^rDg`inhdkvPl+UbEZJ#gh{4lOR5lB-NcB3 zM5Wi$fzzSjUcRZ&78~?i_5Z<=4HU==e%G^Ozb^F80&zsYm#=okuj@JB+uUI1fU^X7 zaN1Ac@m9~!UmKay$CyaDy#-r7Y%z(1Q;ex9khs!C>aeT9!?9IIcmV~jq|bXWmma%N zm*7!pcM?%c;cleVkX4LsY<>9St8CklYsc(g$0F0>cUWB5}9 z+YWwdiuoC__5jq`eyag0?SG93Z$PQJ1LCpK?a3)3ghHG6rwanb@t*VbW&c>Nbc%0d zXb15)gi*Um^R#Rg$tBPMz!gJ}c+{r*n|bkI(k|YfJB0)_=0I_W;dG`Tud($3@M^;O{UIs=GB`D2*ZX%bxrnB zSVrM-*E#z+`!2fZ5?U#jT8gvaQCR5gqwM#ecLcQ#g5W68oH4^OH6g$ z`Wzv%s!$5J#K=xGinyGKAJctRt%U#_gcS^Q{G!F%N*)S~E&?mF3*)_Ac1;~hfzcIm zb75D!na0l~^J!Yp&%Lpk%W3(1-#e&qwAh@``c*(<5%N6ln|eCs0GL<|-ff*MHu<)N z)SEzlR(F%xdGjAIl7{{R|1~n=KUEz2C{pXm@uo!azJ1#xS8?z@=v+R0$k&ibLT==%h7TRjj#K~eQ&;N3oA8) zG3zkd1t5T&q9n+u%8C?z3Q-F?k6gf=xxh^}&4x9=htsi#O2_~P z`|EF5yNI6<3vYOSb1tHGEv|j7g{PrwPG;^_in!5dJN$<^IrT)rsXMgof$72#QvlJK zO*hROkqi}bqsP&`qEXtNiqJLH_KU~N&IzgN(AZMpHExC1X#v1Z3MV?!r#-weaA{+aK$B!g&?lO0LlurMhWv1)`n{;c-Z?AdC@fU2DG_?+{Nmo7 zSBl$Td75{8X{=b$_shnSn_&%dOFQdUopRY*043#(xG3ool{NQD*PdG;W`~MP1j+eW z^^Ykv(hP?+v7klK3c%CFEOC2s$2TKi!|JM)09g9V!L)X$Xg8i{KI?oP7w2VKIClT4 zXcE*Q!oEXX9)szR!S0y8T{0PN)uC2TWw;vmo?u8mdvesTN#J7%kH8~{?t1J+PsXvu zq2hF@&Z{Y*qdQf4YfSUN;IXYcN z*|%D}y73IQ^U+15-NyG{IIi8pBxs|vnH~lO^Kg?3G*$Y4U!JiC8|r*-V@4%?`VI@% z=_nzrLBl@nsQFDEgZe=T?DDj0JmRQr^Rpz|b#8q?^xGruxr>W!FxfOfR8p&+uTkUr z>fzpopbh=)ROZ23)>Vlv;dz@KaL1Mjs#cu-lDSHlI~(>7SYSM}sT-vpjA(Z7y`KBX z%eZljK8b5cdOrh#26rD~vJKFtQ32>^yw~)tzg39=BFzb&`02#8_HPa+LpwuU8zUpq zgPnA66bFq)P|qC%AgDEmn6{4X>hmhcgSmj-gU5rno5T|S7YiVYmr$x-%bcnSf2-kf zTB&8@qGq%Lp-BruqSzCE9HQuvn6%5wvu#d&^ounTnYb#l#zQ2LgZf6#mu_@6EI#(4 z6-DOeM{!PZJvt^CMa`zxmwNGoPE@1{D4pKO1(^;m~|>aZmBp%d*nP$_HK6v z9{*SZO0Wbj6F_T%;1(uy__x92T6k;rr^fR_0j-t5jyS zuIWe)E2FJ~F-h*G6P3*OPC1<9YT_~b;abtiA9Lw<8T}5=*OyxOmqTt$;87iXi-{J=s$-&DZb$R57RNwD?zgvUfot z^&NoOpY@MbjFx7MP_mz8S16z(rIB0AjoI`lIWoOUIflb5FDL&@%gi5$z-07!iC5nUrQ_bQ>6Etiyo@v>blN253JLc2g}ECG4IU)>TRUD$0_a8N(G9MyL?C%$@uM zH}g5;tT0}AkNNP{k1Na)rg8T3PbE3`*rvIoWboce7wmbz6c5X{m;-Gl70zIHk0$29 z<+n>LD~9u#=cFyZbJ!6RQ#W!jpV{v4wX#{{;a{g}KZm1D$~al^&OT;`_&8Bewc+#u zL7Y$H{pYX71ToG2UJ6wIlfjUs9bcZnvC&jPqQ{@F)@GQ4q`C#pW7;`Y&FJYbG5_*> zjTduG5bR^qP6W-ZCA4OVkAvjAdi$3aWM4mUXMFg0r!3O?9R($I?_Zy zU}y>26W^SCQL8~d`2c6{aBykz`p7MV^$$~T9I_{Pov*}Ts2}%GLg;;){7M43DtDva zJ>}?}wFI4?AD#w0AKgiS_;1y0JQdO)mWXYi^WogHmuglp(pewJir}x?(9H}4=VdW# z33+_RE;jf`WB6pI753&*XPw2gj9he~|Fsnh)`I-6IPUI+!3;yZ~k_qTuYp2e}hvPIppC#{8CAS}(A1$sWk#E%U zRL|6j$3C2Y4Bl=TefH^z)Rc$M7mV0KEl*v$$P-sICOM<@h3m}g8|=~YgN2E1EWaf6 z>`VCc*39L)7LIKn_q}KL`D$Cmt+)T+b9Hv74q zxZTCO#L+CWlVn&9p~AfDyIOMLJ-d2)Cvfb0JmQ^#!n*p=qAXjFW_mJ!@Ho&v>SzcQNoL)-|x|=&-ZnrPNj$%qe!)YYi&t zxyc0nlqA1Oxe|AKR4|-xYCAEh^Yvd=HD7TZiyJ%|Y^bvu!f~bwuINh5OCdtO{B(Ze z{RhtRAI;#z;Z`soJ9i@)_{2A%`C$>u9pZSBT=@o5T;SO(8*VnK?Ojxh5`H3(W7eS0~ z*u8E>yCCt~ZOv%6tNIEeYa98>O=od~=X;ti8*z&XRYEQ;f=`8*zlmp8J)4;$DvJIg zQPrtP*wc5v^d&>Skw-h%#3+=0mMxeop8knDRulUk#ODnUB}o}b$-8Ygl;%D^{d703 z@LE9rw}7XgZfeBO*@hTt>JUfJQWvD^abVvIX@d`GE`j?++3^#4H6HF(9`9G?&7~c( z&yw>*ZLVD~#K)RvWAH9+ki8i8Pn{d@U`zA% zix>Y=J#t-q|B3MhSG+cPglnloyIeJgIGVl=To;)Z0EfoJW_ONS8rxBaD`uCoun;U_YFlqE_${kHMT~* z&1!Y6wp|*>Z|`6rByqN-pi^jqIsA=7PoZG4V>se}fR(n6kePlbg-4^H1Jj}kE!%(eDkxqpn#?FZTDM z*m>>dK4WczlVX=Lna>%P#Y+3!i!W^vOJ?D2{8l71cY`O+X`3XEv*fCd0Ns@Vd|?4T zV`_Ww*F%AvAJGy%2WN0RT%uFwhJWAN!j;&3?!Zo7R5HZ`9St1NCx;#i5O&H;vTL7` z`=cd7w%)TS4U3v?)A*c{jCnxz%ai@#c~UAp=#i$iri8yfngoOvhFzA1Ed=RZvw{Q2~^G&Qd7 zqhb9YuBMW2{ygKO@{{WcW^MUQ@u){*nIaJA$7{d(HXm_JAfPSiRFsv=ZKv8&CjCprD3?ZvTDR{RWW;O_KYL0q-M z;abOfV~35|*YyMNq3Nx*N=-Fw z^Tuck5FnMrfrpeXWwgoE=A3}CEWZi&jgOwx6_C=oGs9Ohe{L-I#j5@dD_2EI(~Fpz zq`0DN30*hFSwBl@B%i4OY{oTEx#hB-_vQ zmR;V+0?^~D5|6LrD<&yxg<7HPyQu;4e6=o{SB^Y$ol>bH|7kP|EIo0wsBzW#D1eGC zyEpiCEkQ+RnwH_1|E_V}q)fo`yM<-nd4F-2T)D4hKaU9V1PZRx?riiQ?;Ial_se9O z+VD)9NDUi4mA?N%m$>tQceRGO>IkqhrzSU_OLy(u%rts)YOb$O(aw$k^Y(r#n(;b; zlL#%)OVHj&kV~&bN}7dRR029V4c(!SQo^@)@nCJxQuT<-)b#+@Bhd5>Q!PR8kW^6&4)1I~?8ZpV&Xz zU?-L8+>q5g9g_RF-1_^6ztuZufjh8p5UJTd0aycPrN`>LmXMx;x^clqUmF?61gygt z!w*Bi931suUm=Z3Ri~OM8&0NLxi(sBV)GrthajG{t9TxT z+ddos%t_kSmb8?(F|3%u*Y%|0?X!7_OL9XuL_a42Le`os{G z!*_K0)#@(pqprNgc~)W^-6d{wNXpUavzJ`LRLBDjmoiy)@!x2Dcu_iu73%keyyrtTWOe}S^cP8XTu!_E+&a1FNoz+P;CCOA?f*yZkO+Z(Gadd|~q$y+jg zF__IPy|`z!uE3F1#rb-T%ny`Z)t9rYIWI&skw2m1+vT}&uCVZiULmky}}U!!KT7wd=JU`C>X}nbGXGFR&X)QeMb0^r9p|)<^-J^8pbkO_c#^` zA3bfsHonWfEV|n8$zJAOXbe~6a04XuJt2+S#qjmt&JX1ZMlE_R*3s0FErgl{5f%gR zhH7{bN}vD<=IyAkMU9>Ryz&o`oSstHrLxV(F5-Xl!kCD=7k)mrW*2XjrSJB1hGIKJN%A%u~}?d>maw*ziHP*h4PnJe*S!PZ%m@t-FRq2|qlg*h(fxOzL_J zxw4BsMv0QHF`lVvXQq&LR*@Js+&3hFQ&mWP5c@BrwF7ILu7-$KXt1h6OFUPOSJscHL19a%>%)_1Yd$;JX zmmPPTIT>-Ed8O;5bBC0QgzBLJNs6jIJDQ3ZLw3)vkAsdsF(kG+()8qU_lcf{u3cXQ zgRRcx8NZ-TWKVul9KvDGWTD4ThojJ$%=W=k6^YpOi3KHT=359_wZQdXNl`Ylz5usP z;lj*Pr%a9LQ4}XO8_wms+-Ivu(PbFhjb*L?lsQJfU7qf;_>iYjPt;|}SCJQ-_O66W zL|9O8lt|KiiDX#ky-4>sVlB9e%&Uo6yzq%}A5ZAeX6Ax|0EKR5H_tv``0L57D56jk z)`?qCNLc4G0-*0##hJgU!NJ42yJ#&=E!okR%2?VUak>~f%Qckj@!?+{dz=8tBVW5~ z)rn6}@uA<_{77;ovENL$PdGzytE4_*OM`q{)}NY5B@QYV2jfJB`EI@yey+U-__uKq z+01`$8@wxc<_8_cDotEB7*=!Kb`78fb=mia#2=Tof=7uTARN_GYCLD7r9;&y<+Q^0 z1*1w;uRFPX$Z|VM7Qz18r=#+j{KIxG`qR?K`XfETU5ZKf^f-N9YSW9M-Se&IyOnM! zOY41m=Sbt9qm{so{VT7LELFBshMyU z14{Fa=PFK|O=vFC_4BvhJ#VLeq_rET>ee5F5m!r^-Ha1snQP8_wsY;inSOaAuTf}i zyL{N)=Y#SnHT+g~>|$dxSM1^>s-Q|Q6+?ub!%{S;+bh`E z*x*b|xjBEJA{YiagNU{6+&{0Nba73Eb3R8YrLD_M=B8HW)aE09<5@~2@-6XDaXWE> z?Dw9Ig6pm(1f4-O9TiX2OKlBpQAJtzzuZWCw`ieH&=FYSZR+$avYomhjSqE5za=cq zsZyABkCrv)aD5sKofV_=nq8~YW$J{r`Ia)u2sSNNkK?)GDX3J%^ahK!{Zg9bjiTP9 zHB=DSNqX{|h2&2sgJIn4a~JJ%py9`G8+?;mI`fI{V?5%&0gA zR{X=#mjuDK8nj8|Hr#mLX{Uz7N*5z;llBmnYZ=zm{QoRDIC>`^{J<0l((Z!mE=YzoiX+rYbQCv2&Ie-4vvZx z4cQF7O`>Q;PO1~OdHdHc2tC2_VtvMV6W{3+*PF-~^)$5EoT)ggDV&e$=7|uQ(U(f2 z45M5pA$f$8M8&$j0NU=GJ+1@cmRPLzmgVst&t*{(JSmlt8mAeh*=?4hfrcu9dM~Bm z^-i;TkDHdl%vv&yN?(hy@i;mgycb2Hv}TJ7>)9wTXFWqOxFDmbt5Ns51H?R5ga^~97rP2~tvhN#|Lz_vv-O>P0w7WC4 z?~m4)y`pXo{U+4ZAhVRqNIq(mhhjzuFVn^G0>Qe|x0{+B6aI3K3)m4^@kSZ=_T3u! z+r+7XUgg@iyM(^dq>WzH)74w2ntrc>;hTE5K;f%)RUYY``aq`O59<|5c1riElQxe> zt(D5cklfUecQ+5MqtFLY*D@*Lh-tHelx%v9%3rF+rD)Mua z%8^-LCL}gj-wTLQjVgIqIg$;B6lML3yRj}*JO3%9=O8MDPOwQZ*=|kZx@WDyG;Krkr?Cs>KV2ge04Y)ANb@N#qJ?$N0ol z>Z6x&FR?76nE=&q7eqDB)n>6K5IfK>%$KAw(Z0)rFL*t@719tU7$SeOj@5np+jY%o zKmSxmp0AvtCOO%opf}-sIsMuBJm-fBJibl2Lp(_)0tt9F(B!&X=ozAw6Aj7IFJuNy z(o=D}MwPryeFrJ`I3C8JpYnLx+Ha%CpN! z=MZf#Ru!sN|KL;~Jek2K3FfOEt*tU`Ljn#b)SukTz$(<5!_hN2zb41oLZW57U6_?> z8+Qp$X%JDPrd}(U+=(}TCg4lcP0M};O@7Ffm~76c`hN% z=&Z+}Ws0e~+*tRNJdzG3mu5$m&|9gZ(eHe8uHA^eU&kw`MlP!@!)53ij}6DR>-yfS z>6y}Eou13|4EBh|T)WK`baGOz#83CB#n~La$#}Qy1aM9~0u{ox%C_ZnLTN~4vzb=* zKGQ`4x4<{aHxaOI*+#!ruNe|rVG>rsJz_6Dp4N!wNmMvL z&u1%wzES4t^MhV3^nhyy8+vMw$Lv9cetcmb&Vp-`o|u5S$#2MX@59T)lm{k#r&6f* zN_VoIt`l;_(qDaY^br9>;GyG>Mg)Pyom-?)q?7hnR+oUYS(;d*Z8^3mzJH&k_9s)1 z^5(+J_xt`4R&xF`?Q4(kCJr!2e`MTw`cmm3?x>q$Ffo}u^&e9ttRnf9BD@bab&e5$ z|M+WHQ?G_)wtf=-Ch;xBk;r=9_?MENH^wd*>0Tksc6jOzE?j%`$=TrXdf^oB)6vCP zJ*pCJjUwpmNZIEa_&+3-2J#wuuNZ0 z=JEc_#r|DB;RKetK1igC-#QIkCT5mai| zxp?tQ=oQ+d_#IUHNh_We7Q7(72$t(z2XO`N4<|pD%rsZ1SyzkL>&1Ku+Y*TC-l-Wc zs!BdE!x*$3R+nGDymI@?OSthkgc?PTtqloNwS<1c8u>OVk3QE^F#(?WV_TXQu<=byPR>QE^L4 zEKLjRLo>d3gQ^F()nUi&3#F3^;3p?Q@_ASGlR?f{m7w0-n5`A!d?Fp=V-HmP{~G3Z z)*PP?{7M#2p3%NeYlWHjXn3&rx`rR1)(II5b>Cm{ue_WNc{$rn*1j1}8&Q=ZU3;;3 zq@X8UzQ>K_&$kV)gReri|B59pD{C9&J3CA!af#FAYn*p-4c90CXvb5XedC zwV|^AEU+5hl#e16?v~Xed!#6SU6%f*wKf;I~g_=GSz*6-yvPS z&|Z!{sUKfO|2iDk;U6PLz<2c?B;k&zQ*Luc_c;p1W+La->vZ+e=DQV?7DAF5`LHZCtuzxp3SI# zdir){i&z{@;yj6K&^4|YjHw&FLYmvZdY2Gi7N7uLd9Ackzj)^qr91SAw5mj51}-)1 zo3|^X=@-5Co?c^Sc(7|ea^~9^uL6JB8$BwW#=-aSc+1avtX~&pHGSE_o{F>29K6u? zs*@*czlOVAOE>h#>+i&1<{CzY)sC9Q#La2*l7dKSym$Lk@ih5?@?o=$CId56J{Ok^ zVWmIqeV-T*!aX6p z47H#-37^^ZBP!{QH>bq!wNtzkCQVF>p#T%=P zEDBm^N>v+O>O2zKoTghDyfkjcwl@SRUt42@`O`O3G!{4@^#PR0X}L7%mpl zyU0Da+yRhX&mcnD2|ls)>Z!;>o`0WZo(PQU?!Qa(b-$g7zA@L|E7i_K$GgEB>-Jo% zl8qHGCt)=pPYjro{~}(`b);I*^PMh#BgtB`-2q@v!)7Ij(|3Yiq>Y3DS>QT41_8k?Be9YV%p|)ta%C`ki5viT& ztyQg7*$80Fo99E|{Zyad_u`H90(TmxuTXYZ*RTR)Gdsj{?OcNEI;UbGG^=|lgVX`)#%^cy z;P0*Z>jm5%@lsG-OF)3h6!O+&O6S1~)rxrY77A-!fQ&XO+ghq0oshz9Qb@Yw`;(eswz zLoHWdBP_mmzn-r8#5iAE|EJ107edJ^4#a%o!1v z>o+Y{!Wm*{zP zrkqn%1dzsy(ELF#38Zk9b>>!jK3D*#^UY?Zw`NGCr`9T=?j-Rqb!CfMyN2|J)2|ob zog-7OR$y(K-;ZfV(0C`Kf%t#hfy(D94b0+CGe<`R)2|I zX|~FBf?z&Y!hKeKx>wTTo?ThX1E+Gdx@sKf^s~#vSx8HRbG{M+*3du>XG7`RjMV>N z2aJ!A9+zXudw6$@4#)nn@A91In4;_WL-RX#f)2lBbR^S?Q5Uq&=OwJN)!&ryS@*DQ z`M16N+h(QwB7BEl&1Zvaa4f>$x@K8xl7!Ka^R5XZL1{-TgVVNp%jqO{>2h*;Hyr0m z!+MB#z-*Z9%KYVXcP$kCEE~M~>!vHtDc2`vVZJ0X8~>FZcQa=J#$>Yx+Fi>`m%j3p zJ(ZJ2oUtToZD(ikN!}1ScL351aLq! zdB$k?g7R*WxP7dnhV*#X>;p=iO5At2`?ObTw$oz%94Q2jdQRdMBLiVa42-OeAt-;2 z|7%|0&$K$6w2-d>@YwSq6!fLj@h14~F7@CIy-7MFj0!pe{lzF2#BQ)Zck{y)#~x@K z#!b>oyGa)0LsybVS?FLtVzZS0e;)e37MVU{$7J?4eD;dTg#~%Z`_d3JP*JQj&&|@|2gT?)islA(CYkO;OOyRuCi$%67@8ui!53Umy_o@jM zumT^?H8LQrbVv^&@x?o5xQAADjNvK1|5Kf*@QOuk1TI?t4Q*WC2flwiLZDN2b9{Uw z4-^f^ImQlm*M=6?J1R2s@hoq~VfJ5N{<;7#t}tj@vZK3xcWypu=fIhHnu~#*f^2d- zGMo165{sg?RUgeMKE1}3pHICfY5(!+l|e&Frq4HL#YpeJ*FUu<*2(7Dwg9?@-d8mP z9{LTn_#18!Avm-IyC-j?-ZlL&{B(lm=_@IPy)>`aQrUjANADXNr&ppW)%UZTrXr2W}_yp5Ghf zReX#8;8kakSJ^&#Bi(bmCFhUg{TFNP2OnsG!XRje)MWOZq2I^Uc7Wq6ZUWc8;rip} z7ho<$?)wUSTZufVY94ihRfut^1G25W#hao+mQsct@TG)jk4)DUC3lzM4 zsK3~~T8HW{s=agZWutJ^D$50- zLzj;;o(t*k8gnWIlUW`yeSs}HT?e~*AaGF_x|q^DL;vHFz-+oB6*3mv5)RnBpIpo? zNL5foRyInlG)y189GQ=2v(>$^zg+i?%TnAn&`jColJTGhVR!8?-WG(HE`ML>!XSe> zfbJ-m-bz7)11KMPcW`HT#c44757ZJ)pIzRr8eexSJAA}}If zjbLjk(=(06Hcuac5#$~EZ)=m4QI5)fV2op%r^#qc``T*V7#&Eq zkSg-URNDkd87T7YyQ#7oEWmByon2zc>IO3x_O|(h0eEOsU}ZjdiYyyLHj64o?)SgWAB!ZAQcgLTF2>ubziSMG6$OH_% zUzh^W#(!JG7K#4M#|T%TfxZ`Qap$|AXk=iU*Qd+U;cy#9AkEM3lUTJwkj>&$#p>b; zN6LJ5{wC6zXf#>-U>;UJ5T3nv!}pduF@l(mI4rwSLy_nNGAFiwCK_!&7V3uQ#4;=@ z!e>ujsPAD%?mr37H^K_=Kib9ekS=XMq$!dtiL5F1k+5mE0u^vRifmpY|866iMsjZR ze8S??DLyU6maLsnm=H0+@g7d3$-QvITNGYj4`cI=b!*Jl|Jnh~`yHU=MpB|2 zr=Qj-&e){nhH(mqe<=L9sptv{0$`tSFV8nEwn4q7$D-SV$slMVWIBdn9$(%QSDF$2 zL~iK6r_s8~WGkDbuH*l0C(iAJ&(~|(Wy{=$PLvMd>R@|*>B&_YTf31`6I)Q7vxOqC zE%dRm70wj&A%q=n!dC5jQo?$d#&@sa#!8hpGe28*4^TTAS{4fqdu{lb!FseSLZh)x zt^~|ScU%)U2N6En!lRL|%<8RN48x})!`1P*$B1%+i4=Nt$`P92Z9&0dkAKMoQw;ZD zd%46K@=;ph1Jf;1)~tV^S3kSj_hV`{Bf`#+r?fos;)`*9(fg}?A7Jo}D+cD^Kcv8X zq_#FSaS_EG`mer~X8!jIG>)X3iXJm@W{XR;U))tq!h({ESfq_L383-pRG7XvE9&I} zBbU7Lo9|n;(7CVJwVUd`O)Kpo`PLSCpZ>p%yS7^jV1Fs+To;S9{!n&C85L-4eR|3I z{shvF7p&<`(ZS%PNsw8NE@-Fya2W8_=hM^dssCfxVlcjfzL(_@OP;sz7MA-4EH~lS zvn-I>8FGi6_nm*`uXd&ir0K$x){FQ9Yi_|%~m=HvpLJkR9dBJjo8$-j8d9%Xk^jJDZAM)+9_>qj-3qcmWB>G z(3XzW^5(*}SkeJ?)%(2%t*f{9kN5xl<+^5`-}C(L-~HTkKhN*^e(u}*WKLF|D{JYD zkWGX57!u`iXqmfg?L^26!|A=-zG1I=QYS(SbN=$nn5KA3&)tvGR?7_A&vN%F3N1L= z5lKI0z3M_3Ie{b3GXFfG-iXRv?A=X^VY9iRxSmDYu-Wq&V1?2-B#A`DEjS>@*hscZn^FdXcWO zQCe8qyuSzl6X}b({OS$}S~^L$2UJ6;pP;8c4YzICt0wA$Nv&W;{Me)tjv*eGphe{8 zY|#b1FXM)f7!#LY5@NT{@tjEh=iy8Rpv4tXy!QpUC9NvQEo=X$snv#okW9 z(#srzTE4>B`f>U`EZ3O5SQP0+_o4nG6U1sJUhq$Ty&{VjLvgZZV;BJzVn{Jt_*^st z|A*neM@O_9iAPZd7lRO_;zgoHg+<4ca|P0fj(lv?Ha7Mprz+HkIA%{lRNeOq26I=xcTTh=;dWp+nsRvEV_=VP49%mY$qPekW#KATlIO)2Oo~DtmhsNUN3U%<%iB#g z6&#pXi__t7PVQqKtby&m#X|@lkuwXaCrkgGu9z*+#Fay!YU446s`s8Em|S+O`i2}n zJrP&gF{bV)uEL>A!XTTl8muFfaKzJe`goO(cKPTp|KB{~rTdny5Av0ZgjZ`iU6iCN z_0RbV0VW*gq8=uBy-xo!9kq)V%X^zMmD1dvfWrP6SEkw_f1E^whobh=bnR=NR#~->rZ9j=L*zVf?nNvUz$<4mBxeyZ;KhpnT4-Sl0U7F{sy9xA^ zX{aK%=-OR(n%}Kz5}foa?Aj1sFApm<#wugP-m&z`-qH$T=idd{X4rDtvY-KSM@_)@ zwZU?4`}Pv)vEBT)sjt|0+vj(8VeJuq>DRgcqp3>7V~{*KBuam&@l~j6uo4*Am#M-) zNf*O_+DhyqZD20tn~t@xPD7l5Mg$R5VCzaJgQex4jGU))C*N;ub4|`4ud3UL>c*33 zboRQ3NClhjn`@5xM^)%{fVeT5Wg=|=O7yvmxG<4o78o?NQ>SZ~FrK0!xFt_L(i+xN zKG=4z_zNa>s3xsgIh9h$fwRp?2)g=W#z?zo-g1E96wm&=wMt=AxdWMA^6PRV357y? zHczq}c#8xu;$}CPaKS?85&t+V(99NuPkUeQ1bV^;XwhEges?wP1bC{vd`zVRMM-10`ioFz)XA&2n(45l&n z6a9=Im-J)=?A|-|26R{b-XcRhx4hyp$BPQHRy{)>dTc^B(R4hff z$6vP#23pg2?P)^{Wlq_WVH5Fb!-%$L)Q%D>GCF?LH+LBU$-$P)!<{|#ehQuVyWUgr zOTlH;wlAhHkHnXs>6{h}>3s%pdml2ja_qRD?|7(eLrJwKGwt*8R1hg&6*5zO1eghqYtRm1gCwfljWwYcs=v5D9Ey>2NE*_Fe5zc!SL4Y}J z>x*@GF3UzG(3)YOWv8X2Ady9BHlZ!YLu0cZ=kTA0u+)U>;8s3f0QZ`qG|*;ykf{0LCBhLVONMexXd;zkl73 zF;BeSq52`0ETkW-Tm$w-S)j0P>qTDS$-UTMA29s|b2Md4K>18v}G#VYuW6qK`32mB4vJ3r zAM3O6(I+P}ya3!9aF4-ncu!#H$tr*FFv$EA7@H!H7WCF=Cjq4N-uEzlSn{rmh7}Ov z>UAI7^B}D>s+&2hUI!lGQ+{+daF&Cf`>y+)^ZikSuMxO^b(9K8ax^|AUZBJ!Sea5HUqa#0mz-66|&H7 zK0g5An;fQVc);9s?_F2v0KhVJKyA27ajnwN-8`owL17T}z=aJ_W&WUAv+{TM=qB=)WAd3~^HY*%%z{D2)jocsaT5@IUw7qd;b8?hz@ zn(b=j%A8&}8z0XN4313cN8-IVP4Hnb+>L*34Qsd3tj0A`+q;B Date: Wed, 18 Sep 2024 16:31:46 -0400 Subject: [PATCH 03/73] Updated reassembler constructor to (a) remove dpV6 flag and (b) explicitly take ip::address and port parameter instead of using what is in the URI, since in a real deployment URI data address will be that of LB. Modified python bindings and all the tests, however did not check python examples - those related to reassembler are probably broken --- bin/e2sar_perf.cpp | 22 +++++++++++--- docs | 2 +- include/e2sarDPReassembler.hpp | 20 +++++++------ reassembler_config.ini | 3 -- .../EJFAT/E2SAR-development-tester.ipynb | 2 +- .../EJFAT/E2SAR-release-tester.ipynb | 2 +- src/e2sarDPReassembler.cpp | 20 ++++++------- src/e2sarDPSegmenter.cpp | 2 +- src/pybind/py_e2sarDP.cpp | 5 ++-- test/e2sar_reas_live_test.cpp | 4 ++- test/e2sar_reas_test.cpp | 30 ++++++++++++------- 11 files changed, 67 insertions(+), 45 deletions(-) diff --git a/bin/e2sar_perf.cpp b/bin/e2sar_perf.cpp index a27a5629..51164171 100644 --- a/bin/e2sar_perf.cpp +++ b/bin/e2sar_perf.cpp @@ -159,6 +159,9 @@ result recvEvents(Reassembler &r, int durationSec) { auto nowT = boost::chrono::steady_clock::now(); + // register the worker (will be NOOP if withCP is set to false) + //r.registerWorker(name, weight, min_factor, max_factor); + // receive loop while(true) { @@ -238,6 +241,8 @@ int main(int argc, char **argv) int durationSec; bool withCP; std::string iniFile; + std::string recvIP; + u_int16_t recvStartPort; // parameters opts("send,s", "send traffic"); @@ -258,6 +263,8 @@ int main(int argc, char **argv) opts("withcp,c", po::bool_switch()->default_value(false), "enable control plane interactions"); opts("ini,i", po::value(&iniFile)->default_value(""), "INI file to initialize SegmenterFlags [s]] or ReassemblerFlags [r]." " Values found in the file override --withcp, --mtu and --bufsize"); + opts("ip", po::value(&recvIP)->default_value("127.0.0.1"), "IP address (IPv4 or IPv6) on which receiver listens. Defaults to 127.0.0.1. [r]"); + opts("port", po::value(&recvStartPort)->default_value(10000), "Starting UDP port number on which receiver listens. Defaults to 10000. [r] "); po::variables_map vm; @@ -279,6 +286,8 @@ int main(int argc, char **argv) conflicting_options(vm, "recv", "rate"); conflicting_options(vm, "send", "threads"); conflicting_options(vm, "send", "period"); + option_dependency(vm, "recv", "ip"); + option_dependency(vm, "vm", "port"); } catch (const std::logic_error &le) { @@ -294,9 +303,6 @@ int main(int argc, char **argv) } withCP = vm["withcp"].as(); - std::cout << "Control plane will be " << (withCP ? "ON" : "OFF") << std::endl; - std::cout << (withCP ? "*** Make sure the LB has been reserved and the URI reflects the reserved instance information." : - "*** Make sure the URI reflects proper data address, other parts are ignored.") << std::endl; // make sure the token is interpreted as the correct type, depending on the call EjfatURI::TokenType tt{EjfatURI::TokenType::instance}; @@ -328,6 +334,9 @@ int main(int argc, char **argv) sflags.mtu = mtu; sflags.sndSocketBufSize = sockBufSize; } + std::cout << "Control plane will be " << (sflags.useCP ? "ON" : "OFF") << std::endl; + std::cout << (sflags.useCP ? "*** Make sure the LB has been reserved and the URI reflects the reserved instance information." : + "*** Make sure the URI reflects proper data address, other parts are ignored.") << std::endl; try { Segmenter seg(uri, dataId, eventSourceId, sflags); @@ -359,8 +368,13 @@ int main(int argc, char **argv) rflags.withLBHeader = not withCP; rflags.rcvSocketBufSize = sockBufSize; } + std::cout << "Control plane will be " << (rflags.useCP ? "ON" : "OFF") << std::endl; + std::cout << (rflags.useCP ? "*** Make sure the LB has been reserved and the URI reflects the reserved instance information." : + "*** Make sure the URI reflects proper data address, other parts are ignored.") << std::endl; + try { - Reassembler reas(uri, numThreads, rflags); + ip::address ip = ip::make_address(vm["ip"].as()); + Reassembler reas(uri, ip, recvStartPort, numThreads, rflags); reasPtr = &reas; boost::thread statT(&recvStatsThread, &reas); auto res = recvEvents(reas, durationSec); diff --git a/docs b/docs index f6c7612a..42ed8636 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit f6c7612a9449a5827d883cc1171fafb0d3876a19 +Subproject commit 42ed8636c9383c3358b4d368dbaa53f9c5cad90b diff --git a/include/e2sarDPReassembler.hpp b/include/e2sarDPReassembler.hpp index f66c64ba..9de838c6 100644 --- a/include/e2sarDPReassembler.hpp +++ b/include/e2sarDPReassembler.hpp @@ -231,9 +231,8 @@ namespace e2sar // receive related parameters const std::vector cpuCoreList; - const ip::address dataIP; // from URI - const u_int16_t dataPort; // starting receive port from URI - const bool dpV6; // prefer V6 over v4 address from URI + const ip::address dataIP; + const u_int16_t dataPort; const int portRange; // translates into 2^portRange - 1 ports we listen to const size_t numRecvThreads; const size_t numRecvPorts; @@ -319,7 +318,6 @@ namespace e2sar public: /** * Structure for flags governing Reassembler behavior with sane defaults - * - dpV6 - prefer the IPv6 address/port in the URI data address. Reassembler will bind to IPv6 instead of IPv4 address. {false} * - cpV6 - use IPv6 address if cp node specified by name and has IPv4 and IPv6 resolution {false} * - useCP - whether to use the control plane (sendState, registerWorker) {true} * - period_ms - period of the send state thread in milliseconds {100} @@ -339,7 +337,6 @@ namespace e2sar */ struct ReassemblerFlags { - bool dpV6; bool cpV6; bool useCP; u_int16_t period_ms; @@ -350,7 +347,7 @@ namespace e2sar bool withLBHeader; int eventTimeout_ms; int rcvSocketBufSize; - ReassemblerFlags(): dpV6{false}, cpV6{false}, useCP{true}, + ReassemblerFlags(): cpV6{false}, useCP{true}, period_ms{100}, validateCert{true}, Ki{0.}, Kp{0.}, Kd{0.}, setPoint{0.}, epoch_ms{1000}, portRange{-1}, withLBHeader{false}, eventTimeout_ms{500}, rcvSocketBufSize{1024*1024*3} {} @@ -367,20 +364,25 @@ namespace e2sar * the number of cores on the list. For the started receive threads affinity will be * set to these cores. * @param uri - EjfatURI with lb_id and instance token, so we can register a worker and then SendState + * @param data_ip - IP address (v4 or v6) on which we are listening + * @param starting_port - starting port number on which we are listening * @param cpuCoreList - list of core identifiers to be used for receive threads * @param rflags - optional ReassemblerFlags structure with additional flags */ - Reassembler(const EjfatURI &uri, std::vector cpuCoreList, + Reassembler(const EjfatURI &uri, ip::address data_ip, u_int16_t starting_port, + std::vector cpuCoreList, const ReassemblerFlags &rflags = ReassemblerFlags()); /** * Create a reassembler object to run on a specified number of receive threads * without taking into account thread-to-CPU and CPU-to-NUMA affinity. * @param uri - EjfatURI with lb_id and instance token, so we can register a worker and then SendState + * @param data_ip - IP address (v4 or v6) on which we are listening + * @param starting_port - starting port number on which we are listening * @param numRecvThreads - number of threads * @param rflags - optional ReassemblerFlags structure with additional flags */ - Reassembler(const EjfatURI &uri, size_t numRecvThreads = 1, - const ReassemblerFlags &rflags = ReassemblerFlags()); + Reassembler(const EjfatURI &uri, ip::address data_ip, u_int16_t starting_port, + size_t numRecvThreads = 1, const ReassemblerFlags &rflags = ReassemblerFlags()); Reassembler(const Reassembler &r) = delete; Reassembler & operator=(const Reassembler &o) = delete; ~Reassembler() diff --git a/reassembler_config.ini b/reassembler_config.ini index 90a84615..2fd829ed 100644 --- a/reassembler_config.ini +++ b/reassembler_config.ini @@ -9,9 +9,6 @@ validateCert = true cpV6 = false [data-plane] -; prefer the IPv6 address/port in the URI data address. -; Reassembler will bind to IPv6 instead of IPv4 address -dpV6 = false ; 2^portRange (0<=portRange<=14) listening ports will be open starting from dataPort. ; If -1, then the number of ports matches either the number of CPU cores or the number of threads. Normally ; this value is calculated based on the number of cores or threads requested, but diff --git a/scripts/notebooks/EJFAT/E2SAR-development-tester.ipynb b/scripts/notebooks/EJFAT/E2SAR-development-tester.ipynb index 4b7f575b..18af02ea 100644 --- a/scripts/notebooks/EJFAT/E2SAR-development-tester.ipynb +++ b/scripts/notebooks/EJFAT/E2SAR-development-tester.ipynb @@ -672,7 +672,7 @@ "numEvents = 10000 # number of events to send\n", "bufSize = 300 * 1024 * 1024 # 100MB send and receive buffers\n", "\n", - "recv_command = f\"cd E2SAR; PATH=$HOME/.local/bin:/usr/local/bin:$PATH LD_LIBRARY_PATH=/usr/local/lib/ ./build/bin/e2sar_perf -r -u '{e2sarPerfURI}' -d {recverDuration} -b {bufSize}\"\n", + "recv_command = f\"cd E2SAR; PATH=$HOME/.local/bin:/usr/local/bin:$PATH LD_LIBRARY_PATH=/usr/local/lib/ ./build/bin/e2sar_perf -r -u '{e2sarPerfURI}' -d {recverDuration} -b {bufSize} --ip {recver_addr} --port 19522\"\n", "send_command = f\"cd E2SAR; PATH=$HOME/.local/bin:/usr/local/bin:$PATH LD_LIBRARY_PATH=/usr/local/lib/ ./build/bin/e2sar_perf -s -u '{e2sarPerfURI}' --mtu {mtu} --rate {rate} --length {length} -n {numEvents} -b {bufSize}\"\n", "\n", "# start the receiver for 10 seconds and log its output\n", diff --git a/scripts/notebooks/EJFAT/E2SAR-release-tester.ipynb b/scripts/notebooks/EJFAT/E2SAR-release-tester.ipynb index 130cc076..2e962fd7 100644 --- a/scripts/notebooks/EJFAT/E2SAR-release-tester.ipynb +++ b/scripts/notebooks/EJFAT/E2SAR-release-tester.ipynb @@ -586,7 +586,7 @@ "numEvents = 10000 # number of events to send\n", "bufSize = 300 * 1024 * 1024 # 100MB send and receive buffers\n", "\n", - "recv_command = f\"LD_LIBRARY_PATH=/usr/local/lib e2sar_perf -r -u '{e2sarPerfURI}' -d {recverDuration} -b {bufSize}\"\n", + "recv_command = f\"LD_LIBRARY_PATH=/usr/local/lib e2sar_perf -r -u '{e2sarPerfURI}' -d {recverDuration} -b {bufSize} --ip {recver_addr} --port 19522\"\n", "send_command = f\"LD_LIBRARY_PATH=/usr/local/lib e2sar_perf -s -u '{e2sarPerfURI}' --mtu {mtu} --rate {rate} --length {length} -n {numEvents} -b {bufSize}\"\n", "\n", "# start the receiver for 10 seconds and log its output\n", diff --git a/src/e2sarDPReassembler.cpp b/src/e2sarDPReassembler.cpp index 81de4cdf..d403e427 100644 --- a/src/e2sarDPReassembler.cpp +++ b/src/e2sarDPReassembler.cpp @@ -34,7 +34,8 @@ namespace e2sar error, integral); // control output } - Reassembler::Reassembler(const EjfatURI &uri, std::vector cpuCoreList, + Reassembler::Reassembler(const EjfatURI &uri, ip::address data_ip, u_int16_t starting_port, + std::vector cpuCoreList, const ReassemblerFlags &rflags): dpuri(uri), lbman(dpuri, rflags.validateCert), @@ -42,9 +43,8 @@ namespace e2sar Kp{rflags.Kp}, Ki{rflags.Ki}, Kd{rflags.Kd}, pidSampleBuffer(rflags.epoch_ms/rflags.period_ms), // ring buffer size (usually 10 = 1sec/100ms) cpuCoreList{cpuCoreList}, - dataIP{(rflags.dpV6 ? uri.get_dataAddrv6().value().first : uri.get_dataAddrv4().value().first)}, - dataPort{(rflags.dpV6 ? uri.get_dataAddrv6().value().second : uri.get_dataAddrv4().value().second)}, - dpV6{rflags.dpV6}, + dataIP{data_ip}, + dataPort{starting_port}, portRange{rflags.portRange != -1 ? rflags.portRange : get_PortRange(cpuCoreList.size())}, numRecvThreads{cpuCoreList.size()}, // as many as there are cores numRecvPorts{static_cast(portRange > 0 ? 2 << (portRange - 1): 1)}, @@ -67,17 +67,16 @@ namespace e2sar assignPortsToThreads(); } - Reassembler::Reassembler(const EjfatURI &uri, size_t numRecvThreads, - const ReassemblerFlags &rflags): + Reassembler::Reassembler(const EjfatURI &uri, ip::address data_ip, u_int16_t starting_port, + size_t numRecvThreads, const ReassemblerFlags &rflags): dpuri(uri), lbman(dpuri, rflags.validateCert), epochMs{rflags.epoch_ms}, setPoint{rflags.setPoint}, Kp{rflags.Kp}, Ki{rflags.Ki}, Kd{rflags.Kd}, pidSampleBuffer(rflags.epoch_ms/rflags.period_ms), // ring buffer size (usually 10 = 1sec/100ms) cpuCoreList{std::vector()}, // no core list given - dataIP{(rflags.dpV6 ? uri.get_dataAddrv6().value().first : uri.get_dataAddrv4().value().first)}, - dataPort{(rflags.dpV6 ? uri.get_dataAddrv6().value().second : uri.get_dataAddrv4().value().second)}, - dpV6{rflags.dpV6}, + dataIP{data_ip}, + dataPort{starting_port}, portRange{rflags.portRange != -1 ? rflags.portRange : get_PortRange(numRecvThreads)}, numRecvThreads{numRecvThreads}, numRecvPorts{static_cast(portRange > 0 ? 2 << (portRange - 1): 1)}, @@ -540,14 +539,13 @@ namespace e2sar } // general - rFlags.dpV6 = paramTree.get("general.useCP", rFlags.dpV6); + rFlags.useCP = paramTree.get("general.useCP", rFlags.useCP); rFlags.validateCert = paramTree.get("general.validateCert", rFlags.validateCert); // control plane rFlags.cpV6 = paramTree.get("control-plane.cpV6", rFlags.cpV6); // data plane - rFlags.dpV6 = paramTree.get("data-plane.dpV6", rFlags.dpV6); rFlags.portRange = paramTree.get("data-plane.portRange", rFlags.portRange); rFlags.withLBHeader = paramTree.get("data-plane.withLBHeader", rFlags.withLBHeader); rFlags.eventTimeout_ms = paramTree.get("data-plane.eventTimeoutMS", rFlags.eventTimeout_ms); diff --git a/src/e2sarDPSegmenter.cpp b/src/e2sarDPSegmenter.cpp index 83c39ee8..17642460 100644 --- a/src/e2sarDPSegmenter.cpp +++ b/src/e2sarDPSegmenter.cpp @@ -612,7 +612,7 @@ namespace e2sar } // general - sFlags.dpV6 = paramTree.get("general.useCP", sFlags.dpV6); + sFlags.useCP = paramTree.get("general.useCP", sFlags.useCP); // control plane sFlags.syncPeriods = paramTree.get("control-plane.syncPeriods", diff --git a/src/pybind/py_e2sarDP.cpp b/src/pybind/py_e2sarDP.cpp index 3380feb6..82555fc0 100644 --- a/src/pybind/py_e2sarDP.cpp +++ b/src/pybind/py_e2sarDP.cpp @@ -161,7 +161,6 @@ void init_e2sarDP_reassembler(py::module_ &m) // Bind the ReassemblerFlags struct as a nested class of Reassembler py::class_(m, "ReassemblerFlags") .def(py::init<>()) // The default values will be the same in Python after binding. - .def_readwrite("dpV6", &Reassembler::ReassemblerFlags::dpV6) .def_readwrite("cpV6", &Reassembler::ReassemblerFlags::cpV6) .def_readwrite("useCP", &Reassembler::ReassemblerFlags::useCP) .def_readwrite("period_ms", &Reassembler::ReassemblerFlags::period_ms) @@ -177,9 +176,11 @@ void init_e2sarDP_reassembler(py::module_ &m) // Constructor reas.def( - py::init(), + py::init(), "Init the Reassembler object with number of recv threads.", py::arg("uri"), // must-have arg when init + py::arg("ip"), + py::arg("port"), py::arg("num_recv_threads") = (size_t)1, py::arg("rflags") = Reassembler::ReassemblerFlags()); diff --git a/test/e2sar_reas_live_test.cpp b/test/e2sar_reas_live_test.cpp index b8c8ab4e..f7eb6a5b 100644 --- a/test/e2sar_reas_live_test.cpp +++ b/test/e2sar_reas_live_test.cpp @@ -36,8 +36,10 @@ BOOST_AUTO_TEST_CASE(DPReasTest1) Reassembler::ReassemblerFlags rflags; rflags.validateCert = false; + ip::address loopback = ip::make_address("127.0.0.1"); + u_int16_t listen_port = 10000; // create a reassembler and start the threads - Reassembler reas(uri, 1, rflags); + Reassembler reas(uri, loopback, listen_port, 1, rflags); auto oas_r = reas.openAndStart(); diff --git a/test/e2sar_reas_test.cpp b/test/e2sar_reas_test.cpp index 714bb4de..2f27bd0d 100644 --- a/test/e2sar_reas_test.cpp +++ b/test/e2sar_reas_test.cpp @@ -25,7 +25,7 @@ BOOST_AUTO_TEST_CASE(DPReasTest1) std::cout << "DPReasTest1: Test segmentation and reassembly on local host with no control plane (no segmentation)" << std::endl; // create URI for segmenter - since we will turn off CP only the data part of the query is used - std::string segUriString{"ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data=127.0.0.1"}; + std::string segUriString{"ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data=127.0.0.1:10000"}; // create URI for reassembler - since we turn off CP, none of it is actually used std::string reasUriString{"ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data=127.0.0.1"}; @@ -51,7 +51,9 @@ BOOST_AUTO_TEST_CASE(DPReasTest1) rflags.useCP = false; // turn off CP rflags.withLBHeader = true; // LB header will be attached since there is no LB - Reassembler reas(reasUri, 1, rflags); + ip::address loopback = ip::make_address("127.0.0.1"); + u_int16_t listen_port = 10000; + Reassembler reas(reasUri, loopback, listen_port, 1, rflags); std::cout << "This reassembler has " << reas.get_numRecvThreads() << " receive threads and is listening on ports " << reas.get_recvPorts().first << ":" << reas.get_recvPorts().second << " using portRange " << reas.get_portRange() << @@ -163,7 +165,7 @@ BOOST_AUTO_TEST_CASE(DPReasTest2) std::cout << "DPReasTest2: Test segmentation and reassembly on local host with no control plane (basic segmentation)" << std::endl; // create URI for segmenter - since we will turn off CP only the data part of the query is used - std::string segUriString{"ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data=127.0.0.1"}; + std::string segUriString{"ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data=127.0.0.1:10000"}; // create URI for reassembler - since we turn off CP, none of it is actually used std::string reasUriString{"ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data=127.0.0.1"}; @@ -190,7 +192,9 @@ BOOST_AUTO_TEST_CASE(DPReasTest2) rflags.useCP = false; // turn off CP rflags.withLBHeader = true; // LB header will be attached since there is no LB - Reassembler reas(reasUri, 1, rflags); + ip::address loopback = ip::make_address("127.0.0.1"); + u_int16_t listen_port = 10000; + Reassembler reas(reasUri, loopback, listen_port, 1, rflags); std::cout << "This reassmebler has " << reas.get_numRecvThreads() << " receive threads and is listening on ports " << reas.get_recvPorts().first << ":" << reas.get_recvPorts().second << " using portRange " << reas.get_portRange() << @@ -306,9 +310,11 @@ BOOST_AUTO_TEST_CASE(DPReasTest3) // create reassembler with no control plane Reassembler::ReassemblerFlags rflags; + ip::address loopback = ip::make_address("127.0.0.1"); + u_int16_t listen_port = 19522; { // one thread - Reassembler reas(reasUri, 1, rflags); + Reassembler reas(reasUri, loopback, listen_port, 1, rflags); std::cout << "This reassmebler has " << reas.get_numRecvThreads() << " receive threads and is listening on ports " << reas.get_recvPorts().first << ":" << reas.get_recvPorts().second << " using portRange " << reas.get_portRange() << @@ -321,7 +327,7 @@ BOOST_AUTO_TEST_CASE(DPReasTest3) { // 4 threads - Reassembler reas(reasUri, 4, rflags); + Reassembler reas(reasUri, loopback, listen_port, 4, rflags); std::cout << "This reassmebler has " << reas.get_numRecvThreads() << " receive threads and is listening on ports " << reas.get_recvPorts().first << ":" << reas.get_recvPorts().second << " using portRange " << reas.get_portRange() << @@ -334,7 +340,7 @@ BOOST_AUTO_TEST_CASE(DPReasTest3) { // 7 threads - Reassembler reas(reasUri, 7, rflags); + Reassembler reas(reasUri, loopback, listen_port, 7, rflags); std::cout << "This reassmebler has " << reas.get_numRecvThreads() << " receive threads and is listening on ports " << reas.get_recvPorts().first << ":" << reas.get_recvPorts().second << " using portRange " << reas.get_portRange() << @@ -348,7 +354,7 @@ BOOST_AUTO_TEST_CASE(DPReasTest3) { // 4 threads with portRange override rflags.portRange = 10; - Reassembler reas(reasUri, 4, rflags); + Reassembler reas(reasUri, loopback, listen_port, 4, rflags); std::cout << "This reassmebler has " << reas.get_numRecvThreads() << " receive threads and is listening on ports " << reas.get_recvPorts().first << ":" << reas.get_recvPorts().second << " using portRange " << reas.get_portRange() << @@ -361,7 +367,7 @@ BOOST_AUTO_TEST_CASE(DPReasTest3) { // 4 threads with low portRange override rflags.portRange = 1; - Reassembler reas(reasUri, 4, rflags); + Reassembler reas(reasUri, loopback, listen_port, 4, rflags); std::cout << "This reassmebler has " << reas.get_numRecvThreads() << " receive threads and is listening on ports " << reas.get_recvPorts().first << ":" << reas.get_recvPorts().second << " using portRange " << reas.get_portRange() << @@ -399,7 +405,7 @@ BOOST_AUTO_TEST_CASE(DPReasTest4) std::cout << "DPReasTest4: Test segmentation and reassembly on local host with no control plane (with segmentation and multiple senders)" << std::endl; // create URIs for segmenters - since we will turn off CP only the data part of the query is used - std::string segUriString1{"ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data=127.0.0.1"}; + std::string segUriString1{"ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data=127.0.0.1:19522"}; std::string segUriString2{"ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data=127.0.0.1:19523"}; std::string segUriString3{"ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data=127.0.0.1:19524"}; std::string segUriString4{"ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data=127.0.0.1:19525"}; @@ -436,8 +442,10 @@ BOOST_AUTO_TEST_CASE(DPReasTest4) rflags.withLBHeader = true; // LB header will be attached since there is no LB rflags.portRange = 2; + ip::address loopback = ip::make_address("127.0.0.1"); + u_int16_t listen_port = 19522; // 1 thread for 4 ports - Reassembler reas(reasUri, 1, rflags); + Reassembler reas(reasUri, loopback, listen_port, 1, rflags); std::cout << "This reassmebler has " << reas.get_numRecvThreads() << " receive threads and is listening on ports " << reas.get_recvPorts().first << ":" << reas.get_recvPorts().second << " using portRange " << reas.get_portRange() << From 1306cf8f34a7bdf9930608ba20ce3a48558c6b85 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Wed, 18 Sep 2024 16:54:49 -0400 Subject: [PATCH 04/73] Restored building code on receiver --- scripts/notebooks/EJFAT/E2SAR-development-tester.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/notebooks/EJFAT/E2SAR-development-tester.ipynb b/scripts/notebooks/EJFAT/E2SAR-development-tester.ipynb index 18af02ea..ef968293 100644 --- a/scripts/notebooks/EJFAT/E2SAR-development-tester.ipynb +++ b/scripts/notebooks/EJFAT/E2SAR-development-tester.ipynb @@ -493,7 +493,7 @@ " f\"cd E2SAR/build; EJFAT_URI='ejfats://udplbd@{cpnode_addr}:18347/lb/1?data=127.0.0.1&sync=192.168.88.199:1234' PATH=$HOME/.local/bin:/usr/local/bin:$PATH LD_LIBRARY_PATH=/usr/local/lib/ meson test {e2sar_test_suite} --suite live --timeout 0 -j 1\"\n", "]\n", " \n", - "execute_commands([sender], commands)" + "execute_commands([sender, recver], commands)" ] }, { From 3aa4903ff1ca45272448481dae5448678b4901df Mon Sep 17 00:00:00 2001 From: Xinxin Mei Date: Wed, 18 Sep 2024 16:48:45 +0000 Subject: [PATCH 05/73] More updates to the control plane binding of v0.1.1 - Add members to gprc WorkerStatus - Pretty print of protobuf TimeStamp --- .../example_ControlPlane.ipynb | 209 +++++++++++------- .../pybind11_examples/example_DataPlane.ipynb | 2 +- .../pybind11_examples/example_EjfatURI.ipynb | 4 +- .../pybind11_examples/example_Headers.ipynb | 81 +------ src/pybind/py_e2sarCP.cpp | 53 ++--- 5 files changed, 173 insertions(+), 176 deletions(-) diff --git a/scripts/notebooks/pybind11_examples/example_ControlPlane.ipynb b/scripts/notebooks/pybind11_examples/example_ControlPlane.ipynb index 6eb45bab..f7cc61b2 100644 --- a/scripts/notebooks/pybind11_examples/example_ControlPlane.ipynb +++ b/scripts/notebooks/pybind11_examples/example_ControlPlane.ipynb @@ -15,24 +15,16 @@ "cell_type": "code", "execution_count": 1, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0.1.1\n" - ] - } - ], + "outputs": [], "source": [ "import sys\n", "\n", "## IMPORTANT: Update the path to your built Python module\n", "sys.path.append(\n", - " '/home/ubuntu/E2SAR/build/src/pybind')\n", + " '/home/ubuntu/dev-e2sar/build/src/pybind')\n", "\n", "import e2sar_py\n", - "print(e2sar_py.get_version())" + "# print(e2sar_py.get_version())" ] }, { @@ -57,7 +49,7 @@ "source": [ "#### \"Timestamp\" class\n", "\n", - "The Python `e2sar_py.Timestamp` class acts as a bridge to the C++ `google::protobuf::Timestamp` class. The following code demonstrates how to use it." + "The Python `e2sar_py.ControlPlane.Timestamp` class serves as a bridge to the C++ `google::protobuf::Timestamp` class. The following code illustrates how to use it. Note that the object is printed as a string." ] }, { @@ -72,7 +64,7 @@ "Defaulting to user installation because normal site-packages is not writeable\n", "Requirement already satisfied: protobuf in /home/ubuntu/.local/lib/python3.10/site-packages (5.28.1)\n", "Note: you may need to restart the kernel to use updated packages.\n", - "Timestamp: seconds = 1726512856, nanos = 809628000\n" + "Timestamp: 2024-09-18T22:24:39.348087Z, seconds = 1726698279, nanos = 348087000\n" ] } ], @@ -85,7 +77,7 @@ "from e2sar_py.ControlPlane import Timestamp\n", "\n", "\n", - "def get_timestamp_from_gts() -> e2sar_py.ControlPlane.Timestamp:\n", + "def get_currtimestamp_from_gts() -> e2sar_py.ControlPlane.Timestamp:\n", " g_ts = gts()\n", " g_ts.GetCurrentTime()\n", " curr_ts = Timestamp()\n", @@ -93,8 +85,8 @@ " curr_ts.set_nanos(g_ts.nanos)\n", " return curr_ts\n", "\n", - "ts = get_timestamp_from_gts()\n", - "print(f\"Timestamp: seconds = {ts.get_seconds()}, nanos = {ts.get_nanos()}\")" + "ts = get_currtimestamp_from_gts()\n", + "print(f\"Timestamp: {ts}, seconds = {ts.get_seconds()}, nanos = {ts.get_nanos()}\")" ] }, { @@ -103,7 +95,7 @@ "source": [ "#### \"LBWorkerStatus\" class\n", "\n", - "The following code block demonstrates how to manipulate the `ControlPlane.LBWorkerStatus` class." + "The following code block demonstrates how to manipulate the `ControlPlane.LBWorkerStatus` class. The `last_updated` attribute is a string." ] }, { @@ -119,10 +111,7 @@ "Worker fill percent: 75.5\n", "Worker control signal: 0.8999999761581421\n", "Worker slots assigned: 10\n", - "Worker last_updated (following 2 lines):\n", - "seconds: 1726512856\n", - "nanos: 809628000\n", - "\n" + "Worker last updated: 2024-09-18T22:24:39.348087Z\n" ] } ], @@ -133,14 +122,17 @@ "print(f\"Worker fill percent: {worker.fill_percent}\")\n", "print(f\"Worker control signal: {worker.control_signal}\")\n", "print(f\"Worker slots assigned: {worker.slots_assigned}\")\n", - "print(f\"Worker last_updated (following 2 lines):\\n{worker.last_updated}\")" + "print(f\"Worker last updated: {worker.last_updated}\")\n", + "\n", + "assert(worker.last_updated == str(ts))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "#### \"LBStatus\" class" + "#### \"LBStatus\" class\n", + "The attributes `timestamp` and `expiresAt` are bound as Python strings." ] }, { @@ -152,22 +144,32 @@ "name": "stdout", "output_type": "stream", "text": [ - "1234\n", - "['192.168.100.1', '192.168.100.2']\n", - "[]\n" + "Timestamp: 2024-09-18T22:24:39.348087Z\n", + "Expires at: 2024-09-18T23:24:39.348087Z\n" ] } ], "source": [ "ip_list = [\"192.168.100.1\", \"192.168.100.2\"]\n", "\n", + "# Set the expire timestamp\n", + "expire_ts = Timestamp() # DO NOT decalre as \"expire_ts = ts\"\n", + "expire_ts.set_seconds(ts.get_seconds() + 3600) # 1 hr\n", + "expire_ts.set_nanos(ts.get_nanos()) \n", + "\n", "# Create an LBStatus object with empty WorkerStatus list\n", - "status = e2sar_py.ControlPlane.LBStatus(ts, 1234, 5678, [], ip_list, ts)\n", + "status = e2sar_py.ControlPlane.LBStatus(ts, 1234, 5678, [], ip_list, expire_ts)\n", "\n", "# Access members\n", - "print(status.currentEpoch)\n", - "print(status.senderAddresses)\n", - "print(status.workers)" + "assert(status.timestamp == str(ts))\n", + "assert(status.currentEpoch == 1234)\n", + "assert(status.currentPredictedEventNumber == 5678)\n", + "assert(status.senderAddressList == ip_list)\n", + "assert(status.workerStatusList == [])\n", + "assert(status.expiresAt == str(expire_ts))\n", + "\n", + "print(\"Timestamp: \", status.timestamp)\n", + "print(\"Expires at: \", status.expiresAt)" ] }, { @@ -176,15 +178,14 @@ "source": [ "#### \"OverviewEntry\" class\n", "\n", - "The members of this class are read-only. Its usage will be demonstrated in `LBMLiveTest6`." + "The attributes of this class are read-only. Its usage will be demonstrated in `LBMLiveTest6`." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## \"LBManager\" class\n", - "Intialize `LBManager`." + "## \"LBManager\" class" ] }, { @@ -200,11 +201,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Tests with a mock EJFAT Load Balancer\n", + "### Examples with a mock EJFAT Load Balancer\n", "\n", - "This section requires a functional (mock) load balancer. Ensure the `udplbd` container is running on the FABRIC `cpnode` before executing the code blocks.\n", + "This section requires a functional (mock) load balancer. If you are launching with a FABRIC slice, ensure the `udplbd` container is running on the FABRIC `cpnode` before executing the code blocks.\n", "\n", - "The Python code blocks below replicate the tests conducted in the C++ `e2sar_lbcp_live_test`." + "The Python code blocks below replicate the tests performed in the C++ `e2sar_lbcp_live_test`." ] }, { @@ -327,7 +328,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Instance token: 9b0a986e06d5983e79e71904a6dba3fdb3aef7065d4b81e639c7ea66fedc2b45\n" + "Instance token: af539d34168742fcb7885b500ce36c69bd78c53ad7ce42e6ae8bc050e9a67929\n" ] } ], @@ -398,7 +399,7 @@ "output_type": "stream", "text": [ "Sync IPv4 addr is: 192.168.0.3\n", - "LB id is: 16\n" + "LB id is: 32\n" ] } ], @@ -554,7 +555,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "LB ID is: 17\n" + "LB ID is: 33\n" ] } ], @@ -677,8 +678,8 @@ "\n", "Send state succeeded\n", "\n", - "Session id is: 9\n", - "Session token is: 53d47321eecfc42cdb7999649c9bc9bd197d39e3b32830e5a3028bc83cfba473\n", + "Session id is: 16\n", + "Session token is: 5eda10833ab10f3f322f8026615bb6da872052993a2dfc0ce36bb08b14216b50\n", "\n", "Deregister worker succeeded\n" ] @@ -795,7 +796,7 @@ "\n", "Send state for 25 times\n", "\n", - "Get LB status succeeded: \n" + "Get LB status succeeded: \n" ] } ], @@ -840,48 +841,96 @@ "cell_type": "code", "execution_count": 30, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Worker status:\n", + " name: my_node\n", + " fill_percent: 0.800000011920929\n", + " control_signal: 1.0\n", + " slots_assigned: 512\n", + " last_updated: 2024-09-18T22:24:42.738418623Z\n" + ] + } + ], "source": [ "# Get worker status\n", "workers = lbm.get_worker_statuses(status_res)\n", "\n", - "assert len(workers) == 1" + "assert len(workers) == 1\n", + "\n", + "def print_workerstatus(w : e2sar_py.ControlPlane.WorkerStatus):\n", + " print(f\"Worker status:\")\n", + " print(f\" name: {w.get_name()}\")\n", + " print(f\" fill_percent: {w.get_fill_percent()}\")\n", + " print(f\" control_signal: {w.get_control_signal()}\")\n", + " print(f\" slots_assigned: {w.get_slots_assigned()}\")\n", + " print(f\" last_updated: {w.get_last_updated()}\")\n", + "\n", + "# print a worker's status\n", + "print_workerstatus(workers[0])" ] }, { "cell_type": "code", "execution_count": 31, "metadata": {}, + "outputs": [], + "source": [ + "assert(workers[0].get_name() == \"my_node\")\n", + "\n", + "DELTA = 0.000001\n", + "assert(abs(workers[0].get_fill_percent() - 0.8) < DELTA)\n", + "assert(abs(workers[0].get_control_signal() - 1) < DELTA)" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Timestamp: seconds: 1726512860\n", - "nanos: 102211657\n", - "\n", - "expiresAt: seconds: 1726516457\n", - "\n", + "Timestamp: 2024-09-18T22:24:42.840649988Z\n", + "expiresAt: 2024-09-18T23:24:40Z\n", "currentEpoch: 3\n", "currentPredictedEventNumber: 9223372036854775808\n", - "workers: []\n" + "workerStatusList: []\n" ] } ], "source": [ "lb_status = lbm.as_lb_status(status_res)\n", - "assert(lb_status.senderAddresses == ip_list)\n", - "assert(len(lb_status.workers) == 1)\n", + "dir(lb_status)\n", + "assert(lb_status.senderAddressList == ip_list)\n", + "assert(len(lb_status.workerStatusList) == 1)\n", "print(\"Timestamp: \", lb_status.timestamp)\n", "print(\"expiresAt: \", lb_status.expiresAt)\n", "print(\"currentEpoch: \", lb_status.currentEpoch)\n", "print(\"currentPredictedEventNumber: \", lb_status.currentPredictedEventNumber)\n", - "print(\"workers: \", lb_status.workers)\n" + "print(\"workerStatusList: \", lb_status.workerStatusList)\n" ] }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 33, + "metadata": {}, + "outputs": [], + "source": [ + "DELTA = 0.000001\n", + "\n", + "assert(lb_status.workerStatusList[0].get_name() == \"my_node\")\n", + "assert(abs(lb_status.workerStatusList[0].get_fill_percent() - 0.8) < DELTA)\n", + "assert(abs(lb_status.workerStatusList[0].get_control_signal() - 1) < DELTA)" + ] + }, + { + "cell_type": "code", + "execution_count": 34, "metadata": {}, "outputs": [ { @@ -915,7 +964,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 35, "metadata": {}, "outputs": [ { @@ -925,16 +974,16 @@ "Create LBManager obj succeeded: 0\n", "Register worker succeeded\n", "\n", - "Session id is: 11\n", - "Session token is: 07589d480e99cdb41bf26b263e236eae381f0ba33feec5054c62a38a3c41f811\n", + "Session id is: 18\n", + "Session token is: 6367c9176ad83fbeaf43d4a53e36b0f01695a8d80789ecf1cd2e0c435e145369\n", "\n", "Send state for 25 times\n", "Add senders succeeded.\n", "\n", - "Get LB status succeeded: \n", + "Get LB status succeeded: \n", "Remove senders succeeded.\n", "\n", - "Get LB status succeeded: \n", + "Get LB status succeeded: \n", "\n", "Deregister worker succeeded\n", "Free LB succeeded!\n" @@ -1005,6 +1054,9 @@ "assert(res == ip_list)\n", "workers = lbm.get_worker_statuses(status_res)\n", "assert len(workers) == 1\n", + "assert(workers[0].get_name() == \"my_node\")\n", + "assert(abs(workers[0].get_fill_percent() - 0.8) < DELTA)\n", + "assert(abs(workers[0].get_control_signal() - 1) < DELTA)\n", "\n", "# Remove senders\n", "res = lbm.remove_senders(ip_list)\n", @@ -1042,7 +1094,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 36, "metadata": {}, "outputs": [ { @@ -1076,7 +1128,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 37, "metadata": {}, "outputs": [ { @@ -1086,25 +1138,22 @@ "Create LBManager obj succeeded: 0\n", "Register worker succeeded\n", "\n", - "Session id is: 14\n", - "Session token is: 24409feb705a6ae6a6ef7dce008a7c726fa6134bba00e19e47ca6d342d81d162\n", + "Session id is: 19\n", + "Session token is: b91e53badc06c7547526b6a2a1e3c45283c7033e575749a276d2aca27dbad63a\n", "\n", "Send state for 25 times\n", "\n", - "Get LB status succeeded: \n", + "Get LB status succeeded: \n", "Sender addresses: ['192.168.20.1', '192.168.20.2']\n", "Current Epoch: 2\n", "Current predicted Event No: 9223372036854775808\n", - "Workers: []\n", - "Timestamp: seconds: 1726512975\n", - "nanos: 658369981\n", - "\n", - "expiresAt: seconds: 1726516573\n", - "\n", + "Workers: []\n", + "Timestamp: 2024-09-18T22:24:48.319546597Z\n", + "expiresAt: 2024-09-18T23:24:45Z\n", "\n", "Send state for 25 times\n", "\n", - "LB id: 23\n", + "LB id: 37\n", "1\n" ] } @@ -1171,15 +1220,17 @@ "\n", "# as_lb_status() usage\n", "res = lbm.as_lb_status(status_res)\n", - "print(f\"Sender addresses: {res.senderAddresses}\")\n", + "print(f\"Sender addresses: {res.senderAddressList}\")\n", "print(f\"Current Epoch: {res.currentEpoch}\")\n", "print(f\"Current predicted Event No: {res.currentPredictedEventNumber}\")\n", - "print(f\"Workers: {res.workers}\")\n", + "print(f\"Workers: {res.workerStatusList}\")\n", "print(f\"Timestamp: {res.timestamp}\")\n", "print(f\"expiresAt: {res.expiresAt}\")\n", - "assert(res.senderAddresses == ip_list)\n", - "# print(res.workers, dir(res.workers))\n", - "assert(len(res.workers) == 1)\n", + "assert(res.senderAddressList == ip_list)\n", + "assert(len(res.workerStatusList) == 1)\n", + "assert(res.workerStatusList[0].get_name() == \"my_node\")\n", + "assert(abs(res.workerStatusList[0].get_fill_percent() - 0.8) < DELTA)\n", + "assert(abs(res.workerStatusList[0].get_control_signal() - 1) < DELTA)\n", "\n", "print(\"\\nSend state for 25 times\")\n", "for i in range(25):\n", @@ -1203,13 +1254,13 @@ " free_lbmanager(lbm)\n", "print(\"LB id: \", res[0].lb_id)\n", "assert(res[0].name == \"mylb\")\n", - "assert(res[0].lb_status.senderAddresses == ip_list)\n", - "print(len(res[0].lb_status.workers))" + "assert(res[0].lb_status.senderAddressList == ip_list)\n", + "print(len(res[0].lb_status.workerStatusList))" ] }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 38, "metadata": {}, "outputs": [ { diff --git a/scripts/notebooks/pybind11_examples/example_DataPlane.ipynb b/scripts/notebooks/pybind11_examples/example_DataPlane.ipynb index 79b135a2..8e95f1a6 100644 --- a/scripts/notebooks/pybind11_examples/example_DataPlane.ipynb +++ b/scripts/notebooks/pybind11_examples/example_DataPlane.ipynb @@ -20,7 +20,7 @@ "\n", "## IMPORTANT: Update the path to your built Python module. Use the absolute path to make life easier.\n", "sys.path.append(\n", - " '/home/ubuntu/E2SAR/build/src/pybind')\n", + " '/home/ubuntu/dev-e2sar/build/src/pybind')\n", "\n", "import e2sar_py" ] diff --git a/scripts/notebooks/pybind11_examples/example_EjfatURI.ipynb b/scripts/notebooks/pybind11_examples/example_EjfatURI.ipynb index 2e51157c..b27ee6e2 100644 --- a/scripts/notebooks/pybind11_examples/example_EjfatURI.ipynb +++ b/scripts/notebooks/pybind11_examples/example_EjfatURI.ipynb @@ -27,11 +27,11 @@ "\n", "# IMPORTANT: Adjust the path to your built Python module as necessary\n", "sys.path.append(\n", - " '/home/ubuntu/E2SAR/build/src/pybind')\n", + " '/home/ubuntu/dev-e2sar/build/src/pybind')\n", "\n", "import e2sar_py\n", "\n", - "print(e2sar_py.get_version())" + "# print(e2sar_py.get_version())" ] }, { diff --git a/scripts/notebooks/pybind11_examples/example_Headers.ipynb b/scripts/notebooks/pybind11_examples/example_Headers.ipynb index f91a9015..7105219a 100644 --- a/scripts/notebooks/pybind11_examples/example_Headers.ipynb +++ b/scripts/notebooks/pybind11_examples/example_Headers.ipynb @@ -11,47 +11,27 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0.1.1\n" - ] - } - ], + "outputs": [], "source": [ "import sys\n", "\n", "## IMPORTANT: Update the path to your built Python module. Prefer absolute path.\n", "sys.path.append(\n", - " '/home/ubuntu/E2SAR/build/src/pybind')\n", + " '/home/ubuntu/dev-e2sar/build/src/pybind')\n", "\n", "import e2sar_py\n", "\n", "# Get the version\n", - "print(e2sar_py.get_version())\n" + "# print(e2sar_py.get_version())\n" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Default data plane port: 19522\n", - "Default Reassembler Header version: 1\n", - "Default Reassembler Header nibble: 16\n", - "Default Load Balancer Header version: 2\n", - "Default Sync Header version: 1\n" - ] - } - ], + "outputs": [], "source": [ "# Print the constant atrributes\n", "print(f\"Default data plane port: {e2sar_py._dp_port}\")\n", @@ -63,20 +43,9 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "IP header length: 20\n", - "UDP header length: 8\n", - "Total header length: 64\n", - "LB + RE header length: 36\n" - ] - } - ], + "outputs": [], "source": [ "\n", "# The Hdr lengths\n", @@ -97,22 +66,9 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Before setting fields: (0, 0, 0, 0)\n", - "After setting fields: (1, 2, 4, 8)\n", - " data_id=1\n", - " buff_off=2\n", - " buff_len=4\n", - " event_num=8\n" - ] - } - ], + "outputs": [], "source": [ "# Setting the Reaasembler header fields\n", "rehdr = e2sar_py.REHdr()\n", @@ -140,22 +96,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Before (2, 1, 0, 0)\n", - "2\n", - "1\n", - "0\n", - "0\n", - "After (2, 1, 200, 50)\n" - ] - } - ], + "outputs": [], "source": [ "# Load balancer header\n", "lbhdr = e2sar_py.LBHdr()\n", diff --git a/src/pybind/py_e2sarCP.cpp b/src/pybind/py_e2sarCP.cpp index 82221793..bae545bf 100644 --- a/src/pybind/py_e2sarCP.cpp +++ b/src/pybind/py_e2sarCP.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include "e2sarUtil.hpp" #include "e2sarCP.hpp" @@ -47,7 +48,16 @@ void init_e2sarCP(py::module_ &m) { // Expose the grpc classes py::class_(e2sarCP, "WorkerStatus") - .def(py::init<>()); + .def(py::init<>()) + .def("get_name", &WorkerStatus::name) + .def("get_fill_percent", &WorkerStatus::fillpercent) + .def("get_control_signal", &WorkerStatus::controlsignal) + .def("get_slots_assigned", &WorkerStatus::slotsassigned) + .def("get_last_updated", + [](const WorkerStatus &self) { + return google::protobuf::util::TimeUtil::ToString(self.lastupdated()); + } + ); py::class_(e2sarCP, "LoadBalancerStatusReply") .def(py::init<>()); py::class_(e2sarCP, "OverviewReply") @@ -58,7 +68,11 @@ void init_e2sarCP(py::module_ &m) { .def("get_seconds", &google::protobuf::Timestamp::seconds) .def("get_nanos", &google::protobuf::Timestamp::nanos) .def("set_seconds", &google::protobuf::Timestamp::set_seconds) - .def("set_nanos", &google::protobuf::Timestamp::set_nanos); + .def("set_nanos", &google::protobuf::Timestamp::set_nanos) + // pretty print in Python: use TimeUtil::ToString to get the default string representation + .def("__str__", [](const google::protobuf::Timestamp& self) { + return google::protobuf::util::TimeUtil::ToString(self); + }); /** * Bindings for struct "LBWorkerStatus" @@ -78,12 +92,8 @@ void init_e2sarCP(py::module_ &m) { .def_readonly("slots_assigned", &LBWorkerStatus::slotsAssigned) .def_property_readonly("last_updated", [](const LBWorkerStatus &self) { - return convert_timestamp_to_python(self.lastUpdated); // Access the member directly + return google::protobuf::util::TimeUtil::ToString(self.lastUpdated); // Access the member directly } - // , - // [](LBWorkerStatus &self, py::object py_timestamp) { - // self.lastUpdated = convert_timestamp_to_cpp(py_timestamp); // Assign the converted value - // } ); /** @@ -101,32 +111,25 @@ void init_e2sarCP(py::module_ &m) { std::vector&, google::protobuf::Timestamp>(), py::arg("timestamp"), - py::arg("currentEpoch"), py::arg("currentPredictedEventNumber"), - py::arg("workers"), + py::arg("current_epoch"), + py::arg("current_predicted_event_number"), + py::arg("worker_status_list"), py::arg("sender_addresses"), - py::arg("expiresAt")) + py::arg("expires_at")) // Expose the struct members - .def_property("timestamp", + .def_property_readonly(/* a Python string */"timestamp", [](const LBStatus &self) { - return convert_timestamp_to_python(self.timestamp); // Access the member directly - }, - [](LBStatus &self, py::object py_timestamp) { - self.timestamp = convert_timestamp_to_cpp(py_timestamp); // Assign the converted value + return google::protobuf::util::TimeUtil::ToString(self.timestamp); }) .def_readonly("currentEpoch", &LBStatus::currentEpoch) .def_readonly("currentPredictedEventNumber", &LBStatus::currentPredictedEventNumber) - .def_readonly("workers", &LBStatus::workers) - .def_readonly("senderAddresses", &LBStatus::senderAddresses) - .def_property_readonly("expiresAt", + .def_readonly("workerStatusList", &LBStatus::workers) + .def_readonly("senderAddressList", &LBStatus::senderAddresses) + .def_property_readonly(/* a Python string */"expiresAt", [](const LBStatus &self) { - return convert_timestamp_to_python(self.expiresAt); // Access the member directly - } - // , - // [](LBStatus &self, py::object py_timestamp) { - // self.expiresAt= convert_timestamp_to_cpp(py_timestamp); // Assign the converted value - // } - ); + return google::protobuf::util::TimeUtil::ToString(self.expiresAt); // Access the member directly + }); /** * Bindings for struct "OverviewEntry" From 610c64d16cff4c4259cbc4349b8ce7aa42d87eaf Mon Sep 17 00:00:00 2001 From: Xinxin Mei Date: Thu, 19 Sep 2024 01:29:15 +0000 Subject: [PATCH 06/73] Match DP C++ updates in v0.1.2 --- .../pybind11_examples/example_DataPlane.ipynb | 90 +++++++++++-------- src/pybind/py_e2sar.cpp | 2 + src/pybind/py_e2sarDP.cpp | 12 ++- 3 files changed, 61 insertions(+), 43 deletions(-) diff --git a/scripts/notebooks/pybind11_examples/example_DataPlane.ipynb b/scripts/notebooks/pybind11_examples/example_DataPlane.ipynb index 8e95f1a6..3177e0fe 100644 --- a/scripts/notebooks/pybind11_examples/example_DataPlane.ipynb +++ b/scripts/notebooks/pybind11_examples/example_DataPlane.ipynb @@ -43,7 +43,8 @@ "metadata": {}, "outputs": [], "source": [ - "DP_IPV4 = \"127.0.0.1\"\n", + "DP_IPV4_ADDR = \"127.0.0.1\"\n", + "DP_IPV4_PORT = 10000\n", "data_id = 0x0505 # decimal value: 1085\n", "eventSrc_id = 0x11223344 # decimal value: 287454020" ] @@ -79,7 +80,7 @@ ], "source": [ "# Set up URI for segmenter\n", - "SEG_URI = f\"ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data={DP_IPV4}\"\n", + "SEG_URI = f\"ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data={DP_IPV4_ADDR}:{DP_IPV4_PORT}\"\n", "seg_uri = e2sar_py.EjfatURI(uri=SEG_URI, tt=e2sar_py.EjfatURI.TokenType.instance)\n", "\n", "# Set up sflags\n", @@ -163,7 +164,6 @@ " epoch_ms = 1000\n", " setPoint = 0.0\n", " [Kp, Ki, Kd] = [0.0, 0.0, 0.0]\n", - " dpV6 = False\n", " cpV6 = False\n", " portRange = -1\n", " withLBHeader = True\n" @@ -172,7 +172,7 @@ ], "source": [ "# Set the reassembler URI\n", - "REAS_URI_ = f\"ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data={DP_IPV4}\"\n", + "REAS_URI_ = f\"ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data={DP_IPV4_ADDR}\"\n", "reas_uri = e2sar_py.EjfatURI(uri=REAS_URI_, tt=e2sar_py.EjfatURI.TokenType.instance)\n", "\n", "# Make sure the token matches the one in the string\n", @@ -204,14 +204,9 @@ "print(f\" epoch_ms = {rflags.epoch_ms}\")\n", "print(f\" setPoint = {rflags.setPoint}\")\n", "print(f\" [Kp, Ki, Kd] = [{rflags.Kp}, {rflags.Ki}, {rflags.Kd}]\")\n", - "print(f\" dpV6 = {rflags.dpV6}\")\n", "print(f\" cpV6 = {rflags.cpV6}\")\n", "print(f\" portRange = {rflags.portRange}\")\n", - "print(f\" withLBHeader = {rflags.withLBHeader}\")\n", - "\n", - "# Init the reassembler object\n", - "# print(type(reas_uri), type(1), type(rflags))\n", - "reas = e2sar_py.DataPlane.Reassembler(reas_uri, 1, rflags)\n" + "print(f\" withLBHeader = {rflags.withLBHeader}\")" ] }, { @@ -219,6 +214,17 @@ "execution_count": 8, "metadata": {}, "outputs": [], + "source": [ + "# Init the reassembler object\n", + "reas = e2sar_py.DataPlane.Reassembler(\n", + " reas_uri, e2sar_py.IPAddress.from_string(DP_IPV4_ADDR), DP_IPV4_PORT, 1, rflags)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], "source": [ "\n", "res = reas.OpenAndStart() # the DP address must be available\n", @@ -227,7 +233,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -252,7 +258,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -268,7 +274,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -308,7 +314,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -374,7 +380,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -411,17 +417,17 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ - "SEG_URI2 = f\"ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data={DP_IPV4}\"\n", + "SEG_URI2 = f\"ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data={DP_IPV4_ADDR}:{DP_IPV4_PORT}\"\n", "seg_uri2 = e2sar_py.EjfatURI(uri=SEG_URI2, tt=e2sar_py.EjfatURI.TokenType.instance)" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -439,7 +445,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 17, "metadata": {}, "outputs": [ { @@ -480,7 +486,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -545,7 +551,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -563,7 +569,8 @@ "# Create reassembler with 1 recv thread\n", "\n", "rflags_m = e2sar_py.DataPlane.ReassemblerFlags()\n", - "reas_a = e2sar_py.DataPlane.Reassembler(reas_uri, 1, rflags_m)\n", + "reas_a = e2sar_py.DataPlane.Reassembler(\n", + " reas_uri, e2sar_py.IPAddress.from_string(DP_IPV4_ADDR), 19522, 1, rflags_m)\n", "\n", "print(\"This reassembler has\")\n", "print(f\" {reas_a.get_numRecvThreads()} threads;\")\n", @@ -577,7 +584,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -593,7 +600,8 @@ ], "source": [ "# Create reassembler with 4 recv threads\n", - "reas_b = e2sar_py.DataPlane.Reassembler(reas_uri, 4, rflags_m)\n", + "reas_b = e2sar_py.DataPlane.Reassembler(\n", + " reas_uri, e2sar_py.IPAddress.from_string(DP_IPV4_ADDR), 19522, 4, rflags_m)\n", "\n", "print(\"This reassembler has\")\n", "print(f\" {reas_b.get_numRecvThreads()} threads;\")\n", @@ -607,7 +615,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -623,7 +631,8 @@ ], "source": [ "# Create reassembler with 7 recv threads\n", - "reas_c = e2sar_py.DataPlane.Reassembler(reas_uri, 7, rflags_m)\n", + "reas_c = e2sar_py.DataPlane.Reassembler(\n", + " reas_uri, e2sar_py.IPAddress.from_string(DP_IPV4_ADDR), 19522, 7, rflags_m)\n", "\n", "print(\"This reassembler has\")\n", "print(f\" {reas_c.get_numRecvThreads()} threads;\")\n", @@ -637,7 +646,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 22, "metadata": {}, "outputs": [ { @@ -655,7 +664,8 @@ "# 4 threads with portRange override\n", "rflags_m.portRange = 10\n", "\n", - "reas_d = e2sar_py.DataPlane.Reassembler(reas_uri, 4, rflags_m)\n", + "reas_d = e2sar_py.DataPlane.Reassembler(\n", + " reas_uri, e2sar_py.IPAddress.from_string(DP_IPV4_ADDR), 19522, 4, rflags_m)\n", "\n", "print(\"This reassembler has\")\n", "print(f\" {reas_d.get_numRecvThreads()} threads;\")\n", @@ -669,7 +679,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 23, "metadata": {}, "outputs": [ { @@ -687,7 +697,8 @@ "# 4 threads with low portRange override\n", "rflags_m.portRange = 1\n", "\n", - "reas_e = e2sar_py.DataPlane.Reassembler(reas_uri, 4, rflags_m)\n", + "reas_e = e2sar_py.DataPlane.Reassembler(\n", + " reas_uri, e2sar_py.IPAddress.from_string(DP_IPV4_ADDR), 19522, 4, rflags_m)\n", "\n", "print(\"This reassembler has\")\n", "print(f\" {reas_e.get_numRecvThreads()} threads;\")\n", @@ -710,7 +721,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 25, "metadata": {}, "outputs": [ { @@ -731,7 +742,7 @@ "source": [ "# Create 4 segmenter objects\n", "\n", - "SEG_URI_BASE = SEG_URI\n", + "SEG_URI_BASE = f\"ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data={DP_IPV4_ADDR}\"\n", "SEG_URI_PORT_BASE = 19522\n", "\n", "sflags = e2sar_py.DataPlane.SegmenterFlags()\n", @@ -762,7 +773,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 26, "metadata": {}, "outputs": [ { @@ -784,7 +795,8 @@ "rflags.withLBHeader = True # LB header will be attached since there is no LB\n", "rflags.portRange = 2 # use 2^portRange=4 ports\n", "\n", - "reas = e2sar_py.DataPlane.Reassembler(reas_uri, 1, rflags)\n", + "reas = e2sar_py.DataPlane.Reassembler(\n", + " reas_uri, e2sar_py.IPAddress.from_string(DP_IPV4_ADDR), 19522, 1, rflags)\n", "print(\"This reassembler has\")\n", "print(f\" {reas.get_numRecvThreads()} threads;\")\n", "print(f\" listening on ports {reas.get_recvPorts()[0]}:{reas.get_recvPorts()[1]};\")\n", @@ -793,7 +805,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 27, "metadata": {}, "outputs": [ { @@ -819,7 +831,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -832,7 +844,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 29, "metadata": {}, "outputs": [], "source": [ @@ -845,7 +857,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 30, "metadata": {}, "outputs": [ { @@ -893,7 +905,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 31, "metadata": {}, "outputs": [ { diff --git a/src/pybind/py_e2sar.cpp b/src/pybind/py_e2sar.cpp index 928771fe..78326e65 100644 --- a/src/pybind/py_e2sar.cpp +++ b/src/pybind/py_e2sar.cpp @@ -132,4 +132,6 @@ void init_e2sarResultTypes(py::module_ &m) bind_result(m, "E2SARResultUInit32"); bind_result>(m, "E2SARResultPairIP"); bind_result>(m, "E2SARResultPairString"); + bind_result(m, "E2SARResultReassemblerFlags"); + bind_result(m, "E2SARResultSegmenterFlags"); } diff --git a/src/pybind/py_e2sarDP.cpp b/src/pybind/py_e2sarDP.cpp index 82555fc0..a9e0700a 100644 --- a/src/pybind/py_e2sarDP.cpp +++ b/src/pybind/py_e2sarDP.cpp @@ -77,7 +77,9 @@ void init_e2sarDP_segmenter(py::module_ &m) .def_readwrite("syncPeriodMs", &Segmenter::SegmenterFlags::syncPeriodMs) .def_readwrite("syncPeriods", &Segmenter::SegmenterFlags::syncPeriods) .def_readwrite("mtu", &Segmenter::SegmenterFlags::mtu) - .def_readwrite("numSendSockets", &Segmenter::SegmenterFlags::numSendSockets); + .def_readwrite("numSendSockets", &Segmenter::SegmenterFlags::numSendSockets) + .def_readwrite("sndSocketBufSize", &Segmenter::SegmenterFlags::sndSocketBufSize) + .def("getFromINI", &Segmenter::SegmenterFlags::getFromINI); // Constructor seg.def( @@ -172,15 +174,17 @@ void init_e2sarDP_reassembler(py::module_ &m) .def_readwrite("epoch_ms", &Reassembler::ReassemblerFlags::epoch_ms) .def_readwrite("portRange", &Reassembler::ReassemblerFlags::portRange) .def_readwrite("withLBHeader", &Reassembler::ReassemblerFlags::withLBHeader) - .def_readwrite("eventTimeout_ms", &Reassembler::ReassemblerFlags::eventTimeout_ms); + .def_readwrite("eventTimeout_ms", &Reassembler::ReassemblerFlags::eventTimeout_ms) + .def_readwrite("rcvSocketBufSize", &Reassembler::ReassemblerFlags::rcvSocketBufSize) + .def("getFromINI", &Reassembler::ReassemblerFlags::getFromINI); // Constructor reas.def( py::init(), "Init the Reassembler object with number of recv threads.", py::arg("uri"), // must-have arg when init - py::arg("ip"), - py::arg("port"), + py::arg("data_ip"), + py::arg("starting_port"), py::arg("num_recv_threads") = (size_t)1, py::arg("rflags") = Reassembler::ReassemblerFlags()); From 6a8d7c3a9defd6bbdcd4d5ea10dcd45878264e5b Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Thu, 19 Sep 2024 11:20:55 -0400 Subject: [PATCH 07/73] Made weight, min and max_factor settable via flags; added sane defaults --- bin/e2sar_perf.cpp | 68 ++++++++++++++++++++++++++++------ docs | 2 +- include/e2sarDPReassembler.hpp | 18 +++++---- include/e2sarNetUtil.hpp | 5 +++ reassembler_config.ini | 12 +++++- src/e2sarDPReassembler.cpp | 9 ++++- src/e2sarNetUtil.cpp | 16 +++++++- 7 files changed, 105 insertions(+), 25 deletions(-) diff --git a/bin/e2sar_perf.cpp b/bin/e2sar_perf.cpp index 51164171..0d0c4603 100644 --- a/bin/e2sar_perf.cpp +++ b/bin/e2sar_perf.cpp @@ -32,15 +32,28 @@ bool threadsRunning = true; u_int16_t reportThreadSleepMs{1000}; Reassembler *reasPtr{nullptr}; Segmenter *segPtr{nullptr}; +LBManager *lbmPtr{nullptr}; +std::vector senders; void ctrlCHandler(int sig) { std::cout << "Stopping threads" << std::endl; - if (segPtr != nullptr) + if (segPtr != nullptr) { + if (lbmPtr != nullptr) { + auto rmres = lbmPtr->removeSenders(senders); + if (rmres.has_error()) + std::cerr << "Unable to remove sender from list on exit: " << rmres.error().message() << std::endl; + } segPtr->stopThreads(); + } if (reasPtr != nullptr) + { + auto deregres = reasPtr->deregisterWorker(); + if (deregres.has_error()) + std::cerr << "Unable to deregister worker on exit: " << deregres.error().message() << std::endl; reasPtr->stopThreads(); + } threadsRunning = false; // instead of join boost::chrono::milliseconds duration(1000); @@ -160,7 +173,17 @@ result recvEvents(Reassembler &r, int durationSec) { auto nowT = boost::chrono::steady_clock::now(); // register the worker (will be NOOP if withCP is set to false) - //r.registerWorker(name, weight, min_factor, max_factor); + auto hostname_res = NetUtil::getHostName(); + if (hostname_res.has_error()) + { + return E2SARErrorInfo{hostname_res.error().code(), hostname_res.error().message()}; + } + auto regres = r.registerWorker(hostname_res.value()); + if (regres.has_error()) + { + return E2SARErrorInfo{E2SARErrorc::RPCError, + "Unable to register worker node due to " + regres.error().message()}; + } // receive loop while(true) @@ -240,8 +263,8 @@ int main(int argc, char **argv) int sockBufSize; int durationSec; bool withCP; + std::string sndrcvIP; std::string iniFile; - std::string recvIP; u_int16_t recvStartPort; // parameters @@ -263,7 +286,7 @@ int main(int argc, char **argv) opts("withcp,c", po::bool_switch()->default_value(false), "enable control plane interactions"); opts("ini,i", po::value(&iniFile)->default_value(""), "INI file to initialize SegmenterFlags [s]] or ReassemblerFlags [r]." " Values found in the file override --withcp, --mtu and --bufsize"); - opts("ip", po::value(&recvIP)->default_value("127.0.0.1"), "IP address (IPv4 or IPv6) on which receiver listens. Defaults to 127.0.0.1. [r]"); + opts("ip", po::value(&sndrcvIP)->default_value("127.0.0.1"), "IP address (IPv4 or IPv6) from which sender sends from or on which receiver listens. Defaults to 127.0.0.1. [s,r]"); opts("port", po::value(&recvStartPort)->default_value(10000), "Starting UDP port number on which receiver listens. Defaults to 10000. [r] "); po::variables_map vm; @@ -288,6 +311,8 @@ int main(int argc, char **argv) conflicting_options(vm, "send", "period"); option_dependency(vm, "recv", "ip"); option_dependency(vm, "vm", "port"); + option_dependency(vm, "send", "ip"); + conflicting_options(vm, "recv", "port"); } catch (const std::logic_error &le) { @@ -319,13 +344,33 @@ int main(int argc, char **argv) } auto uri = uri_r.value(); if (vm.count("send")) { + // if using control plane + if (withCP) + { + // add to senders list of 1 element + senders.push_back(sndrcvIP); + + // create LBManager + lbmPtr = new LBManager(uri, false); + + // register senders + auto addres = lbmPtr->addSenders(senders); + if (addres.has_error()) + { + std::cerr << "Unable to add a sender due to error " << addres.error().message() + << ", exiting" << std::endl; + return -1; + } + } + Segmenter::SegmenterFlags sflags; - if (!vm["ini"].as().empty()) + if (!iniFile.empty()) { - auto sflagsRes = Segmenter::SegmenterFlags::getFromINI(vm["ini"].as()); + std::cout << "Loading SegmenterFlags from " << iniFile << std::endl; + auto sflagsRes = Segmenter::SegmenterFlags::getFromINI(iniFile); if (sflagsRes.has_error()) { - std::cerr << "Unable to parse SegmenterFlags INI file " << vm["ini"].as() << std::endl; + std::cerr << "Unable to parse SegmenterFlags INI file " << iniFile << std::endl; return -1; } sflags = sflagsRes.value(); @@ -353,12 +398,13 @@ int main(int argc, char **argv) } else if (vm.count("recv")) { Reassembler::ReassemblerFlags rflags; - if (!vm["ini"].as().empty()) + if (!iniFile.empty()) { - auto rflagsRes = Reassembler::ReassemblerFlags::getFromINI(vm["ini"].as()); + std::cout << "Loading ReassemblerFlags from " << iniFile << std::endl; + auto rflagsRes = Reassembler::ReassemblerFlags::getFromINI(iniFile); if (rflagsRes.has_error()) { - std::cerr << "Unable to parse ReassemblerFlags INI file " << vm["ini"].as() << std::endl; + std::cerr << "Unable to parse ReassemblerFlags INI file " << iniFile << std::endl; return -1; } rflags = rflagsRes.value(); @@ -373,7 +419,7 @@ int main(int argc, char **argv) "*** Make sure the URI reflects proper data address, other parts are ignored.") << std::endl; try { - ip::address ip = ip::make_address(vm["ip"].as()); + ip::address ip = ip::make_address(sndrcvIP); Reassembler reas(uri, ip, recvStartPort, numThreads, rflags); reasPtr = &reas; boost::thread statT(&recvStatsThread, &reas); diff --git a/docs b/docs index 42ed8636..719131ca 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 42ed8636c9383c3358b4d368dbaa53f9c5cad90b +Subproject commit 719131ca2c65911d4b420e9472faadc2fd367062 diff --git a/include/e2sarDPReassembler.hpp b/include/e2sarDPReassembler.hpp index 9de838c6..fe373a2e 100644 --- a/include/e2sarDPReassembler.hpp +++ b/include/e2sarDPReassembler.hpp @@ -166,6 +166,9 @@ namespace e2sar const float Kp; // PID proportional const float Ki; // PID integral const float Kd; // PID derivative + const float weight; // processing power factor + const float min_factor; + const float max_factor; struct PIDSample { UnixTimeMicro_t sampleTime; // in usec since epoch float error; @@ -334,6 +337,11 @@ namespace e2sar * - eventTimeout_ms - how long (in ms) we allow events to remain in assembly before we give up {500} * - rcvSocketBufSize - socket buffer size for receiving set via SO_RCVBUF setsockopt. Note * that this requires systemwide max set via sysctl (net.core.rmem_max) to be higher. {3MB} + * - weight - weight given to this node in terms of processing power + * - min_factor - multiplied with the number of slots that would be assigned evenly to determine min number of slots + * for example, 4 nodes with a minFactor of 0.5 = (512 slots / 4) * 0.5 = min 64 slots + * - max_factor - multiplied with the number of slots that would be assigned evenly to determine max number of slots + * for example, 4 nodes with a maxFactor of 2 = (512 slots / 4) * 2 = max 256 slots set to 0 to specify no maximum */ struct ReassemblerFlags { @@ -347,10 +355,11 @@ namespace e2sar bool withLBHeader; int eventTimeout_ms; int rcvSocketBufSize; + float weight, min_factor, max_factor; ReassemblerFlags(): cpV6{false}, useCP{true}, period_ms{100}, validateCert{true}, Ki{0.}, Kp{0.}, Kd{0.}, setPoint{0.}, epoch_ms{1000}, portRange{-1}, withLBHeader{false}, eventTimeout_ms{500}, - rcvSocketBufSize{1024*1024*3} {} + rcvSocketBufSize{1024*1024*3}, weight{1.0}, min_factor{0.5}, max_factor{2.0} {} /** * Initialize flags from an INI file * @param iniFile - path to the INI file @@ -405,14 +414,9 @@ namespace e2sar /** * Register a worker with the control plane * @param node_name - name of this node (any unique string) - * @param weight - weight given to this node in terms of processing power - * @param min_factor - multiplied with the number of slots that would be assigned evenly to determine min number of slots - * for example, 4 nodes with a minFactor of 0.5 = (512 slots / 4) * 0.5 = min 64 slots - * @param max_factor - multiplied with the number of slots that would be assigned evenly to determine max number of slots - * for example, 4 nodes with a maxFactor of 2 = (512 slots / 4) * 2 = max 256 slots set to 0 to specify no maximum * @return - 0 on success or an error condition */ - result registerWorker(const std::string &node_name, float weight, float min_factor, float max_factor) noexcept; + result registerWorker(const std::string &node_name) noexcept; /** * Deregister this worker diff --git a/include/e2sarNetUtil.hpp b/include/e2sarNetUtil.hpp index f60df460..461aa64c 100644 --- a/include/e2sarNetUtil.hpp +++ b/include/e2sarNetUtil.hpp @@ -29,6 +29,11 @@ namespace e2sar * @return MTU or 1500 as the best guess */ static u_int16_t getMTU(const std::string &interfaceName); + /** + * Get the hostname of the host + */ + static result getHostName(); + #ifdef NETLINK_CAPABLE /** * Get the outgoing interface and its MTU for a given IPv4 or IPv6 diff --git a/reassembler_config.ini b/reassembler_config.ini index 2fd829ed..bdce44bc 100644 --- a/reassembler_config.ini +++ b/reassembler_config.ini @@ -1,4 +1,4 @@ -[genera] +[general] ; whether to use the control plane (gRPC sendState, registerWorker) useCP = true ; validate control plane TLS certificate in gRPC communications @@ -33,4 +33,12 @@ setPoint = 0.0 ; PID gains (integral, proportional and derivative) Ki = 0.0 Kp = 0.0 -Kd = 0.0 \ No newline at end of file +Kd = 0.0 +; schedule weight parameters +weight = 1.0 +; multiplied with the number of slots that would be assigned evenly to determine min number of slots +; for example, 4 nodes with a minFactor of 0.5 = (512 slots / 4) * 0.5 = min 64 slots +min_factor = 0.5 +; multiplied with the number of slots that would be assigned evenly to determine max number of slots +; for example, 4 nodes with a maxFactor of 2 = (512 slots / 4) * 2 = max 256 slots set to 0 to specify no maximum +max_factor = 2.0 diff --git a/src/e2sarDPReassembler.cpp b/src/e2sarDPReassembler.cpp index d403e427..3b9c120c 100644 --- a/src/e2sarDPReassembler.cpp +++ b/src/e2sarDPReassembler.cpp @@ -41,6 +41,7 @@ namespace e2sar lbman(dpuri, rflags.validateCert), epochMs{rflags.epoch_ms}, setPoint{rflags.setPoint}, Kp{rflags.Kp}, Ki{rflags.Ki}, Kd{rflags.Kd}, + weight{rflags.weight}, min_factor{rflags.min_factor}, max_factor{rflags.max_factor}, pidSampleBuffer(rflags.epoch_ms/rflags.period_ms), // ring buffer size (usually 10 = 1sec/100ms) cpuCoreList{cpuCoreList}, dataIP{data_ip}, @@ -73,6 +74,7 @@ namespace e2sar lbman(dpuri, rflags.validateCert), epochMs{rflags.epoch_ms}, setPoint{rflags.setPoint}, Kp{rflags.Kp}, Ki{rflags.Ki}, Kd{rflags.Kd}, + weight{rflags.weight}, min_factor{rflags.min_factor}, max_factor{rflags.max_factor}, pidSampleBuffer(rflags.epoch_ms/rflags.period_ms), // ring buffer size (usually 10 = 1sec/100ms) cpuCoreList{std::vector()}, // no core list given dataIP{data_ip}, @@ -455,8 +457,7 @@ namespace e2sar } } - result Reassembler::registerWorker(const std::string &node_name, float weight, - float min_factor, float max_factor) noexcept + result Reassembler::registerWorker(const std::string &node_name) noexcept { if (useCP) { @@ -558,6 +559,10 @@ namespace e2sar rFlags.Ki = paramTree.get("pid.Ki", rFlags.Ki); rFlags.Kp = paramTree.get("pid.Kp", rFlags.Kp); rFlags.Kd = paramTree.get("pid.Kd", rFlags.Kd); + rFlags.Kd = paramTree.get("pid.weight", rFlags.Kd); + rFlags.Kd = paramTree.get("pid.min_factor", rFlags.Kd); + rFlags.Kd = paramTree.get("pid.max_factor", rFlags.Kd); + return rFlags; } diff --git a/src/e2sarNetUtil.cpp b/src/e2sarNetUtil.cpp index 4e228d1a..3b489f0f 100644 --- a/src/e2sarNetUtil.cpp +++ b/src/e2sarNetUtil.cpp @@ -3,7 +3,8 @@ namespace e2sar { /** - * Get MTU of a given interface + * Get MTU of a given interface. Used in constructors, so doesn't + * return error. */ u_int16_t NetUtil::getMTU(const std::string &interfaceName) { // Default MTU @@ -14,10 +15,21 @@ namespace e2sar strcpy(ifr.ifr_name, interfaceName.c_str()); if (!ioctl(sock, SIOCGIFMTU, &ifr)) { mtu = ifr.ifr_mtu; - } + } close(sock); return mtu; } + + result NetUtil::getHostName() { + char nameBuf[255]; + + if (!gethostname(nameBuf, 255)) + { + std::string ret{nameBuf}; + return ret; + } else + return E2SARErrorInfo{E2SARErrorc::SystemError, "Unable to retrieve hostname"}; + } #ifdef NETLINK_CAPABLE /** * Get the outgoing interface and its MTU for a given IPv4 or IPv6 From dee1aff8cc0db69f6e557f422a14037af7babec3 Mon Sep 17 00:00:00 2001 From: Xinxin Mei Date: Wed, 18 Sep 2024 16:48:45 +0000 Subject: [PATCH 08/73] Update Python bindings to match v0.1.2 Compiled on both Ubuntu 22.04 and RHEL 8.10 platform - Add bindings for members of gprc::WorkerStatus class - Pretty print of protobuf TimeStamp - Match DP C++ changes in v0.1.2 for both bindings and Python examples --- .../example_ControlPlane.ipynb | 209 +++++++++++------- .../pybind11_examples/example_DataPlane.ipynb | 92 ++++---- .../pybind11_examples/example_EjfatURI.ipynb | 4 +- .../pybind11_examples/example_Headers.ipynb | 81 +------ src/pybind/py_e2sar.cpp | 2 + src/pybind/py_e2sarCP.cpp | 53 ++--- src/pybind/py_e2sarDP.cpp | 12 +- 7 files changed, 234 insertions(+), 219 deletions(-) diff --git a/scripts/notebooks/pybind11_examples/example_ControlPlane.ipynb b/scripts/notebooks/pybind11_examples/example_ControlPlane.ipynb index 6eb45bab..f7cc61b2 100644 --- a/scripts/notebooks/pybind11_examples/example_ControlPlane.ipynb +++ b/scripts/notebooks/pybind11_examples/example_ControlPlane.ipynb @@ -15,24 +15,16 @@ "cell_type": "code", "execution_count": 1, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0.1.1\n" - ] - } - ], + "outputs": [], "source": [ "import sys\n", "\n", "## IMPORTANT: Update the path to your built Python module\n", "sys.path.append(\n", - " '/home/ubuntu/E2SAR/build/src/pybind')\n", + " '/home/ubuntu/dev-e2sar/build/src/pybind')\n", "\n", "import e2sar_py\n", - "print(e2sar_py.get_version())" + "# print(e2sar_py.get_version())" ] }, { @@ -57,7 +49,7 @@ "source": [ "#### \"Timestamp\" class\n", "\n", - "The Python `e2sar_py.Timestamp` class acts as a bridge to the C++ `google::protobuf::Timestamp` class. The following code demonstrates how to use it." + "The Python `e2sar_py.ControlPlane.Timestamp` class serves as a bridge to the C++ `google::protobuf::Timestamp` class. The following code illustrates how to use it. Note that the object is printed as a string." ] }, { @@ -72,7 +64,7 @@ "Defaulting to user installation because normal site-packages is not writeable\n", "Requirement already satisfied: protobuf in /home/ubuntu/.local/lib/python3.10/site-packages (5.28.1)\n", "Note: you may need to restart the kernel to use updated packages.\n", - "Timestamp: seconds = 1726512856, nanos = 809628000\n" + "Timestamp: 2024-09-18T22:24:39.348087Z, seconds = 1726698279, nanos = 348087000\n" ] } ], @@ -85,7 +77,7 @@ "from e2sar_py.ControlPlane import Timestamp\n", "\n", "\n", - "def get_timestamp_from_gts() -> e2sar_py.ControlPlane.Timestamp:\n", + "def get_currtimestamp_from_gts() -> e2sar_py.ControlPlane.Timestamp:\n", " g_ts = gts()\n", " g_ts.GetCurrentTime()\n", " curr_ts = Timestamp()\n", @@ -93,8 +85,8 @@ " curr_ts.set_nanos(g_ts.nanos)\n", " return curr_ts\n", "\n", - "ts = get_timestamp_from_gts()\n", - "print(f\"Timestamp: seconds = {ts.get_seconds()}, nanos = {ts.get_nanos()}\")" + "ts = get_currtimestamp_from_gts()\n", + "print(f\"Timestamp: {ts}, seconds = {ts.get_seconds()}, nanos = {ts.get_nanos()}\")" ] }, { @@ -103,7 +95,7 @@ "source": [ "#### \"LBWorkerStatus\" class\n", "\n", - "The following code block demonstrates how to manipulate the `ControlPlane.LBWorkerStatus` class." + "The following code block demonstrates how to manipulate the `ControlPlane.LBWorkerStatus` class. The `last_updated` attribute is a string." ] }, { @@ -119,10 +111,7 @@ "Worker fill percent: 75.5\n", "Worker control signal: 0.8999999761581421\n", "Worker slots assigned: 10\n", - "Worker last_updated (following 2 lines):\n", - "seconds: 1726512856\n", - "nanos: 809628000\n", - "\n" + "Worker last updated: 2024-09-18T22:24:39.348087Z\n" ] } ], @@ -133,14 +122,17 @@ "print(f\"Worker fill percent: {worker.fill_percent}\")\n", "print(f\"Worker control signal: {worker.control_signal}\")\n", "print(f\"Worker slots assigned: {worker.slots_assigned}\")\n", - "print(f\"Worker last_updated (following 2 lines):\\n{worker.last_updated}\")" + "print(f\"Worker last updated: {worker.last_updated}\")\n", + "\n", + "assert(worker.last_updated == str(ts))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "#### \"LBStatus\" class" + "#### \"LBStatus\" class\n", + "The attributes `timestamp` and `expiresAt` are bound as Python strings." ] }, { @@ -152,22 +144,32 @@ "name": "stdout", "output_type": "stream", "text": [ - "1234\n", - "['192.168.100.1', '192.168.100.2']\n", - "[]\n" + "Timestamp: 2024-09-18T22:24:39.348087Z\n", + "Expires at: 2024-09-18T23:24:39.348087Z\n" ] } ], "source": [ "ip_list = [\"192.168.100.1\", \"192.168.100.2\"]\n", "\n", + "# Set the expire timestamp\n", + "expire_ts = Timestamp() # DO NOT decalre as \"expire_ts = ts\"\n", + "expire_ts.set_seconds(ts.get_seconds() + 3600) # 1 hr\n", + "expire_ts.set_nanos(ts.get_nanos()) \n", + "\n", "# Create an LBStatus object with empty WorkerStatus list\n", - "status = e2sar_py.ControlPlane.LBStatus(ts, 1234, 5678, [], ip_list, ts)\n", + "status = e2sar_py.ControlPlane.LBStatus(ts, 1234, 5678, [], ip_list, expire_ts)\n", "\n", "# Access members\n", - "print(status.currentEpoch)\n", - "print(status.senderAddresses)\n", - "print(status.workers)" + "assert(status.timestamp == str(ts))\n", + "assert(status.currentEpoch == 1234)\n", + "assert(status.currentPredictedEventNumber == 5678)\n", + "assert(status.senderAddressList == ip_list)\n", + "assert(status.workerStatusList == [])\n", + "assert(status.expiresAt == str(expire_ts))\n", + "\n", + "print(\"Timestamp: \", status.timestamp)\n", + "print(\"Expires at: \", status.expiresAt)" ] }, { @@ -176,15 +178,14 @@ "source": [ "#### \"OverviewEntry\" class\n", "\n", - "The members of this class are read-only. Its usage will be demonstrated in `LBMLiveTest6`." + "The attributes of this class are read-only. Its usage will be demonstrated in `LBMLiveTest6`." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## \"LBManager\" class\n", - "Intialize `LBManager`." + "## \"LBManager\" class" ] }, { @@ -200,11 +201,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Tests with a mock EJFAT Load Balancer\n", + "### Examples with a mock EJFAT Load Balancer\n", "\n", - "This section requires a functional (mock) load balancer. Ensure the `udplbd` container is running on the FABRIC `cpnode` before executing the code blocks.\n", + "This section requires a functional (mock) load balancer. If you are launching with a FABRIC slice, ensure the `udplbd` container is running on the FABRIC `cpnode` before executing the code blocks.\n", "\n", - "The Python code blocks below replicate the tests conducted in the C++ `e2sar_lbcp_live_test`." + "The Python code blocks below replicate the tests performed in the C++ `e2sar_lbcp_live_test`." ] }, { @@ -327,7 +328,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Instance token: 9b0a986e06d5983e79e71904a6dba3fdb3aef7065d4b81e639c7ea66fedc2b45\n" + "Instance token: af539d34168742fcb7885b500ce36c69bd78c53ad7ce42e6ae8bc050e9a67929\n" ] } ], @@ -398,7 +399,7 @@ "output_type": "stream", "text": [ "Sync IPv4 addr is: 192.168.0.3\n", - "LB id is: 16\n" + "LB id is: 32\n" ] } ], @@ -554,7 +555,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "LB ID is: 17\n" + "LB ID is: 33\n" ] } ], @@ -677,8 +678,8 @@ "\n", "Send state succeeded\n", "\n", - "Session id is: 9\n", - "Session token is: 53d47321eecfc42cdb7999649c9bc9bd197d39e3b32830e5a3028bc83cfba473\n", + "Session id is: 16\n", + "Session token is: 5eda10833ab10f3f322f8026615bb6da872052993a2dfc0ce36bb08b14216b50\n", "\n", "Deregister worker succeeded\n" ] @@ -795,7 +796,7 @@ "\n", "Send state for 25 times\n", "\n", - "Get LB status succeeded: \n" + "Get LB status succeeded: \n" ] } ], @@ -840,48 +841,96 @@ "cell_type": "code", "execution_count": 30, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Worker status:\n", + " name: my_node\n", + " fill_percent: 0.800000011920929\n", + " control_signal: 1.0\n", + " slots_assigned: 512\n", + " last_updated: 2024-09-18T22:24:42.738418623Z\n" + ] + } + ], "source": [ "# Get worker status\n", "workers = lbm.get_worker_statuses(status_res)\n", "\n", - "assert len(workers) == 1" + "assert len(workers) == 1\n", + "\n", + "def print_workerstatus(w : e2sar_py.ControlPlane.WorkerStatus):\n", + " print(f\"Worker status:\")\n", + " print(f\" name: {w.get_name()}\")\n", + " print(f\" fill_percent: {w.get_fill_percent()}\")\n", + " print(f\" control_signal: {w.get_control_signal()}\")\n", + " print(f\" slots_assigned: {w.get_slots_assigned()}\")\n", + " print(f\" last_updated: {w.get_last_updated()}\")\n", + "\n", + "# print a worker's status\n", + "print_workerstatus(workers[0])" ] }, { "cell_type": "code", "execution_count": 31, "metadata": {}, + "outputs": [], + "source": [ + "assert(workers[0].get_name() == \"my_node\")\n", + "\n", + "DELTA = 0.000001\n", + "assert(abs(workers[0].get_fill_percent() - 0.8) < DELTA)\n", + "assert(abs(workers[0].get_control_signal() - 1) < DELTA)" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Timestamp: seconds: 1726512860\n", - "nanos: 102211657\n", - "\n", - "expiresAt: seconds: 1726516457\n", - "\n", + "Timestamp: 2024-09-18T22:24:42.840649988Z\n", + "expiresAt: 2024-09-18T23:24:40Z\n", "currentEpoch: 3\n", "currentPredictedEventNumber: 9223372036854775808\n", - "workers: []\n" + "workerStatusList: []\n" ] } ], "source": [ "lb_status = lbm.as_lb_status(status_res)\n", - "assert(lb_status.senderAddresses == ip_list)\n", - "assert(len(lb_status.workers) == 1)\n", + "dir(lb_status)\n", + "assert(lb_status.senderAddressList == ip_list)\n", + "assert(len(lb_status.workerStatusList) == 1)\n", "print(\"Timestamp: \", lb_status.timestamp)\n", "print(\"expiresAt: \", lb_status.expiresAt)\n", "print(\"currentEpoch: \", lb_status.currentEpoch)\n", "print(\"currentPredictedEventNumber: \", lb_status.currentPredictedEventNumber)\n", - "print(\"workers: \", lb_status.workers)\n" + "print(\"workerStatusList: \", lb_status.workerStatusList)\n" ] }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 33, + "metadata": {}, + "outputs": [], + "source": [ + "DELTA = 0.000001\n", + "\n", + "assert(lb_status.workerStatusList[0].get_name() == \"my_node\")\n", + "assert(abs(lb_status.workerStatusList[0].get_fill_percent() - 0.8) < DELTA)\n", + "assert(abs(lb_status.workerStatusList[0].get_control_signal() - 1) < DELTA)" + ] + }, + { + "cell_type": "code", + "execution_count": 34, "metadata": {}, "outputs": [ { @@ -915,7 +964,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 35, "metadata": {}, "outputs": [ { @@ -925,16 +974,16 @@ "Create LBManager obj succeeded: 0\n", "Register worker succeeded\n", "\n", - "Session id is: 11\n", - "Session token is: 07589d480e99cdb41bf26b263e236eae381f0ba33feec5054c62a38a3c41f811\n", + "Session id is: 18\n", + "Session token is: 6367c9176ad83fbeaf43d4a53e36b0f01695a8d80789ecf1cd2e0c435e145369\n", "\n", "Send state for 25 times\n", "Add senders succeeded.\n", "\n", - "Get LB status succeeded: \n", + "Get LB status succeeded: \n", "Remove senders succeeded.\n", "\n", - "Get LB status succeeded: \n", + "Get LB status succeeded: \n", "\n", "Deregister worker succeeded\n", "Free LB succeeded!\n" @@ -1005,6 +1054,9 @@ "assert(res == ip_list)\n", "workers = lbm.get_worker_statuses(status_res)\n", "assert len(workers) == 1\n", + "assert(workers[0].get_name() == \"my_node\")\n", + "assert(abs(workers[0].get_fill_percent() - 0.8) < DELTA)\n", + "assert(abs(workers[0].get_control_signal() - 1) < DELTA)\n", "\n", "# Remove senders\n", "res = lbm.remove_senders(ip_list)\n", @@ -1042,7 +1094,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 36, "metadata": {}, "outputs": [ { @@ -1076,7 +1128,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 37, "metadata": {}, "outputs": [ { @@ -1086,25 +1138,22 @@ "Create LBManager obj succeeded: 0\n", "Register worker succeeded\n", "\n", - "Session id is: 14\n", - "Session token is: 24409feb705a6ae6a6ef7dce008a7c726fa6134bba00e19e47ca6d342d81d162\n", + "Session id is: 19\n", + "Session token is: b91e53badc06c7547526b6a2a1e3c45283c7033e575749a276d2aca27dbad63a\n", "\n", "Send state for 25 times\n", "\n", - "Get LB status succeeded: \n", + "Get LB status succeeded: \n", "Sender addresses: ['192.168.20.1', '192.168.20.2']\n", "Current Epoch: 2\n", "Current predicted Event No: 9223372036854775808\n", - "Workers: []\n", - "Timestamp: seconds: 1726512975\n", - "nanos: 658369981\n", - "\n", - "expiresAt: seconds: 1726516573\n", - "\n", + "Workers: []\n", + "Timestamp: 2024-09-18T22:24:48.319546597Z\n", + "expiresAt: 2024-09-18T23:24:45Z\n", "\n", "Send state for 25 times\n", "\n", - "LB id: 23\n", + "LB id: 37\n", "1\n" ] } @@ -1171,15 +1220,17 @@ "\n", "# as_lb_status() usage\n", "res = lbm.as_lb_status(status_res)\n", - "print(f\"Sender addresses: {res.senderAddresses}\")\n", + "print(f\"Sender addresses: {res.senderAddressList}\")\n", "print(f\"Current Epoch: {res.currentEpoch}\")\n", "print(f\"Current predicted Event No: {res.currentPredictedEventNumber}\")\n", - "print(f\"Workers: {res.workers}\")\n", + "print(f\"Workers: {res.workerStatusList}\")\n", "print(f\"Timestamp: {res.timestamp}\")\n", "print(f\"expiresAt: {res.expiresAt}\")\n", - "assert(res.senderAddresses == ip_list)\n", - "# print(res.workers, dir(res.workers))\n", - "assert(len(res.workers) == 1)\n", + "assert(res.senderAddressList == ip_list)\n", + "assert(len(res.workerStatusList) == 1)\n", + "assert(res.workerStatusList[0].get_name() == \"my_node\")\n", + "assert(abs(res.workerStatusList[0].get_fill_percent() - 0.8) < DELTA)\n", + "assert(abs(res.workerStatusList[0].get_control_signal() - 1) < DELTA)\n", "\n", "print(\"\\nSend state for 25 times\")\n", "for i in range(25):\n", @@ -1203,13 +1254,13 @@ " free_lbmanager(lbm)\n", "print(\"LB id: \", res[0].lb_id)\n", "assert(res[0].name == \"mylb\")\n", - "assert(res[0].lb_status.senderAddresses == ip_list)\n", - "print(len(res[0].lb_status.workers))" + "assert(res[0].lb_status.senderAddressList == ip_list)\n", + "print(len(res[0].lb_status.workerStatusList))" ] }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 38, "metadata": {}, "outputs": [ { diff --git a/scripts/notebooks/pybind11_examples/example_DataPlane.ipynb b/scripts/notebooks/pybind11_examples/example_DataPlane.ipynb index 79b135a2..3177e0fe 100644 --- a/scripts/notebooks/pybind11_examples/example_DataPlane.ipynb +++ b/scripts/notebooks/pybind11_examples/example_DataPlane.ipynb @@ -20,7 +20,7 @@ "\n", "## IMPORTANT: Update the path to your built Python module. Use the absolute path to make life easier.\n", "sys.path.append(\n", - " '/home/ubuntu/E2SAR/build/src/pybind')\n", + " '/home/ubuntu/dev-e2sar/build/src/pybind')\n", "\n", "import e2sar_py" ] @@ -43,7 +43,8 @@ "metadata": {}, "outputs": [], "source": [ - "DP_IPV4 = \"127.0.0.1\"\n", + "DP_IPV4_ADDR = \"127.0.0.1\"\n", + "DP_IPV4_PORT = 10000\n", "data_id = 0x0505 # decimal value: 1085\n", "eventSrc_id = 0x11223344 # decimal value: 287454020" ] @@ -79,7 +80,7 @@ ], "source": [ "# Set up URI for segmenter\n", - "SEG_URI = f\"ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data={DP_IPV4}\"\n", + "SEG_URI = f\"ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data={DP_IPV4_ADDR}:{DP_IPV4_PORT}\"\n", "seg_uri = e2sar_py.EjfatURI(uri=SEG_URI, tt=e2sar_py.EjfatURI.TokenType.instance)\n", "\n", "# Set up sflags\n", @@ -163,7 +164,6 @@ " epoch_ms = 1000\n", " setPoint = 0.0\n", " [Kp, Ki, Kd] = [0.0, 0.0, 0.0]\n", - " dpV6 = False\n", " cpV6 = False\n", " portRange = -1\n", " withLBHeader = True\n" @@ -172,7 +172,7 @@ ], "source": [ "# Set the reassembler URI\n", - "REAS_URI_ = f\"ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data={DP_IPV4}\"\n", + "REAS_URI_ = f\"ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data={DP_IPV4_ADDR}\"\n", "reas_uri = e2sar_py.EjfatURI(uri=REAS_URI_, tt=e2sar_py.EjfatURI.TokenType.instance)\n", "\n", "# Make sure the token matches the one in the string\n", @@ -204,14 +204,9 @@ "print(f\" epoch_ms = {rflags.epoch_ms}\")\n", "print(f\" setPoint = {rflags.setPoint}\")\n", "print(f\" [Kp, Ki, Kd] = [{rflags.Kp}, {rflags.Ki}, {rflags.Kd}]\")\n", - "print(f\" dpV6 = {rflags.dpV6}\")\n", "print(f\" cpV6 = {rflags.cpV6}\")\n", "print(f\" portRange = {rflags.portRange}\")\n", - "print(f\" withLBHeader = {rflags.withLBHeader}\")\n", - "\n", - "# Init the reassembler object\n", - "# print(type(reas_uri), type(1), type(rflags))\n", - "reas = e2sar_py.DataPlane.Reassembler(reas_uri, 1, rflags)\n" + "print(f\" withLBHeader = {rflags.withLBHeader}\")" ] }, { @@ -219,6 +214,17 @@ "execution_count": 8, "metadata": {}, "outputs": [], + "source": [ + "# Init the reassembler object\n", + "reas = e2sar_py.DataPlane.Reassembler(\n", + " reas_uri, e2sar_py.IPAddress.from_string(DP_IPV4_ADDR), DP_IPV4_PORT, 1, rflags)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], "source": [ "\n", "res = reas.OpenAndStart() # the DP address must be available\n", @@ -227,7 +233,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -252,7 +258,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -268,7 +274,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -308,7 +314,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -374,7 +380,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -411,17 +417,17 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ - "SEG_URI2 = f\"ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data={DP_IPV4}\"\n", + "SEG_URI2 = f\"ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data={DP_IPV4_ADDR}:{DP_IPV4_PORT}\"\n", "seg_uri2 = e2sar_py.EjfatURI(uri=SEG_URI2, tt=e2sar_py.EjfatURI.TokenType.instance)" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -439,7 +445,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 17, "metadata": {}, "outputs": [ { @@ -480,7 +486,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -545,7 +551,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -563,7 +569,8 @@ "# Create reassembler with 1 recv thread\n", "\n", "rflags_m = e2sar_py.DataPlane.ReassemblerFlags()\n", - "reas_a = e2sar_py.DataPlane.Reassembler(reas_uri, 1, rflags_m)\n", + "reas_a = e2sar_py.DataPlane.Reassembler(\n", + " reas_uri, e2sar_py.IPAddress.from_string(DP_IPV4_ADDR), 19522, 1, rflags_m)\n", "\n", "print(\"This reassembler has\")\n", "print(f\" {reas_a.get_numRecvThreads()} threads;\")\n", @@ -577,7 +584,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -593,7 +600,8 @@ ], "source": [ "# Create reassembler with 4 recv threads\n", - "reas_b = e2sar_py.DataPlane.Reassembler(reas_uri, 4, rflags_m)\n", + "reas_b = e2sar_py.DataPlane.Reassembler(\n", + " reas_uri, e2sar_py.IPAddress.from_string(DP_IPV4_ADDR), 19522, 4, rflags_m)\n", "\n", "print(\"This reassembler has\")\n", "print(f\" {reas_b.get_numRecvThreads()} threads;\")\n", @@ -607,7 +615,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -623,7 +631,8 @@ ], "source": [ "# Create reassembler with 7 recv threads\n", - "reas_c = e2sar_py.DataPlane.Reassembler(reas_uri, 7, rflags_m)\n", + "reas_c = e2sar_py.DataPlane.Reassembler(\n", + " reas_uri, e2sar_py.IPAddress.from_string(DP_IPV4_ADDR), 19522, 7, rflags_m)\n", "\n", "print(\"This reassembler has\")\n", "print(f\" {reas_c.get_numRecvThreads()} threads;\")\n", @@ -637,7 +646,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 22, "metadata": {}, "outputs": [ { @@ -655,7 +664,8 @@ "# 4 threads with portRange override\n", "rflags_m.portRange = 10\n", "\n", - "reas_d = e2sar_py.DataPlane.Reassembler(reas_uri, 4, rflags_m)\n", + "reas_d = e2sar_py.DataPlane.Reassembler(\n", + " reas_uri, e2sar_py.IPAddress.from_string(DP_IPV4_ADDR), 19522, 4, rflags_m)\n", "\n", "print(\"This reassembler has\")\n", "print(f\" {reas_d.get_numRecvThreads()} threads;\")\n", @@ -669,7 +679,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 23, "metadata": {}, "outputs": [ { @@ -687,7 +697,8 @@ "# 4 threads with low portRange override\n", "rflags_m.portRange = 1\n", "\n", - "reas_e = e2sar_py.DataPlane.Reassembler(reas_uri, 4, rflags_m)\n", + "reas_e = e2sar_py.DataPlane.Reassembler(\n", + " reas_uri, e2sar_py.IPAddress.from_string(DP_IPV4_ADDR), 19522, 4, rflags_m)\n", "\n", "print(\"This reassembler has\")\n", "print(f\" {reas_e.get_numRecvThreads()} threads;\")\n", @@ -710,7 +721,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 25, "metadata": {}, "outputs": [ { @@ -731,7 +742,7 @@ "source": [ "# Create 4 segmenter objects\n", "\n", - "SEG_URI_BASE = SEG_URI\n", + "SEG_URI_BASE = f\"ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data={DP_IPV4_ADDR}\"\n", "SEG_URI_PORT_BASE = 19522\n", "\n", "sflags = e2sar_py.DataPlane.SegmenterFlags()\n", @@ -762,7 +773,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 26, "metadata": {}, "outputs": [ { @@ -784,7 +795,8 @@ "rflags.withLBHeader = True # LB header will be attached since there is no LB\n", "rflags.portRange = 2 # use 2^portRange=4 ports\n", "\n", - "reas = e2sar_py.DataPlane.Reassembler(reas_uri, 1, rflags)\n", + "reas = e2sar_py.DataPlane.Reassembler(\n", + " reas_uri, e2sar_py.IPAddress.from_string(DP_IPV4_ADDR), 19522, 1, rflags)\n", "print(\"This reassembler has\")\n", "print(f\" {reas.get_numRecvThreads()} threads;\")\n", "print(f\" listening on ports {reas.get_recvPorts()[0]}:{reas.get_recvPorts()[1]};\")\n", @@ -793,7 +805,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 27, "metadata": {}, "outputs": [ { @@ -819,7 +831,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -832,7 +844,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 29, "metadata": {}, "outputs": [], "source": [ @@ -845,7 +857,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 30, "metadata": {}, "outputs": [ { @@ -893,7 +905,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 31, "metadata": {}, "outputs": [ { diff --git a/scripts/notebooks/pybind11_examples/example_EjfatURI.ipynb b/scripts/notebooks/pybind11_examples/example_EjfatURI.ipynb index 2e51157c..b27ee6e2 100644 --- a/scripts/notebooks/pybind11_examples/example_EjfatURI.ipynb +++ b/scripts/notebooks/pybind11_examples/example_EjfatURI.ipynb @@ -27,11 +27,11 @@ "\n", "# IMPORTANT: Adjust the path to your built Python module as necessary\n", "sys.path.append(\n", - " '/home/ubuntu/E2SAR/build/src/pybind')\n", + " '/home/ubuntu/dev-e2sar/build/src/pybind')\n", "\n", "import e2sar_py\n", "\n", - "print(e2sar_py.get_version())" + "# print(e2sar_py.get_version())" ] }, { diff --git a/scripts/notebooks/pybind11_examples/example_Headers.ipynb b/scripts/notebooks/pybind11_examples/example_Headers.ipynb index f91a9015..7105219a 100644 --- a/scripts/notebooks/pybind11_examples/example_Headers.ipynb +++ b/scripts/notebooks/pybind11_examples/example_Headers.ipynb @@ -11,47 +11,27 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0.1.1\n" - ] - } - ], + "outputs": [], "source": [ "import sys\n", "\n", "## IMPORTANT: Update the path to your built Python module. Prefer absolute path.\n", "sys.path.append(\n", - " '/home/ubuntu/E2SAR/build/src/pybind')\n", + " '/home/ubuntu/dev-e2sar/build/src/pybind')\n", "\n", "import e2sar_py\n", "\n", "# Get the version\n", - "print(e2sar_py.get_version())\n" + "# print(e2sar_py.get_version())\n" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Default data plane port: 19522\n", - "Default Reassembler Header version: 1\n", - "Default Reassembler Header nibble: 16\n", - "Default Load Balancer Header version: 2\n", - "Default Sync Header version: 1\n" - ] - } - ], + "outputs": [], "source": [ "# Print the constant atrributes\n", "print(f\"Default data plane port: {e2sar_py._dp_port}\")\n", @@ -63,20 +43,9 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "IP header length: 20\n", - "UDP header length: 8\n", - "Total header length: 64\n", - "LB + RE header length: 36\n" - ] - } - ], + "outputs": [], "source": [ "\n", "# The Hdr lengths\n", @@ -97,22 +66,9 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Before setting fields: (0, 0, 0, 0)\n", - "After setting fields: (1, 2, 4, 8)\n", - " data_id=1\n", - " buff_off=2\n", - " buff_len=4\n", - " event_num=8\n" - ] - } - ], + "outputs": [], "source": [ "# Setting the Reaasembler header fields\n", "rehdr = e2sar_py.REHdr()\n", @@ -140,22 +96,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Before (2, 1, 0, 0)\n", - "2\n", - "1\n", - "0\n", - "0\n", - "After (2, 1, 200, 50)\n" - ] - } - ], + "outputs": [], "source": [ "# Load balancer header\n", "lbhdr = e2sar_py.LBHdr()\n", diff --git a/src/pybind/py_e2sar.cpp b/src/pybind/py_e2sar.cpp index 928771fe..78326e65 100644 --- a/src/pybind/py_e2sar.cpp +++ b/src/pybind/py_e2sar.cpp @@ -132,4 +132,6 @@ void init_e2sarResultTypes(py::module_ &m) bind_result(m, "E2SARResultUInit32"); bind_result>(m, "E2SARResultPairIP"); bind_result>(m, "E2SARResultPairString"); + bind_result(m, "E2SARResultReassemblerFlags"); + bind_result(m, "E2SARResultSegmenterFlags"); } diff --git a/src/pybind/py_e2sarCP.cpp b/src/pybind/py_e2sarCP.cpp index 82221793..bae545bf 100644 --- a/src/pybind/py_e2sarCP.cpp +++ b/src/pybind/py_e2sarCP.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include "e2sarUtil.hpp" #include "e2sarCP.hpp" @@ -47,7 +48,16 @@ void init_e2sarCP(py::module_ &m) { // Expose the grpc classes py::class_(e2sarCP, "WorkerStatus") - .def(py::init<>()); + .def(py::init<>()) + .def("get_name", &WorkerStatus::name) + .def("get_fill_percent", &WorkerStatus::fillpercent) + .def("get_control_signal", &WorkerStatus::controlsignal) + .def("get_slots_assigned", &WorkerStatus::slotsassigned) + .def("get_last_updated", + [](const WorkerStatus &self) { + return google::protobuf::util::TimeUtil::ToString(self.lastupdated()); + } + ); py::class_(e2sarCP, "LoadBalancerStatusReply") .def(py::init<>()); py::class_(e2sarCP, "OverviewReply") @@ -58,7 +68,11 @@ void init_e2sarCP(py::module_ &m) { .def("get_seconds", &google::protobuf::Timestamp::seconds) .def("get_nanos", &google::protobuf::Timestamp::nanos) .def("set_seconds", &google::protobuf::Timestamp::set_seconds) - .def("set_nanos", &google::protobuf::Timestamp::set_nanos); + .def("set_nanos", &google::protobuf::Timestamp::set_nanos) + // pretty print in Python: use TimeUtil::ToString to get the default string representation + .def("__str__", [](const google::protobuf::Timestamp& self) { + return google::protobuf::util::TimeUtil::ToString(self); + }); /** * Bindings for struct "LBWorkerStatus" @@ -78,12 +92,8 @@ void init_e2sarCP(py::module_ &m) { .def_readonly("slots_assigned", &LBWorkerStatus::slotsAssigned) .def_property_readonly("last_updated", [](const LBWorkerStatus &self) { - return convert_timestamp_to_python(self.lastUpdated); // Access the member directly + return google::protobuf::util::TimeUtil::ToString(self.lastUpdated); // Access the member directly } - // , - // [](LBWorkerStatus &self, py::object py_timestamp) { - // self.lastUpdated = convert_timestamp_to_cpp(py_timestamp); // Assign the converted value - // } ); /** @@ -101,32 +111,25 @@ void init_e2sarCP(py::module_ &m) { std::vector&, google::protobuf::Timestamp>(), py::arg("timestamp"), - py::arg("currentEpoch"), py::arg("currentPredictedEventNumber"), - py::arg("workers"), + py::arg("current_epoch"), + py::arg("current_predicted_event_number"), + py::arg("worker_status_list"), py::arg("sender_addresses"), - py::arg("expiresAt")) + py::arg("expires_at")) // Expose the struct members - .def_property("timestamp", + .def_property_readonly(/* a Python string */"timestamp", [](const LBStatus &self) { - return convert_timestamp_to_python(self.timestamp); // Access the member directly - }, - [](LBStatus &self, py::object py_timestamp) { - self.timestamp = convert_timestamp_to_cpp(py_timestamp); // Assign the converted value + return google::protobuf::util::TimeUtil::ToString(self.timestamp); }) .def_readonly("currentEpoch", &LBStatus::currentEpoch) .def_readonly("currentPredictedEventNumber", &LBStatus::currentPredictedEventNumber) - .def_readonly("workers", &LBStatus::workers) - .def_readonly("senderAddresses", &LBStatus::senderAddresses) - .def_property_readonly("expiresAt", + .def_readonly("workerStatusList", &LBStatus::workers) + .def_readonly("senderAddressList", &LBStatus::senderAddresses) + .def_property_readonly(/* a Python string */"expiresAt", [](const LBStatus &self) { - return convert_timestamp_to_python(self.expiresAt); // Access the member directly - } - // , - // [](LBStatus &self, py::object py_timestamp) { - // self.expiresAt= convert_timestamp_to_cpp(py_timestamp); // Assign the converted value - // } - ); + return google::protobuf::util::TimeUtil::ToString(self.expiresAt); // Access the member directly + }); /** * Bindings for struct "OverviewEntry" diff --git a/src/pybind/py_e2sarDP.cpp b/src/pybind/py_e2sarDP.cpp index 82555fc0..a9e0700a 100644 --- a/src/pybind/py_e2sarDP.cpp +++ b/src/pybind/py_e2sarDP.cpp @@ -77,7 +77,9 @@ void init_e2sarDP_segmenter(py::module_ &m) .def_readwrite("syncPeriodMs", &Segmenter::SegmenterFlags::syncPeriodMs) .def_readwrite("syncPeriods", &Segmenter::SegmenterFlags::syncPeriods) .def_readwrite("mtu", &Segmenter::SegmenterFlags::mtu) - .def_readwrite("numSendSockets", &Segmenter::SegmenterFlags::numSendSockets); + .def_readwrite("numSendSockets", &Segmenter::SegmenterFlags::numSendSockets) + .def_readwrite("sndSocketBufSize", &Segmenter::SegmenterFlags::sndSocketBufSize) + .def("getFromINI", &Segmenter::SegmenterFlags::getFromINI); // Constructor seg.def( @@ -172,15 +174,17 @@ void init_e2sarDP_reassembler(py::module_ &m) .def_readwrite("epoch_ms", &Reassembler::ReassemblerFlags::epoch_ms) .def_readwrite("portRange", &Reassembler::ReassemblerFlags::portRange) .def_readwrite("withLBHeader", &Reassembler::ReassemblerFlags::withLBHeader) - .def_readwrite("eventTimeout_ms", &Reassembler::ReassemblerFlags::eventTimeout_ms); + .def_readwrite("eventTimeout_ms", &Reassembler::ReassemblerFlags::eventTimeout_ms) + .def_readwrite("rcvSocketBufSize", &Reassembler::ReassemblerFlags::rcvSocketBufSize) + .def("getFromINI", &Reassembler::ReassemblerFlags::getFromINI); // Constructor reas.def( py::init(), "Init the Reassembler object with number of recv threads.", py::arg("uri"), // must-have arg when init - py::arg("ip"), - py::arg("port"), + py::arg("data_ip"), + py::arg("starting_port"), py::arg("num_recv_threads") = (size_t)1, py::arg("rflags") = Reassembler::ReassemblerFlags()); From 54f7ae19fb10d289385b31c5fce6fc0651cf0b23 Mon Sep 17 00:00:00 2001 From: Xinxin Mei Date: Thu, 19 Sep 2024 13:14:45 -0400 Subject: [PATCH 09/73] Fix: "deb" to "rpm" for Rocky workflows --- .github/workflows/deps2debs.yml | 2 +- .github/workflows/distro.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deps2debs.yml b/.github/workflows/deps2debs.yml index 9e7da166..3c0790e4 100644 --- a/.github/workflows/deps2debs.yml +++ b/.github/workflows/deps2debs.yml @@ -67,7 +67,7 @@ jobs: if [[ ${{ matrix.os }} == *"ubuntu"* ]]; then fpm -s tar -t deb -n e2sar-deps -v ${{ env.E2SAR_VER }} --prefix=/ e2sar-deps.tar.gz elif [[ ${{ matrix.os }} == *"rocky"* ]]; then - fpm -s tar -t deb -n e2sar-deps -v ${{ env.E2SAR_VER }} --prefix=/ e2sar-deps.tar.gz + fpm -s tar -t rpm -n e2sar-deps -v ${{ env.E2SAR_VER }} --prefix=/ e2sar-deps.tar.gz fi - name: Upload artifacts diff --git a/.github/workflows/distro.yml b/.github/workflows/distro.yml index 8c51e6d2..f45d718b 100644 --- a/.github/workflows/distro.yml +++ b/.github/workflows/distro.yml @@ -100,7 +100,7 @@ jobs: if [[ ${{ matrix.os }} == *"ubuntu"* ]]; then fpm -s tar -t deb -n e2sar -v ${{ env.E2SAR_VER }} --prefix=/ e2sar.tar.gz elif [[ ${{ matrix.os }} == *"rocky"* ]]; then - fpm -s tar -t deb -n e2sar -v ${{ env.E2SAR_VER }} --prefix=/ e2sar.tar.gz + fpm -s tar -t rpm -n e2sar -v ${{ env.E2SAR_VER }} --prefix=/ e2sar.tar.gz fi - name: Upload artifacts From e706f006e6b98052de5f97bb26a10c2bf1cbbcd4 Mon Sep 17 00:00:00 2001 From: Xinxin Mei Date: Thu, 19 Sep 2024 13:23:57 -0400 Subject: [PATCH 10/73] 0.1.1 to 0.1.2 in all workflow yml files --- .github/workflows/builddeps.yml | 2 +- .github/workflows/deploydebs.yml | 2 +- .github/workflows/deps2debs.yml | 2 +- .github/workflows/distro.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/builddeps.yml b/.github/workflows/builddeps.yml index 75af4105..5313aa3a 100644 --- a/.github/workflows/builddeps.yml +++ b/.github/workflows/builddeps.yml @@ -6,7 +6,7 @@ on: env: GRPC_VER: 1.54.1 BOOST_VER: 1.85.0 - E2SAR_VER: 0.1.1 + E2SAR_VER: 0.1.2 jobs: build: diff --git a/.github/workflows/deploydebs.yml b/.github/workflows/deploydebs.yml index 68894053..7ce7ceb1 100644 --- a/.github/workflows/deploydebs.yml +++ b/.github/workflows/deploydebs.yml @@ -6,7 +6,7 @@ name: Deploy E2sar debs and dependency debs to releases on: workflow_dispatch: env: - E2SAR_VER: 0.1.1 + E2SAR_VER: 0.1.2 E2SAR_DEP: 1.85.0-boost-1.54.1-grpc jobs: diff --git a/.github/workflows/deps2debs.yml b/.github/workflows/deps2debs.yml index 3c0790e4..f0eed0ed 100644 --- a/.github/workflows/deps2debs.yml +++ b/.github/workflows/deps2debs.yml @@ -3,7 +3,7 @@ name: Build gRPC+Boost .deb and .rpms on: workflow_dispatch: env: - E2SAR_VER: 0.1.1 + E2SAR_VER: 0.1.2 DEPS_VER: 1.85.0-boost-1.54.1-grpc FINAL_INSTALL: /usr/local diff --git a/.github/workflows/distro.yml b/.github/workflows/distro.yml index f45d718b..95113e20 100644 --- a/.github/workflows/distro.yml +++ b/.github/workflows/distro.yml @@ -3,7 +3,7 @@ name: Build E2SAR .deb and .rpms on: workflow_dispatch: env: - E2SAR_VER: 0.1.1 + E2SAR_VER: 0.1.2 E2SAR_DEP: boost-1.85.0-grpc-1.54.1 jobs: From 274897ab53b633ced486ef24e689554073c8ed23 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Thu, 19 Sep 2024 11:20:55 -0400 Subject: [PATCH 11/73] Made weight, min and max_factor settable via flags; added sane defaults From 157664f4fa07b06e3a95e6b011e0786d1ec004f3 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Thu, 19 Sep 2024 15:00:28 -0400 Subject: [PATCH 12/73] Upped version in Doxyfile --- Doxyfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doxyfile b/Doxyfile index adecb190..9f0d377f 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = "E2SAR" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 0.1.0 +PROJECT_NUMBER = 0.1.2 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a From ed498f9ddd45dfd94bf6731f992c30f7c61e7a6f Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Thu, 19 Sep 2024 15:54:25 -0400 Subject: [PATCH 13/73] Updated docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 719131ca..1e6195a6 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 719131ca2c65911d4b420e9472faadc2fd367062 +Subproject commit 1e6195a6ead60a573bb19c6bb96924a44e4773c4 From 399f2dd5f16c594bcdcb2c6c9e71692a16e17f0b Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Thu, 19 Sep 2024 17:15:50 -0400 Subject: [PATCH 14/73] Correcting option conflict --- bin/e2sar_perf.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/e2sar_perf.cpp b/bin/e2sar_perf.cpp index 0d0c4603..2365f6a5 100644 --- a/bin/e2sar_perf.cpp +++ b/bin/e2sar_perf.cpp @@ -310,9 +310,9 @@ int main(int argc, char **argv) conflicting_options(vm, "send", "threads"); conflicting_options(vm, "send", "period"); option_dependency(vm, "recv", "ip"); - option_dependency(vm, "vm", "port"); + option_dependency(vm, "recv", "port"); option_dependency(vm, "send", "ip"); - conflicting_options(vm, "recv", "port"); + conflicting_options(vm, "send", "port"); } catch (const std::logic_error &le) { From a1a9295efcd51c3454fd3c9694be97a64c7324d1 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Mon, 23 Sep 2024 16:52:17 -0400 Subject: [PATCH 15/73] Added more logging --- bin/e2sar_perf.cpp | 11 +++++++++++ include/e2sarDPReassembler.hpp | 2 +- src/e2sarDPReassembler.cpp | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/bin/e2sar_perf.cpp b/bin/e2sar_perf.cpp index 2365f6a5..9cacd505 100644 --- a/bin/e2sar_perf.cpp +++ b/bin/e2sar_perf.cpp @@ -41,6 +41,10 @@ void ctrlCHandler(int sig) if (segPtr != nullptr) { if (lbmPtr != nullptr) { + std::cout << "Removing senders: "; + for (auto s: senders) + std::cout << s << " "; + std::cout << std::endl; auto rmres = lbmPtr->removeSenders(senders); if (rmres.has_error()) std::cerr << "Unable to remove sender from list on exit: " << rmres.error().message() << std::endl; @@ -49,6 +53,7 @@ void ctrlCHandler(int sig) } if (reasPtr != nullptr) { + std::cout << "Deregistering worker" << std::endl; auto deregres = reasPtr->deregisterWorker(); if (deregres.has_error()) std::cerr << "Unable to deregister worker on exit: " << deregres.error().message() << std::endl; @@ -184,6 +189,8 @@ result recvEvents(Reassembler &r, int durationSec) { return E2SARErrorInfo{E2SARErrorc::RPCError, "Unable to register worker node due to " + regres.error().message()}; } + if (regres.value() == 1) + std::cout << "Registered the worker" << std::endl; // receive loop while(true) @@ -354,6 +361,10 @@ int main(int argc, char **argv) lbmPtr = new LBManager(uri, false); // register senders + std::cout << "Adding senders to LB: "; + for (auto s: senders) + std::cout << s << " "; + std::cout << std::endl; auto addres = lbmPtr->addSenders(senders); if (addres.has_error()) { diff --git a/include/e2sarDPReassembler.hpp b/include/e2sarDPReassembler.hpp index fe373a2e..e949fb93 100644 --- a/include/e2sarDPReassembler.hpp +++ b/include/e2sarDPReassembler.hpp @@ -414,7 +414,7 @@ namespace e2sar /** * Register a worker with the control plane * @param node_name - name of this node (any unique string) - * @return - 0 on success or an error condition + * @return - 0 if CP was off and nothing was done, 1 on success or an error condition */ result registerWorker(const std::string &node_name) noexcept; diff --git a/src/e2sarDPReassembler.cpp b/src/e2sarDPReassembler.cpp index 3b9c120c..2f3ac20d 100644 --- a/src/e2sarDPReassembler.cpp +++ b/src/e2sarDPReassembler.cpp @@ -464,6 +464,7 @@ namespace e2sar registeredWorker = true; return lbman.registerWorker(node_name, std::make_pair(dataIP, dataPort), weight, numRecvPorts, min_factor, max_factor); + return 1; } return 0; } From 32a6eeda7b477315a9b2177a9ec3fdce947e9ba9 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Mon, 23 Sep 2024 17:19:43 -0400 Subject: [PATCH 16/73] Fixing return --- include/e2sarDPReassembler.hpp | 2 +- include/e2sarHeaders.hpp | 2 +- src/e2sarDPReassembler.cpp | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/include/e2sarDPReassembler.hpp b/include/e2sarDPReassembler.hpp index e949fb93..fe373a2e 100644 --- a/include/e2sarDPReassembler.hpp +++ b/include/e2sarDPReassembler.hpp @@ -414,7 +414,7 @@ namespace e2sar /** * Register a worker with the control plane * @param node_name - name of this node (any unique string) - * @return - 0 if CP was off and nothing was done, 1 on success or an error condition + * @return - 0 on success or an error condition */ result registerWorker(const std::string &node_name) noexcept; diff --git a/include/e2sarHeaders.hpp b/include/e2sarHeaders.hpp index 11692962..7952d92f 100644 --- a/include/e2sarHeaders.hpp +++ b/include/e2sarHeaders.hpp @@ -163,7 +163,7 @@ namespace e2sar struct REHdr re; } __attribute__((__packed__)); - constexpr u_int8_t synchdrVersion = 1; + constexpr u_int8_t synchdrVersion = 2; /** The Syncr Header. You should always use the provided methods to set and interrogate fields as the structure maintains Big-Endian order diff --git a/src/e2sarDPReassembler.cpp b/src/e2sarDPReassembler.cpp index 2f3ac20d..3b9c120c 100644 --- a/src/e2sarDPReassembler.cpp +++ b/src/e2sarDPReassembler.cpp @@ -464,7 +464,6 @@ namespace e2sar registeredWorker = true; return lbman.registerWorker(node_name, std::make_pair(dataIP, dataPort), weight, numRecvPorts, min_factor, max_factor); - return 1; } return 0; } From 60eab03f6b9f4310a5459353d7639e642686b363 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Mon, 23 Sep 2024 23:06:21 -0400 Subject: [PATCH 17/73] Added useHostAddress option to LBManager to force using a previously resolved address in the URI instead of hostname --- bin/e2sar_perf.cpp | 25 ++++++++++++++++++++++--- bin/lbadm.cpp | 15 +++++++++++---- bin/lbmonitor.cpp | 14 ++++++++++---- include/e2sarCP.hpp | 18 ++++++++++++++---- include/e2sarDPReassembler.hpp | 12 +++++------- include/e2sarUtil.hpp | 2 +- src/e2sarDPReassembler.cpp | 12 ++++++------ src/pybind/py_e2sarCP.cpp | 3 ++- src/pybind/py_e2sarDP.cpp | 2 +- 9 files changed, 72 insertions(+), 31 deletions(-) diff --git a/bin/e2sar_perf.cpp b/bin/e2sar_perf.cpp index 9cacd505..95fdd7a1 100644 --- a/bin/e2sar_perf.cpp +++ b/bin/e2sar_perf.cpp @@ -295,6 +295,9 @@ int main(int argc, char **argv) " Values found in the file override --withcp, --mtu and --bufsize"); opts("ip", po::value(&sndrcvIP)->default_value("127.0.0.1"), "IP address (IPv4 or IPv6) from which sender sends from or on which receiver listens. Defaults to 127.0.0.1. [s,r]"); opts("port", po::value(&recvStartPort)->default_value(10000), "Starting UDP port number on which receiver listens. Defaults to 10000. [r] "); + opts("ipv6,6", "force using IPv6 control plane address if URI specifies hostname (disables cert validation) [s,r]"); + opts("ipv4,4", "force using IPv4 control plane address if URI specifies hostname (disables cert validation) [s,r]"); + opts("novalidate,v", "don't validate server certificate"); po::variables_map vm; @@ -316,6 +319,7 @@ int main(int argc, char **argv) conflicting_options(vm, "recv", "rate"); conflicting_options(vm, "send", "threads"); conflicting_options(vm, "send", "period"); + conflicting_options(vm, "ipv4", "ipv6"); option_dependency(vm, "recv", "ip"); option_dependency(vm, "recv", "port"); option_dependency(vm, "send", "ip"); @@ -336,14 +340,29 @@ int main(int argc, char **argv) withCP = vm["withcp"].as(); + bool preferV6 = false; + if (vm.count("ipv6")) + { + preferV6 = true; + } + + // if ipv4 or ipv6 requested explicitly + bool preferHostAddr = false; + if (vm.count("ipv6") || vm.count("ipv4")) + preferHostAddr = true; + + bool noValidate = false; + if (vm.count("novalidate")) + noValidate = true; + // make sure the token is interpreted as the correct type, depending on the call EjfatURI::TokenType tt{EjfatURI::TokenType::instance}; std::string ejfat_uri; if (vm.count("send") || vm.count("recv")) { - auto uri_r = (vm.count("uri") ? EjfatURI::getFromString(vm["uri"].as(), tt) : - EjfatURI::getFromEnv("EJFAT_URI"s, tt)); + auto uri_r = (vm.count("uri") ? EjfatURI::getFromString(vm["uri"].as(), tt, preferV6) : + EjfatURI::getFromEnv("EJFAT_URI"s, tt, preferV6)); if (uri_r.has_error()) { std::cerr << "Error in parsing URI from command-line, error "s + uri_r.error().message() << std::endl; @@ -358,7 +377,7 @@ int main(int argc, char **argv) senders.push_back(sndrcvIP); // create LBManager - lbmPtr = new LBManager(uri, false); + lbmPtr = new LBManager(uri, noValidate, preferHostAddr); // register senders std::cout << "Adding senders to LB: "; diff --git a/bin/lbadm.cpp b/bin/lbadm.cpp index bb2be917..6ddf8e9c 100644 --- a/bin/lbadm.cpp +++ b/bin/lbadm.cpp @@ -358,7 +358,8 @@ int main(int argc, char **argv) opts("novalidate,v", "don't validate server certificate (conflicts with 'root')"); opts("minfactor", po::value(), "node min factor, multiplied with the number of slots that would be assigned evenly to determine min number of slots for example, 4 nodes with a minFactor of 0.5 = (512 slots / 4) * 0.5 = min 64 slots"); opts("maxfactor", po::value(), "multiplied with the number of slots that would be assigned evenly to determine max number of slots for example, 4 nodes with a maxFactor of 2 = (512 slots / 4) * 2 = max 256 slots set to 0 to specify no maximum"); - opts("ipv6,6", "prefer IPv6 control plane address if URI specifies hostname"); + opts("ipv6,6", "force using IPv6 control plane address if URI specifies hostname (disables cert validation)"); + opts("ipv4,4", "force using IPv4 control plane address if URI specifies hostname (disables cert validation)"); opts("export,e", "suppresses other messages and prints out 'export EJFAT_URI=' returned by the LB"); // commands opts("reserve", "reserve a load balancer (-l, -a, -d required). Uses admin token."); @@ -395,6 +396,7 @@ int main(int argc, char **argv) option_dependency(vm, "state", "ctrl"); option_dependency(vm, "state", "ready"); conflicting_options(vm, "root", "novalidate"); + conflicting_options(vm, "ipv4", "ipv6"); option_dependency(vm,"addsenders", "address"); option_dependency(vm,"removesenders", "address"); } @@ -437,6 +439,11 @@ int main(int argc, char **argv) preferV6 = true; } + // if ipv4 or ipv6 requested explicitly + bool preferHostAddr = false; + if (vm.count("ipv6") || vm.count("ipv4")) + preferHostAddr = true; + std::string ejfat_uri; auto uri_r = (vm.count("uri") ? EjfatURI::getFromString(vm["uri"].as(), tt, preferV6) : EjfatURI::getFromEnv("EJFAT_URI"s, tt, preferV6)); @@ -455,7 +462,7 @@ int main(int argc, char **argv) if (vm.count("lbid")) uri.set_lbId(vm["lbid"].as()); - auto lbman = LBManager(uri); + auto lbman = LBManager(uri, true, preferHostAddr); if (vm.count("root") && !uri.get_useTls()) { @@ -471,14 +478,14 @@ int main(int argc, char **argv) std::cerr << "Unable to read server root certificate file "s; return -1; } - lbman = LBManager(uri, true, opts_res.value()); + lbman = LBManager(uri, true, preferHostAddr, opts_res.value()); } else { if (vm.count("novalidate")) { std::cerr << "Skipping server certificate validation" << std::endl; - lbman = LBManager(uri, false); + lbman = LBManager(uri, false, preferHostAddr); } } } diff --git a/bin/lbmonitor.cpp b/bin/lbmonitor.cpp index a6bc66b5..eff148d3 100644 --- a/bin/lbmonitor.cpp +++ b/bin/lbmonitor.cpp @@ -101,7 +101,8 @@ int main(int argc, char **argv) opts("lbid,i", po::value(), "specify id of the loadbalancer as issued by reserve call instead of using what is in EJFAT_URI"); opts("root,o", po::value(), "root cert for SSL communications"); opts("novalidate,v", "don't validate server certificate (conflicts with 'root')"); - opts("ipv6,6", "prefer IPv6 control plane address if URI specifies hostname"); + opts("ipv6,6", "prefer IPv6 control plane address if URI specifies hostname (disables cert validation)"); + opts("ipv4,4", "prefer IPv4 control plane address if URI specifies hostname (disables cert validation)"); opts("uri,u", po::value(), "specify EJFAT_URI on the command-line instead of the environment variable"); opts("time,t", po::value(), "specify refresh time in ms (default is 5000ms)"); @@ -125,6 +126,11 @@ int main(int argc, char **argv) preferV6 = true; } + // if ipv4 or ipv6 requested explicitly + bool preferHostAddr = false; + if (vm.count("ipv6") || vm.count("ipv4")) + preferHostAddr = true; + std::string ejfat_uri; auto uri_r = (vm.count("uri") ? EjfatURI::getFromString(vm["uri"].as(), tt, preferV6) : EjfatURI::getFromEnv("EJFAT_URI"s, tt, preferV6)); @@ -135,7 +141,7 @@ int main(int argc, char **argv) } auto uri = uri_r.value(); - auto lbman = LBManager(uri); + auto lbman = LBManager(uri, true, preferHostAddr); if (vm.count("root") && !uri.get_useTls()) { @@ -151,14 +157,14 @@ int main(int argc, char **argv) std::cerr << "Unable to read server root certificate file "s; return -1; } - lbman = LBManager(uri, true, opts_res.value()); + lbman = LBManager(uri, true, preferHostAddr, opts_res.value()); } else { if (vm.count("novalidate")) { std::cerr << "Skipping server certificate validation" << std::endl; - lbman = LBManager(uri, false); + lbman = LBManager(uri, false, preferHostAddr); } } } diff --git a/include/e2sarCP.hpp b/include/e2sarCP.hpp index 74c341aa..53725aa5 100644 --- a/include/e2sarCP.hpp +++ b/include/e2sarCP.hpp @@ -131,16 +131,23 @@ namespace e2sar * use makeSslOptions[FromFiles]() methods and pass the output to this constructor. * @param cpuri an EjfatURI object parsed from configuration data * @param validateServer if false, skip server certificate validation (useful for self-signed testing) + * @param useHostAddress even if hostname is provided, use host address as resolved by URI object (with preference for + * IPv4 by default or for IPv6 if explicitly requested) * @param opts grpc::SslCredentialsOptions containing some combination of server root certs, client key and client cert * use of SSL/TLS is governed by the URI scheme ('ejfat' vs 'ejfats') */ - LBManager(const EjfatURI &cpuri, bool validateServer = true, + LBManager(const EjfatURI &cpuri, bool validateServer = true, bool useHostAddress = false, grpc::SslCredentialsOptions opts = grpc::SslCredentialsOptions()) : _cpuri(cpuri) { auto cp_host_r = cpuri.get_cpHost(); std::string addr_string; - if (!cp_host_r.has_error()) + + // using host address automatically disables cert validation + if (useHostAddress) + validateServer = false; + + if (!useHostAddress && !cp_host_r.has_error()) { // try hostname auto cp_host_v = cp_host_r.value(); @@ -153,13 +160,16 @@ namespace e2sar if (cp_addr_r.has_error()) throw E2SARException("Unable to initialize LBManager due to missing CP address in URI"); auto cp_addr_v = cp_addr_r.value(); - addr_string = cp_addr_v.first.to_string() + ":" + std::to_string(cp_addr_v.second); + if (cp_addr_v.first.is_v4()) + addr_string = "ipv4://" + cp_addr_v.first.to_string() + ":" + std::to_string(cp_addr_v.second); + else + addr_string = "ipv6://[" + cp_addr_v.first.to_string() + "]:" + std::to_string(cp_addr_v.second); } if (cpuri.get_useTls()) { grpc::experimental::TlsChannelCredentialsOptions topts; - if (!validateServer) + if (validateServer) { // disable most of server certificate validation std::shared_ptr verifier = std::make_shared(); diff --git a/include/e2sarDPReassembler.hpp b/include/e2sarDPReassembler.hpp index fe373a2e..964d9523 100644 --- a/include/e2sarDPReassembler.hpp +++ b/include/e2sarDPReassembler.hpp @@ -276,14 +276,12 @@ namespace e2sar boost::thread threadObj; const u_int16_t period_ms; - // flags - const bool useV6; // UDP sockets int socketFd{0}; - inline SendStateThreadState(Reassembler &r, bool v6, u_int16_t period_ms): - reas{r}, period_ms{period_ms}, useV6{v6} + inline SendStateThreadState(Reassembler &r, u_int16_t period_ms): + reas{r}, period_ms{period_ms} {} // thread loop. all important behavior is encapsulated inside LBManager @@ -321,8 +319,8 @@ namespace e2sar public: /** * Structure for flags governing Reassembler behavior with sane defaults - * - cpV6 - use IPv6 address if cp node specified by name and has IPv4 and IPv6 resolution {false} * - useCP - whether to use the control plane (sendState, registerWorker) {true} + * - useHostAddress - use IPv4 or IPv6 address for gRPC even if hostname is specified (disables cert validation) {false} * - period_ms - period of the send state thread in milliseconds {100} * - epoch_ms - period of one epoch in milliseconds {1000} * - Ki, Kp, Kd - PID gains (integral, proportional and derivative) {0., 0., 0.} @@ -345,8 +343,8 @@ namespace e2sar */ struct ReassemblerFlags { - bool cpV6; bool useCP; + bool useHostAddress; u_int16_t period_ms; bool validateCert; float Ki, Kp, Kd, setPoint; @@ -356,7 +354,7 @@ namespace e2sar int eventTimeout_ms; int rcvSocketBufSize; float weight, min_factor, max_factor; - ReassemblerFlags(): cpV6{false}, useCP{true}, + ReassemblerFlags(): useCP{true}, useHostAddress{false}, period_ms{100}, validateCert{true}, Ki{0.}, Kp{0.}, Kd{0.}, setPoint{0.}, epoch_ms{1000}, portRange{-1}, withLBHeader{false}, eventTimeout_ms{500}, rcvSocketBufSize{1024*1024*3}, weight{1.0}, min_factor{0.5}, max_factor{2.0} {} diff --git a/include/e2sarUtil.hpp b/include/e2sarUtil.hpp index 6c9f8348..e33556bf 100644 --- a/include/e2sarUtil.hpp +++ b/include/e2sarUtil.hpp @@ -78,7 +78,7 @@ namespace e2sar * @param uri - the URI string * @param tt - convert to this token type (admin, instance, session) * @param preferV6 - when connecting to the control plane, prefer IPv6 address - * (defaults to v4) + * if the name resolves to both (defaults to v4) */ EjfatURI(const std::string &uri, TokenType tt=TokenType::admin, bool preferV6=false); diff --git a/src/e2sarDPReassembler.cpp b/src/e2sarDPReassembler.cpp index 3b9c120c..0ec19a76 100644 --- a/src/e2sarDPReassembler.cpp +++ b/src/e2sarDPReassembler.cpp @@ -38,7 +38,7 @@ namespace e2sar std::vector cpuCoreList, const ReassemblerFlags &rflags): dpuri(uri), - lbman(dpuri, rflags.validateCert), + lbman(dpuri, rflags.validateCert, rflags.useHostAddress), epochMs{rflags.epoch_ms}, setPoint{rflags.setPoint}, Kp{rflags.Kp}, Ki{rflags.Ki}, Kd{rflags.Kd}, weight{rflags.weight}, min_factor{rflags.min_factor}, max_factor{rflags.max_factor}, @@ -54,7 +54,7 @@ namespace e2sar eventTimeout_ms{rflags.eventTimeout_ms}, rcvSocketBufSize{rflags.rcvSocketBufSize}, condLock{recvThreadMtx}, - sendStateThreadState(*this, rflags.cpV6, rflags.period_ms), + sendStateThreadState(*this, rflags.period_ms), useCP{rflags.useCP} { sanityChecks(); @@ -71,7 +71,7 @@ namespace e2sar Reassembler::Reassembler(const EjfatURI &uri, ip::address data_ip, u_int16_t starting_port, size_t numRecvThreads, const ReassemblerFlags &rflags): dpuri(uri), - lbman(dpuri, rflags.validateCert), + lbman(dpuri, rflags.validateCert, rflags.useHostAddress), epochMs{rflags.epoch_ms}, setPoint{rflags.setPoint}, Kp{rflags.Kp}, Ki{rflags.Ki}, Kd{rflags.Kd}, weight{rflags.weight}, min_factor{rflags.min_factor}, max_factor{rflags.max_factor}, @@ -87,7 +87,7 @@ namespace e2sar eventTimeout_ms{rflags.eventTimeout_ms}, rcvSocketBufSize{rflags.rcvSocketBufSize}, condLock{recvThreadMtx}, - sendStateThreadState(*this, rflags.cpV6, rflags.period_ms), + sendStateThreadState(*this, rflags.period_ms), useCP{rflags.useCP} { sanityChecks(); @@ -541,10 +541,10 @@ namespace e2sar // general rFlags.useCP = paramTree.get("general.useCP", rFlags.useCP); - rFlags.validateCert = paramTree.get("general.validateCert", rFlags.validateCert); // control plane - rFlags.cpV6 = paramTree.get("control-plane.cpV6", rFlags.cpV6); + rFlags.useHostAddress = paramTree.get("control-plane.useHostAddress", rFlags.useHostAddress); + rFlags.validateCert = paramTree.get("control-plane.validateCert", rFlags.validateCert); // data plane rFlags.portRange = paramTree.get("data-plane.portRange", rFlags.portRange); diff --git a/src/pybind/py_e2sarCP.cpp b/src/pybind/py_e2sarCP.cpp index bae545bf..b7196544 100644 --- a/src/pybind/py_e2sarCP.cpp +++ b/src/pybind/py_e2sarCP.cpp @@ -152,9 +152,10 @@ void init_e2sarCP(py::module_ &m) { // Constructor lb_manager.def( - py::init(), + py::init(), py::arg("cpuri"), py::arg("validate_server") = true, + py::arg("use_host_address") = false, py::arg("opts") = grpc::SslCredentialsOptions() ); diff --git a/src/pybind/py_e2sarDP.cpp b/src/pybind/py_e2sarDP.cpp index a9e0700a..c99a672a 100644 --- a/src/pybind/py_e2sarDP.cpp +++ b/src/pybind/py_e2sarDP.cpp @@ -163,8 +163,8 @@ void init_e2sarDP_reassembler(py::module_ &m) // Bind the ReassemblerFlags struct as a nested class of Reassembler py::class_(m, "ReassemblerFlags") .def(py::init<>()) // The default values will be the same in Python after binding. - .def_readwrite("cpV6", &Reassembler::ReassemblerFlags::cpV6) .def_readwrite("useCP", &Reassembler::ReassemblerFlags::useCP) + .def_readwrite("useHostAddress", &Reassembler::ReassemblerFlags::useHostAddress) .def_readwrite("period_ms", &Reassembler::ReassemblerFlags::period_ms) .def_readwrite("validateCert", &Reassembler::ReassemblerFlags::validateCert) .def_readwrite("Ki", &Reassembler::ReassemblerFlags::Ki) From 77f898d23d6f0a56a92a11b3cf08264a71d8eaf6 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Mon, 23 Sep 2024 23:40:04 -0400 Subject: [PATCH 18/73] Modified address schemas, added modified example INI file for ReassemblerFlags --- include/e2sarCP.hpp | 6 +++--- reassembler_config.ini | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/e2sarCP.hpp b/include/e2sarCP.hpp index 53725aa5..6fd79608 100644 --- a/include/e2sarCP.hpp +++ b/include/e2sarCP.hpp @@ -161,15 +161,15 @@ namespace e2sar throw E2SARException("Unable to initialize LBManager due to missing CP address in URI"); auto cp_addr_v = cp_addr_r.value(); if (cp_addr_v.first.is_v4()) - addr_string = "ipv4://" + cp_addr_v.first.to_string() + ":" + std::to_string(cp_addr_v.second); + addr_string = "ipv4:///" + cp_addr_v.first.to_string() + ":" + std::to_string(cp_addr_v.second); else - addr_string = "ipv6://[" + cp_addr_v.first.to_string() + "]:" + std::to_string(cp_addr_v.second); + addr_string = "ipv6:///[" + cp_addr_v.first.to_string() + "]:" + std::to_string(cp_addr_v.second); } if (cpuri.get_useTls()) { grpc::experimental::TlsChannelCredentialsOptions topts; - if (validateServer) + if (!validateServer) { // disable most of server certificate validation std::shared_ptr verifier = std::make_shared(); diff --git a/reassembler_config.ini b/reassembler_config.ini index bdce44bc..de7938ce 100644 --- a/reassembler_config.ini +++ b/reassembler_config.ini @@ -1,12 +1,12 @@ [general] ; whether to use the control plane (gRPC sendState, registerWorker) useCP = true -; validate control plane TLS certificate in gRPC communications -validateCert = true [control-plane] -; use IPv6 address if cp node specified by name and has IPv4 and IPv6 resolution -cpV6 = false +; validate control plane TLS certificate in gRPC communications +validateCert = true +; force using address (v4 or v6) even if hostname specified in the URI +useHostAddress = false [data-plane] ; 2^portRange (0<=portRange<=14) listening ports will be open starting from dataPort. From 2c563e0d95589c4af3ec155b68ad531c0d214c11 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Tue, 24 Sep 2024 10:00:41 -0400 Subject: [PATCH 19/73] Passing flags to reassembler --- bin/e2sar_perf.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/e2sar_perf.cpp b/bin/e2sar_perf.cpp index 95fdd7a1..c6c7bece 100644 --- a/bin/e2sar_perf.cpp +++ b/bin/e2sar_perf.cpp @@ -443,6 +443,8 @@ int main(int argc, char **argv) rflags.useCP = withCP; rflags.withLBHeader = not withCP; rflags.rcvSocketBufSize = sockBufSize; + rflags.useHostAddress = preferHostAddr; + rflags.validateCert = noValidate; } std::cout << "Control plane will be " << (rflags.useCP ? "ON" : "OFF") << std::endl; std::cout << (rflags.useCP ? "*** Make sure the LB has been reserved and the URI reflects the reserved instance information." : From 69e1074d32dbd1c47f5e45ffa2ec6b63f026d38f Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Tue, 24 Sep 2024 10:10:15 -0400 Subject: [PATCH 20/73] Passing the flags correctly this time --- bin/e2sar_perf.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/e2sar_perf.cpp b/bin/e2sar_perf.cpp index c6c7bece..2561f8fe 100644 --- a/bin/e2sar_perf.cpp +++ b/bin/e2sar_perf.cpp @@ -351,9 +351,9 @@ int main(int argc, char **argv) if (vm.count("ipv6") || vm.count("ipv4")) preferHostAddr = true; - bool noValidate = false; + bool validate = true; if (vm.count("novalidate")) - noValidate = true; + validate = false; // make sure the token is interpreted as the correct type, depending on the call EjfatURI::TokenType tt{EjfatURI::TokenType::instance}; @@ -377,7 +377,7 @@ int main(int argc, char **argv) senders.push_back(sndrcvIP); // create LBManager - lbmPtr = new LBManager(uri, noValidate, preferHostAddr); + lbmPtr = new LBManager(uri, validate, preferHostAddr); // register senders std::cout << "Adding senders to LB: "; @@ -444,7 +444,7 @@ int main(int argc, char **argv) rflags.withLBHeader = not withCP; rflags.rcvSocketBufSize = sockBufSize; rflags.useHostAddress = preferHostAddr; - rflags.validateCert = noValidate; + rflags.validateCert = validate; } std::cout << "Control plane will be " << (rflags.useCP ? "ON" : "OFF") << std::endl; std::cout << (rflags.useCP ? "*** Make sure the LB has been reserved and the URI reflects the reserved instance information." : From 5da4dfadf3842fc61befd74871c5ab159c108663 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Tue, 24 Sep 2024 10:56:38 -0400 Subject: [PATCH 21/73] Fixing sendstate --- src/e2sarDPReassembler.cpp | 76 +++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/src/e2sarDPReassembler.cpp b/src/e2sarDPReassembler.cpp index 0ec19a76..ce86e568 100644 --- a/src/e2sarDPReassembler.cpp +++ b/src/e2sarDPReassembler.cpp @@ -395,13 +395,24 @@ namespace e2sar void Reassembler::SendStateThreadState::_threadBody() { + // get the time + auto nowT = boost::chrono::high_resolution_clock::now(); + auto nowUsec = boost::chrono::duration_cast(nowT.time_since_epoch()).count(); + UnixTimeMicro_t currentTimeMicros = static_cast(nowUsec); + + // create first PID sample with 0 error and integral values + PIDSample newSample{currentTimeMicros, 0.0, 0.0}; + // push a new entry onto the circular buffer ejecting the oldest + reas.pidSampleBuffer.push_back(newSample); + + // wait before entering the loop + auto until = nowT + boost::chrono::milliseconds(period_ms); + boost::this_thread::sleep_until(until); + while(!reas.threadsStop) { // periodically send state to control plane sampling the queue state - // Get the current time point - auto nowT = boost::chrono::high_resolution_clock::now(); - // principle of operation: // CP requires PID signal and queue fill state in order to come up // with a schedule for a new epoch. The CP and receiver nodes running @@ -418,39 +429,38 @@ namespace e2sar // // fillPercent is always reported as sampled in the current moment - // if there are no samples in the buffer just skip - if (reas.pidSampleBuffer.end() != reas.pidSampleBuffer.begin()) + // Get the current time point + auto nowT = boost::chrono::high_resolution_clock::now(); + auto nowUsec = boost::chrono::duration_cast(nowT.time_since_epoch()).count(); + UnixTimeMicro_t currentTimeMicros = static_cast(nowUsec); + + // at 100msec period and depth of 10 this should normally be about 1 sec + auto deltaTfloat = static_cast(currentTimeMicros - + reas.pidSampleBuffer.begin()->sampleTime)/1000000.; + + // sample queue state + auto fillPercent = static_cast(static_cast(reas.eventQueueDepth)/static_cast(reas.QSIZE)); + // get PID terms (PID value, error, integral accumulator) + auto PIDTuple = pid(reas.setPoint, fillPercent, + deltaTfloat, reas.Kp, reas.Ki, reas.Kd, + reas.pidSampleBuffer.begin()->error, + reas.pidSampleBuffer.end()->integral); + + // create new PID sample using last error and integral accumulated value + PIDSample newSample{currentTimeMicros, PIDTuple.get<1>(), PIDTuple.get<2>()}; + // push a new entry onto the circular buffer ejecting the oldest + reas.pidSampleBuffer.push_back(newSample); + + // send update to CP + auto res = reas.lbman.sendState(fillPercent, PIDTuple.get<0>(), true); + if (res.has_error()) { - auto nowUsec = boost::chrono::duration_cast(nowT.time_since_epoch()).count(); - UnixTimeMicro_t currentTimeMicros = static_cast(nowUsec); - - // at 100msec period and depth of 10 this should normally be about 1 sec - auto deltaTfloat = static_cast(currentTimeMicros - - reas.pidSampleBuffer.begin()->sampleTime)/1000000.; - - // sample queue state - auto fillPercent = static_cast(static_cast(reas.eventQueueDepth)/static_cast(reas.QSIZE)); - // get PID terms (PID value, error, integral accumulator) - auto PIDTuple = pid(reas.setPoint, fillPercent, - deltaTfloat, reas.Kp, reas.Ki, reas.Kd, - reas.pidSampleBuffer.begin()->error, - reas.pidSampleBuffer.end()->integral); - - // create new PID sample using last error and integral accumulated value - PIDSample newSample{currentTimeMicros, PIDTuple.get<1>(), PIDTuple.get<2>()}; - // push a new entry onto the circular buffer ejecting the oldest - reas.pidSampleBuffer.push_back(newSample); - - // send update to CP - auto res = reas.lbman.sendState(fillPercent, PIDTuple.get<0>(), true); - if (res.has_error()) - { - // update error counts - reas.recvStats.grpcErrCnt++; - reas.recvStats.lastE2SARError = res.error().code(); - } + // update error counts + reas.recvStats.grpcErrCnt++; + reas.recvStats.lastE2SARError = res.error().code(); } + // sleep approximately so we wake up every ~100ms auto until = nowT + boost::chrono::milliseconds(period_ms); boost::this_thread::sleep_until(until); From e680196ae25d5f62c131ad0c46eca763b891de3c Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Tue, 24 Sep 2024 11:31:05 -0400 Subject: [PATCH 22/73] Being careful with circular buffer --- src/e2sarDPReassembler.cpp | 8 ++++---- test/boost_test.cpp | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/e2sarDPReassembler.cpp b/src/e2sarDPReassembler.cpp index ce86e568..5572ad4b 100644 --- a/src/e2sarDPReassembler.cpp +++ b/src/e2sarDPReassembler.cpp @@ -404,7 +404,7 @@ namespace e2sar PIDSample newSample{currentTimeMicros, 0.0, 0.0}; // push a new entry onto the circular buffer ejecting the oldest reas.pidSampleBuffer.push_back(newSample); - + // wait before entering the loop auto until = nowT + boost::chrono::milliseconds(period_ms); boost::this_thread::sleep_until(until); @@ -436,15 +436,15 @@ namespace e2sar // at 100msec period and depth of 10 this should normally be about 1 sec auto deltaTfloat = static_cast(currentTimeMicros - - reas.pidSampleBuffer.begin()->sampleTime)/1000000.; + reas.pidSampleBuffer.front().sampleTime)/1000000.; // sample queue state auto fillPercent = static_cast(static_cast(reas.eventQueueDepth)/static_cast(reas.QSIZE)); // get PID terms (PID value, error, integral accumulator) auto PIDTuple = pid(reas.setPoint, fillPercent, deltaTfloat, reas.Kp, reas.Ki, reas.Kd, - reas.pidSampleBuffer.begin()->error, - reas.pidSampleBuffer.end()->integral); + reas.pidSampleBuffer.front().error, + reas.pidSampleBuffer.back().integral); // create new PID sample using last error and integral accumulated value PIDSample newSample{currentTimeMicros, PIDTuple.get<1>(), PIDTuple.get<2>()}; diff --git a/test/boost_test.cpp b/test/boost_test.cpp index 26d28d04..4650349f 100644 --- a/test/boost_test.cpp +++ b/test/boost_test.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include @@ -312,5 +313,22 @@ int main() std::cout << map[std::make_pair(0x123456, 1)] << std::endl; std::cout << map[std::make_pair(0x123456, 2)] << std::endl; std::cout << map[std::make_pair(0x1234567, 10)] << std::endl; + + + std::cout << "Test circular buffer" << std::endl; + + boost::circular_buffer pidSampleBuffer(5); + + for (int i = 0; i<6; i++) + pidSampleBuffer.push_back(i); + + std::cout << "Head of buffer " << pidSampleBuffer.front() << std::endl; + std::cout << "Tail of buffer " << pidSampleBuffer.back() << std::endl; + + for (int i = 10; i<20; i++) + pidSampleBuffer.push_back(i); + + std::cout << "Head of buffer " << pidSampleBuffer.front() << std::endl; + std::cout << "Tail of buffer " << pidSampleBuffer.back() << std::endl; } From 102ffc87334c3b2bffb186fdf9bc7e3013b49bac Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Tue, 24 Sep 2024 16:42:45 -0400 Subject: [PATCH 23/73] Changing the order of registerWorker and openAndStart in reassembler invocation --- bin/e2sar_perf.cpp | 21 ++++++++++++--------- src/e2sarDPReassembler.cpp | 3 +-- src/e2sarDPSegmenter.cpp | 2 +- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/bin/e2sar_perf.cpp b/bin/e2sar_perf.cpp index 2561f8fe..d64cc9d1 100644 --- a/bin/e2sar_perf.cpp +++ b/bin/e2sar_perf.cpp @@ -168,15 +168,6 @@ result recvEvents(Reassembler &r, int durationSec) { EventNum_t evtNum; u_int16_t dataId; - auto openRes = r.openAndStart(); - if (openRes.has_error()) - return openRes; - - // to help print large integers - std::cout.imbue(std::locale("")); - - auto nowT = boost::chrono::steady_clock::now(); - // register the worker (will be NOOP if withCP is set to false) auto hostname_res = NetUtil::getHostName(); if (hostname_res.has_error()) @@ -192,6 +183,18 @@ result recvEvents(Reassembler &r, int durationSec) { if (regres.value() == 1) std::cout << "Registered the worker" << std::endl; + // NOTE: if we switch the order of registerWorker and openAndStart + // you get into a race condition where the sendState thread starts and tries + // to send queue updates, however session token is not yet available... + auto openRes = r.openAndStart(); + if (openRes.has_error()) + return openRes; + + // to help print large integers + std::cout.imbue(std::locale("")); + + auto nowT = boost::chrono::steady_clock::now(); + // receive loop while(true) { diff --git a/src/e2sarDPReassembler.cpp b/src/e2sarDPReassembler.cpp index 5572ad4b..c5c92f65 100644 --- a/src/e2sarDPReassembler.cpp +++ b/src/e2sarDPReassembler.cpp @@ -537,7 +537,7 @@ namespace e2sar return 0; } - result Reassembler::ReassemblerFlags::getFromINI(const std::string &iniFile) + result Reassembler::ReassemblerFlags::getFromINI(const std::string &iniFile) noexcept { boost::property_tree::ptree paramTree; Reassembler::ReassemblerFlags rFlags; @@ -573,7 +573,6 @@ namespace e2sar rFlags.Kd = paramTree.get("pid.min_factor", rFlags.Kd); rFlags.Kd = paramTree.get("pid.max_factor", rFlags.Kd); - return rFlags; } } diff --git a/src/e2sarDPSegmenter.cpp b/src/e2sarDPSegmenter.cpp index 17642460..5814c2a2 100644 --- a/src/e2sarDPSegmenter.cpp +++ b/src/e2sarDPSegmenter.cpp @@ -599,7 +599,7 @@ namespace e2sar return 0; } - result Segmenter::SegmenterFlags::getFromINI(const std::string &iniFile) + result Segmenter::SegmenterFlags::getFromINI(const std::string &iniFile) noexcept { boost::property_tree::ptree paramTree; Segmenter::SegmenterFlags sFlags; From f68b18943025f8f87d117c0df4fc502dac988018 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Tue, 24 Sep 2024 16:44:56 -0400 Subject: [PATCH 24/73] Marking methods noexcept --- include/e2sarDPReassembler.hpp | 2 +- include/e2sarDPSegmenter.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/e2sarDPReassembler.hpp b/include/e2sarDPReassembler.hpp index 964d9523..b6b9c3ac 100644 --- a/include/e2sarDPReassembler.hpp +++ b/include/e2sarDPReassembler.hpp @@ -362,7 +362,7 @@ namespace e2sar * Initialize flags from an INI file * @param iniFile - path to the INI file */ - static result getFromINI(const std::string &iniFile); + static result getFromINI(const std::string &iniFile) noexcept; }; /** * Create a reassembler object to run receive on a specific set of CPU cores diff --git a/include/e2sarDPSegmenter.hpp b/include/e2sarDPSegmenter.hpp index e327a050..0e555325 100644 --- a/include/e2sarDPSegmenter.hpp +++ b/include/e2sarDPSegmenter.hpp @@ -290,7 +290,7 @@ namespace e2sar * Initialize flags from an INI file * @param iniFile - path to the INI file */ - static result getFromINI(const std::string &iniFile); + static result getFromINI(const std::string &iniFile) noexcept; }; /** * Initialize segmenter state. Call openAndStart() to begin operation. From 8d3967234f2f87d74dd79ed571c9a20eecd59380 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Tue, 24 Sep 2024 17:15:53 -0400 Subject: [PATCH 25/73] Added firewall rules to allow UDP traffic --- scripts/notebooks/JIRIAF/FABRIC_JIRIAF.ipynb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/notebooks/JIRIAF/FABRIC_JIRIAF.ipynb b/scripts/notebooks/JIRIAF/FABRIC_JIRIAF.ipynb index 0838b6fc..17b8c2d4 100644 --- a/scripts/notebooks/JIRIAF/FABRIC_JIRIAF.ipynb +++ b/scripts/notebooks/JIRIAF/FABRIC_JIRIAF.ipynb @@ -658,6 +658,9 @@ " for subnet, portlist in open_ports.items():\n", " for port in portlist:\n", " commands.append(f'sudo firewall-cmd --permanent --zone=public --add-rich-rule=\\'rule family=\\\"ipv4\\\" source address=\\\"{subnet}\\\" port protocol=\\\"tcp\\\" port=\\\"{port}\\\" accept\\'')\n", + " \n", + " for subnet in external_subnets:\n", + " commands.append(f'sudo firewall-cmd --permanent --zone=public --add-rich-rule=\\'rule family=\\\"ipv4\\\" source address=\\\"{subnet}\\\" protocol value=\\\"udp\\\" accept\\'')\n", " commands.append(f'sudo firewall-cmd --reload')\n", " commands.append(f'sudo firewall-cmd --list-all --zone=public')\n", " execute_single_node(node, commands)\n", From aeb192ccf4b74049acb1cd0d65bf3319828562be Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Tue, 24 Sep 2024 17:17:37 -0400 Subject: [PATCH 26/73] Added UDP firewall rules now that firewalld is installed by default --- .../EJFAT/E2SAR-development-tester.ipynb | 66 +++++++++++++++++-- .../EJFAT/E2SAR-release-tester.ipynb | 28 +++++++- scripts/notebooks/EJFAT/post-boot/recver.sh | 3 +- scripts/notebooks/EJFAT/post-boot/sender.sh | 3 +- 4 files changed, 91 insertions(+), 9 deletions(-) diff --git a/scripts/notebooks/EJFAT/E2SAR-development-tester.ipynb b/scripts/notebooks/EJFAT/E2SAR-development-tester.ipynb index ef968293..65076414 100644 --- a/scripts/notebooks/EJFAT/E2SAR-development-tester.ipynb +++ b/scripts/notebooks/EJFAT/E2SAR-development-tester.ipynb @@ -54,7 +54,7 @@ "\n", "# branches for UDPLBd and E2SAR that we want checked out on the VMs\n", "udplbd_branch = 'develop'\n", - "e2sar_branch = 'main'\n", + "e2sar_branch = 'e2sar-perf-with-cp'\n", "\n", "# which of the available config files to use with UDPLBd\n", "udplbd_config = 'lb_mock-tls.yml'\n", @@ -104,6 +104,8 @@ "from ipaddress import ip_address, IPv4Address, IPv6Address, IPv4Network, IPv6Network\n", "import ipaddress\n", "\n", + "import json\n", + "\n", "fablib = fablib_manager() \n", "fablib.show_config();\n", "\n", @@ -142,7 +144,8 @@ "# this is the NIC to use\n", "nic_model = 'NIC_Basic'\n", "# the subnet should match IPs\n", - "subnet = IPv4Network(\"192.168.1.0/24\")\n", + "subnet_str = \"192.168.0.0/24\" \n", + "subnet = IPv4Network(subnet_str)\n", "\n", "def execute_single_node(node, commands):\n", " for command in commands:\n", @@ -157,7 +160,32 @@ " for n in node:\n", " execute_single_node(n, commands)\n", " else:\n", - " execute_single_node(node, commands)\n" + " execute_single_node(node, commands)\n", + "\n", + "# until fablib fixes this\n", + "def get_management_os_interface(node) -> str or None:\n", + " \"\"\"\n", + " Gets the name of the management interface used by the node's\n", + " operating system. \n", + "\n", + " :return: interface name\n", + " :rtype: String\n", + " \"\"\"\n", + " stdout, stderr = node.execute(\"sudo ip -j route list\", quiet=True)\n", + " stdout_json = json.loads(stdout)\n", + "\n", + " for i in stdout_json:\n", + " if i[\"dst\"] == \"default\":\n", + " return i[\"dev\"]\n", + "\n", + " stdout, stderr = node.execute(\"sudo ip -6 -j route list\", quiet=True)\n", + " stdout_json = json.loads(stdout)\n", + "\n", + " for i in stdout_json:\n", + " if i[\"dst\"] == \"default\":\n", + " return i[\"dev\"]\n", + "\n", + " return None\n" ] }, { @@ -630,7 +658,7 @@ " f\"sudo sysctl net.core.wmem_max=536870912\",\n", " f\"sysctl net.core.wmem_max net.core.rmem_max\"\n", "]\n", - "execute_commands([sender, recver], commands);" + "execute_commands([sender, recver], commands)" ] }, { @@ -653,6 +681,32 @@ "stdout, stderr = sender.execute(f\"sudo ping -f -s 8972 -c 10 -M do {recver_addr}\")" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "725b9489-2ddb-4f94-b761-6e2eafc7f876", + "metadata": {}, + "outputs": [], + "source": [ + "# We need to setup the firewall to allow traffic to pass to the receiver\n", + "\n", + "mgmt_iface_name = get_management_os_interface(recver)\n", + "data_iface = recver.get_interface(network_name=net_name)\n", + "data_iface_name = data_iface.get_os_interface()\n", + "\n", + "print(f'Adding {mgmt_iface_name} and lo and data interface to trusted zone')\n", + "commands = [\n", + " f'sudo firewall-cmd --permanent --zone=trusted --add-interface={data_iface_name}',\n", + " f'sudo firewall-cmd --permanent --zone=trusted --add-interface=lo',\n", + " f'sudo firewall-cmd --permanent --zone=trusted --add-interface={mgmt_iface_name}',\n", + " f'for i in $(sudo firewall-cmd --zone=public --list-services); do sudo firewall-cmd --zone=public --permanent --remove-service=$i; done',\n", + "]\n", + "commands.append(f'sudo firewall-cmd --reload')\n", + "commands.append(f'sudo firewall-cmd --list-all --zone=public')\n", + "\n", + "execute_commands([recver], commands)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -673,7 +727,7 @@ "bufSize = 300 * 1024 * 1024 # 100MB send and receive buffers\n", "\n", "recv_command = f\"cd E2SAR; PATH=$HOME/.local/bin:/usr/local/bin:$PATH LD_LIBRARY_PATH=/usr/local/lib/ ./build/bin/e2sar_perf -r -u '{e2sarPerfURI}' -d {recverDuration} -b {bufSize} --ip {recver_addr} --port 19522\"\n", - "send_command = f\"cd E2SAR; PATH=$HOME/.local/bin:/usr/local/bin:$PATH LD_LIBRARY_PATH=/usr/local/lib/ ./build/bin/e2sar_perf -s -u '{e2sarPerfURI}' --mtu {mtu} --rate {rate} --length {length} -n {numEvents} -b {bufSize}\"\n", + "send_command = f\"cd E2SAR; PATH=$HOME/.local/bin:/usr/local/bin:$PATH LD_LIBRARY_PATH=/usr/local/lib/ ./build/bin/e2sar_perf -s -u '{e2sarPerfURI}' --mtu {mtu} --rate {rate} --length {length} -n {numEvents} -b {bufSize} --ip {sender_addr}\"\n", "\n", "# start the receiver for 10 seconds and log its output\n", "print(f'Executing command {recv_command} on receiver')\n", @@ -761,7 +815,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.8" + "version": "3.12.3" } }, "nbformat": 4, diff --git a/scripts/notebooks/EJFAT/E2SAR-release-tester.ipynb b/scripts/notebooks/EJFAT/E2SAR-release-tester.ipynb index 2e962fd7..885184f3 100644 --- a/scripts/notebooks/EJFAT/E2SAR-release-tester.ipynb +++ b/scripts/notebooks/EJFAT/E2SAR-release-tester.ipynb @@ -567,6 +567,32 @@ "stdout, stderr = sender.execute(f\"sudo ping -f -s 8972 -c 10 -M do {recver_addr}\")" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "a2cb3e7e-902d-4971-8f37-16e74924a6c0", + "metadata": {}, + "outputs": [], + "source": [ + "# We need to setup the firewall to allow traffic to pass to the receiver\n", + "\n", + "mgmt_iface_name = get_management_os_interface(recver)\n", + "data_iface = recver.get_interface(network_name=net_name)\n", + "data_iface_name = data_iface.get_os_interface()\n", + "\n", + "print(f'Adding {mgmt_iface_name} and lo and data interface to trusted zone')\n", + "commands = [\n", + " f'sudo firewall-cmd --permanent --zone=trusted --add-interface={data_iface_name}',\n", + " f'sudo firewall-cmd --permanent --zone=trusted --add-interface=lo',\n", + " f'sudo firewall-cmd --permanent --zone=trusted --add-interface={mgmt_iface_name}',\n", + " f'for i in $(sudo firewall-cmd --zone=public --list-services); do sudo firewall-cmd --zone=public --permanent --remove-service=$i; done',\n", + "]\n", + "commands.append(f'sudo firewall-cmd --reload')\n", + "commands.append(f'sudo firewall-cmd --list-all --zone=public')\n", + "\n", + "execute_commands([recver], commands)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -683,7 +709,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.8" + "version": "3.12.3" } }, "nbformat": 4, diff --git a/scripts/notebooks/EJFAT/post-boot/recver.sh b/scripts/notebooks/EJFAT/post-boot/recver.sh index 0eff9e9c..00c91387 100644 --- a/scripts/notebooks/EJFAT/post-boot/recver.sh +++ b/scripts/notebooks/EJFAT/post-boot/recver.sh @@ -22,7 +22,8 @@ if [[ ${distro} == 'ubuntu' ]]; then # install missing software sudo apt-get -yq update - sudo apt-get -yq install python3-pip build-essential autoconf cmake libtool pkg-config libglib2.0-dev ninja-build openssl libssl-dev libsystemd-dev protobuf-compiler libre2-dev gdb docker.io + sudo dpkg -r ufw + sudo apt-get -yq install python3-pip build-essential autoconf cmake libtool pkg-config libglib2.0-dev ninja-build openssl libssl-dev libsystemd-dev protobuf-compiler libre2-dev gdb docker.io firewalld # install meson pip3 install --user meson pybind11 diff --git a/scripts/notebooks/EJFAT/post-boot/sender.sh b/scripts/notebooks/EJFAT/post-boot/sender.sh index e60b8a80..f2e88898 100644 --- a/scripts/notebooks/EJFAT/post-boot/sender.sh +++ b/scripts/notebooks/EJFAT/post-boot/sender.sh @@ -22,7 +22,8 @@ if [[ ${distro} == 'ubuntu' ]]; then # install missing software sudo apt-get -yq update - sudo apt-get -yq install python3-pip build-essential autoconf cmake libtool pkg-config libglib2.0-dev ninja-build openssl libssl-dev libsystemd-dev protobuf-compiler libre2-dev gdb docker.io + sudo dpkg -r ufw + sudo apt-get -yq install python3-pip build-essential autoconf cmake libtool pkg-config libglib2.0-dev ninja-build openssl libssl-dev libsystemd-dev protobuf-compiler libre2-dev gdb docker.io firewalld # install meson pip3 install --user meson pybind11 From 3d126e4cd822fb4c45b1b510c630ecd5f5a4e447 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Tue, 24 Sep 2024 17:17:56 -0400 Subject: [PATCH 27/73] Notebook testing against live LB --- .../EJFAT/E2SAR-live-lb-tester.ipynb | 1163 +++++++++++++++++ 1 file changed, 1163 insertions(+) create mode 100644 scripts/notebooks/EJFAT/E2SAR-live-lb-tester.ipynb diff --git a/scripts/notebooks/EJFAT/E2SAR-live-lb-tester.ipynb b/scripts/notebooks/EJFAT/E2SAR-live-lb-tester.ipynb new file mode 100644 index 00000000..d5d8afc2 --- /dev/null +++ b/scripts/notebooks/EJFAT/E2SAR-live-lb-tester.ipynb @@ -0,0 +1,1163 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "80a4ca6f-ab03-4de5-8488-8b67ba0ec4d1", + "metadata": {}, + "source": [ + "# Live LB on FABRIC \n", + "\n", + "This notebook helps sets up a sender and multiple receiver nodes on FABRIC such that they can communicate with the production LB. One of the nodes is designated as a sender and the rest as worker-receivers.\n", + "\n", + "See the following diagram:\n", + "\n", + "
\n", + " \n", + "
\n" + ] + }, + { + "cell_type": "markdown", + "id": "ce5948ab-7637-4fd7-a6c1-0c31e555928e", + "metadata": {}, + "source": [ + "## Preamble\n", + "\n", + "This code should *always* be executed regardless of whether you are starting a new slice or returning to an existing slice." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7d876e2c-21b6-4ac0-9c56-eb04771da7c5", + "metadata": {}, + "outputs": [], + "source": [ + "#\n", + "# EDIT THIS\n", + "#\n", + "\n", + "# GitHub SSH key file (private) registered using the GitHubSSH.ipynb notebook referenced above\n", + "github_key = '/home/fabric/work/fabric_confi/github_ecdsa'\n", + "\n", + "# Note for best management network IPv4 connectivity pick from\n", + "# 'UCSD', 'SRI', 'FIU' or 'TOKY' - these sites have\n", + "# IPv4. Other sites use IPv6 management and have trouble\n", + "# retrieving git-lfs artifacts.\n", + "\n", + "# ESnet-FABRIC gateway is at STAR, so the closer we are to it, the lower\n", + "# the latency and loss.\n", + "\n", + "# site_list_override = None\n", + "\n", + "# if you want to force a site list instead of using random\n", + "#site_list_override = ['SRI', 'UCSD', 'CLEM']\n", + "\n", + "# (super)core sites - should be low loss\n", + "site_list_override = ['NEWY', 'WASH', 'LOSA', 'DALL', 'ATLA']\n", + "\n", + "# grouped around STAR with optical connections to the backbone - should be low loss\n", + "#site_list_override = ['STAR', 'INDI', 'NCSA', 'MICH']\n", + "\n", + "# high capacity sites (may have losses at high bandwidth)\n", + "# site_list_override = ['STAR', 'INDI', 'NCSA', 'TACC', 'UCSD', 'PSC']\n", + "\n", + "# these we always exclude\n", + "site_exclude_list = ['EDUKY', 'EDC']\n", + "\n", + "# how many workers do we want? (in addition to one sender)\n", + "number_of_workers = 3\n", + "\n", + "# base distro 'ubuntu2[012]' or 'rocky[89]'\n", + "distro_name = 'ubuntu22'\n", + "distro_version = distro_name[-2:]\n", + "\n", + "# map from distro to image name\n", + "images = {\n", + " 'ubuntu20': 'default_ubuntu_20',\n", + " 'ununtu21': 'default_ubuntu_21',\n", + " 'ubuntu22': 'default_ubuntu_22',\n", + " 'rocky8': 'default_rocky_8',\n", + " 'rocky9': 'default_rocky_9',\n", + "}\n", + "\n", + "# note that the below is distribution specific ('ubuntu' for ubuntu and so on)\n", + "home_location = {\n", + " 'ubunt': '/home/ubuntu',\n", + " 'rocky' : '/home/rocky'\n", + "}[distro_name[:5]]\n", + "\n", + "vm_key_location = f'{home_location}/.ssh/github_ecdsa'\n", + "\n", + "# worker dimensions\n", + "node_attribs = {\n", + " 'cores': 8,\n", + " 'disk': 100,\n", + " 'ram': 24,\n", + " 'image': images[distro_name]\n", + "}\n", + "\n", + "# slice name\n", + "slice_name = f'{number_of_workers + 1}-node LB Tester Slice using {distro_name}'\n", + "\n", + "# these are subnets we want to be able to route to/from\n", + "# The list has the form ['192.168.100.0/24', '10.100.1.0/24']\n", + "external_subnets = []\n", + "\n", + "# these are the lists of destination ports we allow to be open on the FABNet interface\n", + "# for incoming traffic from different subnets. The dictionary has the form\n", + "# { '192.168.100.0/24': [22, 443] } - the key is the source subnet and the value\n", + "# is a list of destination ports allowed from that subnet\n", + "open_ports = {\n", + "}\n", + "\n", + "# additional accounts and their public keys - they get sudo rights and docker,\n", + "# their public keys are expected to reside under ssh-keys/ in a file\n", + "# named after the account.\n", + "# The list has the form of ['user1', 'user2'] where user1 and user2 accounts\n", + "# will be created on the system. Under ssh-keys/ there should be two files\n", + "# named 'user1' and 'user2' each containing the SSH public key for that user. \n", + "accounts = []\n", + "\n", + "# url of e2sar deps. Find the appropriate version for the OS at https://github.com/JeffersonLab/E2SAR/releases\n", + "e2sar_branch = \"e2sar-perf-with-cp\"\n", + "static_release_url = 'https://github.com/JeffersonLab/E2SAR/releases/download/' # don't need to change this\n", + "e2sar_dep_artifcat = 'e2sar-deps_0.1.1_amd64.deb'\n", + "e2sar_release_ver = 'E2SAR-0.1.1'\n", + "e2sar_dep_url = static_release_url + e2sar_release_ver + \"-\" + distro_name[:-2] + \"-\" + distro_version + \".04/\" + e2sar_dep_artifcat\n", + "\n", + "#\n", + "# SHOULDN'T NEED TO EDIT BELOW\n", + "#\n", + "# Preamble\n", + "import json\n", + "from datetime import datetime\n", + "from datetime import timezone\n", + "from datetime import timedelta\n", + "\n", + "from fabrictestbed_extensions.fablib.fablib import FablibManager as fablib_manager\n", + "\n", + "from ipaddress import ip_address, IPv4Address, IPv6Address, IPv4Network, IPv6Network\n", + "import ipaddress\n", + "\n", + "fablib = fablib_manager() \n", + "fablib.show_config();\n", + "\n", + "# gets prepended to site name - this network is per site\n", + "net_name_prefix = 'fabnetv4ext'\n", + "\n", + "# this is the NIC to use\n", + "nic_model = 'NIC_Basic'\n", + "\n", + "def execute_single_node(node, commands):\n", + " for command in commands:\n", + " print(f'\\tExecuting \"{command}\" on node {node.get_name()}')\n", + " #stdout, stderr = node.execute(command, quiet=True, output_file=node.get_name() + '_install.log')\n", + " stdout, stderr = node.execute(command)\n", + " if not stderr and len(stderr) > 0:\n", + " print(f'Error encountered with \"{command}\": {stderr}')\n", + " \n", + "def execute_commands(node, commands):\n", + " if isinstance(node, list):\n", + " for n in node:\n", + " execute_single_node(n, commands)\n", + " else:\n", + " execute_single_node(node, commands)\n", + "\n", + "def execute_single_node_on_thread(node, commands):\n", + " # concatenate the commands using ';' and execute\n", + " allcommands = ';'.join(commands)\n", + " node.execute_thread(allcommands, output_file=node.get_name() + '_thread.log')\n", + "\n", + "def execute_commands_on_threads(node, commands):\n", + " if isinstance(node, list):\n", + " for n in node:\n", + " execute_single_node_on_thread(n, commands)\n", + " else:\n", + " execute_single_node_on_thread(node, commands)\n", + "\n", + "def make_node_name(site_name, node_idx):\n", + " return '_'.join([f\"Worker{node_idx}\", site_name])\n", + "\n", + "def make_net_name(site_name):\n", + " return '_'.join([net_name_prefix, site_name])\n", + "\n", + "# return slice with one node on one site\n", + "def starter_slice(site_name):\n", + " #node_name = make_node_name(site_name, 1)\n", + " node_name = '_'.join([\"Sender\", site_name])\n", + " net_name = make_net_name(site_name)\n", + "\n", + " slice = fablib.new_slice(name=slice_name)\n", + " node = slice.add_node(name=node_name, site=site_name, **node_attribs)\n", + "\n", + " # postboot configuration is under 'post-boot' directory\n", + " node.add_post_boot_upload_directory('post-boot','.')\n", + " node.add_post_boot_execute(f'chmod +x post-boot/sender.sh && ./post-boot/sender.sh')\n", + " \n", + " # attach to network\n", + " nic_interface = node.add_component(model=nic_model, name='_'.join([node_name, nic_model, 'nic'])).get_interfaces()[0]\n", + " net = slice.add_l3network(name=net_name, interfaces=[nic_interface], type='IPv4Ext')\n", + "\n", + " return slice\n", + "\n", + "def add_node_to_slice(site_name, node_idx, inc, slice):\n", + "\n", + " net_name = make_net_name(site_name)\n", + "\n", + " while inc > 0:\n", + " node_name = make_node_name(site_name, node_idx)\n", + " node_idx += 1\n", + " \n", + " node = slice.add_node(name=node_name, site=site_name, **node_attribs)\n", + " \n", + " # postboot configuration is under 'post-boot' directory\n", + " node.add_post_boot_upload_directory('post-boot','.')\n", + " node.add_post_boot_execute(f'chmod +x post-boot/recver.sh && ./post-boot/recver.sh')\n", + " \n", + " nic_interface = node.add_component(model=nic_model, name='_'.join([node_name, nic_model, 'nic'])).get_interfaces()[0]\n", + " \n", + " # attach to a network, create network if needed\n", + " net = slice.get_network(name=net_name)\n", + " if net is None:\n", + " net = slice.add_l3network(name=net_name, type='IPv4Ext')\n", + " \n", + " net.add_interface(nic_interface)\n", + " inc -= 1\n", + "\n", + " return None\n", + "\n", + "def check_modify(slice, selected_site_list, nodes_in_slice, expected_to_add):\n", + "\n", + " success = True\n", + " idx = 1\n", + " while(expected_to_add >= idx):\n", + " # find sliver reservation for new node\n", + " node_sliver = slice.list_slivers(fields=['name', 'state'], \n", + " filter_function=lambda x: x['type'] == 'node' and \n", + " x['name'] == make_node_name(selected_site_list[0], nodes_in_slice + idx) and \n", + " x['state'] == 'Active')\n", + " # if it is none - it failed\n", + " if node_sliver is None:\n", + " success = False\n", + " break\n", + " else:\n", + " idx += 1\n", + "\n", + " return success\n", + "\n", + "# until fablib fixes this\n", + "def get_management_os_interface(node) -> str or None:\n", + " \"\"\"\n", + " Gets the name of the management interface used by the node's\n", + " operating system. \n", + "\n", + " :return: interface name\n", + " :rtype: String\n", + " \"\"\"\n", + " stdout, stderr = node.execute(\"sudo ip -j route list\", quiet=True)\n", + " stdout_json = json.loads(stdout)\n", + "\n", + " for i in stdout_json:\n", + " if i[\"dst\"] == \"default\":\n", + " return i[\"dev\"]\n", + "\n", + " stdout, stderr = node.execute(\"sudo ip -6 -j route list\", quiet=True)\n", + " stdout_json = json.loads(stdout)\n", + "\n", + " for i in stdout_json:\n", + " if i[\"dst\"] == \"default\":\n", + " return i[\"dev\"]\n", + "\n", + " return None" + ] + }, + { + "cell_type": "markdown", + "id": "6c66e2c1-84d0-4228-b552-601b2909c237", + "metadata": {}, + "source": [ + "## Helpers\n", + "\n", + "If you ever forget which images are available, run this cell:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "38130b75-8dc8-43c8-83cb-497a4768fc53", + "metadata": {}, + "outputs": [], + "source": [ + "# List available images (this step is optional)\n", + "available_images = fablib.get_image_names()\n", + "\n", + "print(f'Available images are: {available_images}')" + ] + }, + { + "cell_type": "markdown", + "id": "fbd114ff-be4f-4be6-8326-e7b088a27e3c", + "metadata": {}, + "source": [ + "## Prepare to create a new slice (skip if exists)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ec766700-8245-4e4e-bf15-9630f4600456", + "metadata": {}, + "outputs": [], + "source": [ + "# list all slices I have running\n", + "output_dataframe = fablib.list_slices(output='pandas')\n", + "if output_dataframe:\n", + " print(output_dataframe)\n", + "else:\n", + " print('No active slices under this project')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9c4249e1-42aa-4575-8053-0edf4d95c3fd", + "metadata": {}, + "outputs": [], + "source": [ + "# Identify sites in continental US we want to use (NOOP if override is set)\n", + "lon_west=-124.3993243\n", + "lon_east=-69.9721573\n", + "candidate_sites = 7\n", + "free_nodes_worth = 3 # how many nodes worth are we looking per site\n", + "\n", + "# get a list of random sites, avoiding thos on the exclude list\n", + "# unless there is an override\n", + "if site_list_override is None:\n", + " selected_site_list = fablib.get_random_sites(count=candidate_sites, avoid=site_exclude_list,\n", + " filter_function=lambda x: x['location'][1] < lon_east\n", + " and x['location'][1] > lon_west \n", + " and x['cores_available'] > free_nodes_worth * node_attribs['cores']\n", + " and x['ram_available'] > free_nodes_worth * node_attribs['ram'] \n", + " and x['disk_available'] > free_nodes_worth * node_attribs['disk']) \n", + "else:\n", + " selected_site_list = site_list_override\n", + "\n", + "if selected_site_list:\n", + " print(f'Selected sites are {selected_site_list}')\n", + "else:\n", + " print('Unable to find a sites matching the requirements')\n" + ] + }, + { + "cell_type": "markdown", + "id": "e7ebd074-4107-483e-b369-79ccd1f99184", + "metadata": {}, + "source": [ + "## Create slice iteratively (skip if exists)\n", + "\n", + "We may or may not get all the nodes we want immediately - we use iteration with slice modify to get to the max/desired number of nodes across the selected sites." + ] + }, + { + "cell_type": "markdown", + "id": "2e9bb82e-6db0-472a-aa3d-66cba06c3d08", + "metadata": {}, + "source": [ + "### Create Starter Slice" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "332540f0-89f5-4038-a28c-d02a454215c4", + "metadata": {}, + "outputs": [], + "source": [ + "# we start by establishing a slice with one sender node at some site, we keep track which sites we failed \n", + "# and don't try those again\n", + "\n", + "keep_trying = True\n", + "succeeded = False\n", + "\n", + "site_list_iter = iter(selected_site_list)\n", + "failed_sites = {}\n", + "site_name = None\n", + "\n", + "while keep_trying:\n", + "\n", + " try:\n", + " site_name = next(site_list_iter)\n", + " print(f'Trying site {site_name} from {selected_site_list}')\n", + " \n", + " # define a starter slice\n", + " slice = starter_slice(site_name)\n", + "\n", + " print(f'Submitting starter slice \"{slice_name}\" with sender on site {site_name}')\n", + " slice_id = slice.submit()\n", + "\n", + " # check the state of this slice\n", + " slices = fablib.get_slices(excludes=[], slice_id=slice_id)\n", + " if slices[0].get_state() == 'Dead':\n", + " print(f'Failed on site {site_name}, proceeding')\n", + " else:\n", + " print(f'Succeeded on site {site_name} with state {slices[0].get_state()}')\n", + " keep_trying = False\n", + " succeeded = True\n", + " except StopIteration: \n", + " print('No more sites to look at, exiting')\n", + " keep_trying = False\n", + " except Exception as e:\n", + " print(f'Unexpected exception {e}, exiting')\n", + " keep_trying = False\n", + "\n", + "if succeeded:\n", + " print(f'Succeeded in creating a slice on {site_name}, will avoid sites {failed_sites}')\n", + " selected_site_list = list(filter(lambda x: x not in failed_sites, selected_site_list))\n", + " print(f'Proceeding with sites {selected_site_list}')" + ] + }, + { + "cell_type": "markdown", + "id": "c50070d1-b89f-427e-9a5d-7af7605c8610", + "metadata": {}, + "source": [ + "### Modify the Slice to add Workers" + ] + }, + { + "cell_type": "markdown", + "id": "b9270746-f34e-44d6-9c0a-ecc05542a9d0", + "metadata": {}, + "source": [ + "Now that the base with the sender slice is created we will iteratively add workers on sites one at a time using first-fit policy until we get to the desired number of workers or run out of sites." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "341fd051-3dee-4c0c-8df8-f4222f1068fd", + "metadata": {}, + "outputs": [], + "source": [ + "remaining_workers = number_of_workers\n", + "node_idx = 1\n", + "node_increment = 3\n", + "nodes_in_slice = 0 # we don't count sender in this case\n", + "\n", + "while remaining_workers > 0 and len(selected_site_list) > 0:\n", + " slice = fablib.get_slice(name=slice_name)\n", + " \n", + " try:\n", + " site_name = selected_site_list[0]\n", + " print(f'There are {remaining_workers} remaining workers to create. Trying site {site_name} from {selected_site_list}')\n", + " expected_to_add = node_increment if remaining_workers >= node_increment else remaining_workers\n", + " add_node_to_slice(site_name, node_idx, expected_to_add, slice)\n", + " \n", + " print(f'Submitting slice modification to \"{slice_name}\" to add {expected_to_add} nodes for site {site_name}')\n", + " slice_id = slice.modify()\n", + " \n", + " # check the state of this slice\n", + " slice = fablib.get_slice(name=slice_name)\n", + "\n", + " if check_modify(slice, selected_site_list, nodes_in_slice, expected_to_add):\n", + " print(f'Succeeded adding {expected_to_add} nodes on site {site_name}.')\n", + " # successfully provisioned\n", + " node_idx += expected_to_add\n", + " remaining_workers -= expected_to_add\n", + " nodes_in_slice += expected_to_add\n", + " else:\n", + " print(f'Failed to provision on site {site_name}.')\n", + " # this site is full, moving on\n", + " selected_site_list.remove(site_name) \n", + " except Exception as e:\n", + " remaining_workers = -1\n", + " print(f'Unexpected exception {e}, exiting')\n", + " break\n", + "\n", + "if remaining_workers == 0:\n", + " print('Succeeded in creating all workers')\n", + "else:\n", + " print(f'Unable to create {remaining_workers}')\n" + ] + }, + { + "cell_type": "markdown", + "id": "3fa05d6d-ac6d-47f0-b351-4401b14c05c3", + "metadata": {}, + "source": [ + "## Get Slice Details (always execute)\n", + "\n", + "The following code sets up data structures so all the follow up cells work properly. Execute it regardless of whether you just created the slice or coming back to an existing slice." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6698360b-1118-4fa2-ace0-09bdf3f99a1f", + "metadata": {}, + "outputs": [], + "source": [ + "def find_net(net_list, name):\n", + " for net in net_list:\n", + " if net.get_name() == name:\n", + " return net\n", + " return None\n", + "\n", + "# get slice details \n", + "slice = fablib.get_slice(name=slice_name)\n", + "\n", + "a = slice.show()\n", + "nets = slice.list_networks()\n", + "nodes = slice.list_nodes()\n", + "\n", + "# arrange nodes and network services by site for future convenience\n", + "net_objects = slice.get_networks()\n", + "node_objects = slice.get_nodes()\n", + "available_ip_cnt = 10\n", + "\n", + "slivers_by_site = dict()\n", + "\n", + "print('Arranging nodes and networks by site and getting available IP addresses')\n", + "for node in node_objects:\n", + " node_site = node.get_site()\n", + " if not slivers_by_site.get(node_site):\n", + " slivers_by_site[node_site] = dict()\n", + " slivers_by_site[node_site]['nodes'] = set()\n", + " slivers_by_site[node_site]['net'] = find_net(net_objects, make_net_name(node_site))\n", + " slivers_by_site[node_site]['nodes'].add(node)\n", + "\n", + "print('Listing public IP addresses per service')\n", + "for net in net_objects:\n", + " print(f'{net.get_name()} has {net.get_public_ips()}')\n" + ] + }, + { + "cell_type": "markdown", + "id": "0f34ea09-c6bb-4bed-b831-35e0f5cf65e8", + "metadata": {}, + "source": [ + "## Perform Hardening and Network Configuration Opening to Outside World" + ] + }, + { + "cell_type": "markdown", + "id": "0f49f98c-d88f-4622-b7da-aa4cf797b100", + "metadata": {}, + "source": [ + "### Set up routing" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d92b9b99-defb-4102-a412-fd388763d19d", + "metadata": {}, + "outputs": [], + "source": [ + "# allocate externally routable IP addresses in each site network services\n", + "# it is NORMAL to see 'IP addresses were updated due to conflicts'\n", + "for site_name, site_slivers in slivers_by_site.items():\n", + " print(f'Processing {site_name}')\n", + " site_net = site_slivers['net']\n", + " site_nodes = site_slivers['nodes']\n", + " site_slivers['ips'] = site_net.get_available_ips(count=len(site_nodes))\n", + " print(f'Requesting available IPs to be publicly routable: {site_slivers[\"ips\"]}')\n", + " site_net.make_ip_publicly_routable(ipv4=[str(x) for x in site_slivers['ips']])\n", + "\n", + "slice.submit()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "34cc4504-46c7-4807-92cd-28053fc63a11", + "metadata": {}, + "outputs": [], + "source": [ + "# get slice details \n", + "slice = fablib.get_slice(name=slice_name)\n", + "\n", + "# check the results\n", + "for site_name, site_slivers in slivers_by_site.items():\n", + " print(f'Processing {site_name}')\n", + " site_net = site_slivers['net']\n", + " site_nodes = site_slivers['nodes']\n", + " print(f'Public IPs are: {site_net.get_public_ips()}')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bafe7c3d-2d25-47b6-9403-c9b01b4492fb", + "metadata": {}, + "outputs": [], + "source": [ + "# configure node interfaces with these IP addresses\n", + "for site_name, site_slivers in slivers_by_site.items():\n", + " print(f'Processing {site_name}')\n", + " site_net = site_slivers['net']\n", + " site_nodes = site_slivers['nodes']\n", + " site_addrs = site_net.get_public_ips()\n", + " for node, addr in zip(site_nodes, site_addrs):\n", + " print(f' Adding address {addr} to node {node.get_name()} in subnet {site_net.get_subnet()}')\n", + " # make sure the interface is UP (in rare cases comes up in DOWN state)\n", + " node_iface = node.get_interface(network_name = site_net.get_name())\n", + " execute_single_node(node, [f'sudo ip link set {node_iface.get_os_interface()} up'])\n", + " node_iface.ip_addr_add(addr=addr, subnet=site_net.get_subnet())\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7c5a0e5c-f332-42be-9877-ee385eea2128", + "metadata": {}, + "outputs": [], + "source": [ + "# configure inter-site routing if you have multiple sites\n", + "for site_name_from, site_slivers_from in slivers_by_site.items():\n", + " for site_name_to, site_slivers_to in slivers_by_site.items():\n", + " if site_name_from == site_name_to:\n", + " continue\n", + " # make sure nodes in site_name_from have a route to site_name_to subnet\n", + " subnet = site_slivers_to['net'].get_subnet()\n", + " gateway = site_slivers_from['net'].get_gateway()\n", + " for node in site_slivers_from['nodes']:\n", + " print(f'Setting up route to {subnet} via {gateway} on node {node.get_name()}')\n", + " node.ip_route_add(subnet=subnet, gateway=gateway)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7808808c-6630-4368-8314-d10779781427", + "metadata": {}, + "outputs": [], + "source": [ + "# configure global routing to indicated subnets \n", + "for site_name, site_slivers in slivers_by_site.items():\n", + " gateway = site_slivers['net'].get_gateway()\n", + " for node in site_slivers['nodes']:\n", + " print(f'Setting up routes on {node.get_name()}')\n", + " for subnet in external_subnets:\n", + " print(f'Setting up route to {subnet} via {gateway} on node {node.get_name()}')\n", + " execute_single_node(node, [f'sudo ip route add {subnet} via {gateway}'])" + ] + }, + { + "cell_type": "markdown", + "id": "d188265a-7204-493c-acf8-4cdf34f10252", + "metadata": {}, + "source": [ + "### Setup Firewall (assuming firewalld is used regardless of distro)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fee04f45-5f8f-4d08-a3eb-177d79c920f2", + "metadata": {}, + "outputs": [], + "source": [ + "# walk the nodes, add lo and management interface to 'trusted' zone where everything is allowed\n", + "# add dataplane interface into 'public' zone where only 'open ports' from specific sources is allowed\n", + "\n", + "for site_name, site_slivers in slivers_by_site.items():\n", + " site_net = site_slivers['net']\n", + " for node in site_slivers['nodes']:\n", + " print(f'Setting up firewalld on node {node.get_name()}')\n", + " # note we are calling our own function - as of 1.7.0 fablib's node.get_management_os_interface()\n", + " # has a bug where it doesn't find management interface on IPv6 sites\n", + " mgmt_iface_name = get_management_os_interface(node)\n", + " if mgmt_iface_name is None:\n", + " print('Unable to determine management interface, skipping')\n", + " continue\n", + " data_iface = node.get_interface(network_name=site_net.get_name())\n", + " data_iface_name = data_iface.get_os_interface()\n", + " print(f' Adding {mgmt_iface_name} and lo to trusted zone and {data_iface_name} to public zone')\n", + " commands = [\n", + " f'sudo firewall-cmd --permanent --zone=public --add-interface={data_iface_name}',\n", + " f'sudo firewall-cmd --permanent --zone=trusted --add-interface=lo',\n", + " f'sudo firewall-cmd --permanent --zone=trusted --add-interface={mgmt_iface_name}',\n", + " f'for i in $(sudo firewall-cmd --zone=public --list-services); do sudo firewall-cmd --zone=public --permanent --remove-service=$i; done',\n", + " ]\n", + " for subnet, portlist in open_ports.items():\n", + " for port in portlist:\n", + " commands.append(f'sudo firewall-cmd --permanent --zone=public --add-rich-rule=\\'rule family=\\\"ipv4\\\" source address=\\\"{subnet}\\\" port protocol=\\\"tcp\\\" port=\\\"{port}\\\" accept\\'')\n", + " for subnet in external_subnets:\n", + " commands.append(f'sudo firewall-cmd --permanent --zone=public --add-rich-rule=\\'rule family=\\\"ipv4\\\" source address=\\\"{subnet}\\\" protocol value=\\\"udp\\\" accept\\'')\n", + " commands.append(f'sudo firewall-cmd --reload')\n", + " commands.append(f'sudo firewall-cmd --list-all --zone=public')\n", + " execute_single_node(node, commands)\n", + " \n", + " " + ] + }, + { + "cell_type": "markdown", + "id": "45c614a4-b6a2-48bb-9f6e-4ce9b9677ddd", + "metadata": {}, + "source": [ + "## Tune Buffers and MTUs\n", + "\n", + "In order to have good performance we need to\n", + "- Make the UDP send/receive socket buffer size limit larger (applications are assumed to know how to make their buffers larger up to this limit)\n", + "- Set MTU to 9k and test with DF=0 ping" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "72c91079-9ce3-4d73-b945-182ffe85dd5a", + "metadata": {}, + "outputs": [], + "source": [ + "# setup UDP socket buffer sizes to 512M\n", + "commands = [\n", + " f\"sudo sysctl net.core.rmem_max=536870912\",\n", + " f\"sudo sysctl net.core.wmem_max=536870912\",\n", + " f\"sysctl net.core.wmem_max net.core.rmem_max\"\n", + "]\n", + "# walk the nodes\n", + "for site_name, site_slivers in slivers_by_site.items():\n", + " for node in site_slivers['nodes']:\n", + " execute_single_node(node, commands)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0472d539-252c-4d50-90f8-bb7f9cdd1beb", + "metadata": {}, + "outputs": [], + "source": [ + "# set 9k MTU on dataplane interfaces\n", + "mtu=9000\n", + "\n", + "for site_name, site_slivers in slivers_by_site.items():\n", + " site_net = site_slivers['net']\n", + " for node in site_slivers['nodes']:\n", + " data_iface = node.get_interface(network_name=site_net.get_name())\n", + " data_iface_name = data_iface.get_os_interface()\n", + " execute_single_node(node, [f\"sudo ip link set dev {data_iface_name} mtu {mtu}\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8f5ef68c-6da4-4ce7-baee-a8efe7d61d3e", + "metadata": {}, + "outputs": [], + "source": [ + "# run a no-DF test from every node to the first public address of the first site on the list\n", + "first_ip = list(slivers_by_site.items())[0][1]['net'].get_public_ips()[0]\n", + "# you can replace first_ip with the IP of a load balancer, but be careful not to interfere\n", + "# with a running experiment as this uses ping flood \n", + "first_ip = \"192.188.29.1\"\n", + "\n", + "for site_name, site_slivers in slivers_by_site.items():\n", + " for node in site_slivers['nodes']:\n", + " print(f'Node {node.get_name()} pinging {first_ip}')\n", + " execute_single_node(node, [f\"sudo ping -q -f -s 8972 -c 100 -M do {first_ip}\"])" + ] + }, + { + "cell_type": "markdown", + "id": "4551c7f3-c87c-4216-9f66-3e10eba38f90", + "metadata": {}, + "source": [ + "## Customize Nodes\n", + "\n", + "Customize node setup by adding E2SAR installation" + ] + }, + { + "cell_type": "markdown", + "id": "527b3798-4fb4-447a-a07f-3885936bc55f", + "metadata": {}, + "source": [ + "### Add E2SAR software" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "84ad8f6b-d909-4076-803e-8adfb9ee9cc5", + "metadata": {}, + "outputs": [], + "source": [ + "# install github ssh key and set up build environment variables for interactive logins\n", + "commands = [\n", + " f\"chmod go-rwx {vm_key_location}\",\n", + " # Meson won't detect boost by merely setting cmake_prefix_path, instead set BOOST_ROOT env variable \n", + " # for gRPC it is enough to set -Dpkg_config_path option to meson\n", + " f\"echo 'export BOOST_ROOT=/usr/local/ LD_LIBRARY_PATH=/usr/local/lib' >> ~/.profile\",\n", + " f\"echo 'export BOOST_ROOT=/usr/local/ LD_LIBRARY_PATH=/usr/local/lib' >> ~/.bashrc\",\n", + "]\n", + "\n", + "for node in slice.get_nodes(): \n", + " # upload the GitHub SSH key onto the VM\n", + " result = node.upload_file(github_key, vm_key_location)\n", + " execute_commands(node, commands)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7200a034-e505-41d5-8f5e-6141b6b6cf38", + "metadata": {}, + "outputs": [], + "source": [ + "# download boost and grpc dependencies from releases\n", + "commands = [\n", + " f\"wget -q -O boost_grpc.deb {e2sar_dep_url}\",\n", + " #f\"sudo apt -yq install ./boost_grpc.deb\",\n", + " f\"sudo dpkg -i ./boost_grpc.deb\"\n", + "]\n", + " \n", + "execute_commands(slice.get_nodes(), commands)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "243ecc28-3208-43ff-9bd8-49bf74be79af", + "metadata": {}, + "outputs": [], + "source": [ + "# checkout E2SAR (including the right branch) using that key, install grpc and boost binary that is stored in the repo\n", + "commands = [\n", + " f\"GIT_SSH_COMMAND='ssh -i {vm_key_location} -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' git clone --recurse-submodules --depth 1 -b {e2sar_branch} git@github.com:JeffersonLab/E2SAR.git\",\n", + "]\n", + " \n", + "execute_commands(slice.get_nodes(), commands)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "89f7a098-c295-4a77-8096-f878ffb92015", + "metadata": {}, + "outputs": [], + "source": [ + "# compile and test E2SAR code\n", + "# note that most live tests only need the simplest URI - ejfats://token@ip:port/\n", + "# however the e2sar_reas_live_test requires data and sync addresses, and data address must\n", + "# be real (so we use loopback). Hence the long form of the URI for live tests \n", + "# (other tests simply ignore the parts of the URI they don't need.)\n", + "\n", + "commands = [\n", + " f\"cd E2SAR; rm -rf build; PATH=$HOME/.local/bin:/usr/local/bin:$PATH BOOST_ROOT=/usr/local/ LD_LIBRARY_PATH=/usr/local/lib/ meson setup -Dpkg_config_path=/usr/local/lib/pkgconfig/:/usr/lib/lib64/pkgconfig/ --prefix {home_location}/e2sar-install build && sed -i 's/-std=c++11//g' build/build.ninja\",\n", + " f\"PATH=$HOME/.local/bin:/usr/local/bin:$PATH LD_LIBRARY_PATH=/usr/local/lib/ meson compile -j 8 -C build\",\n", + " f\"PATH=$HOME/.local/bin:/usr/local/bin:$PATH LD_LIBRARY_PATH=/usr/local/lib/ meson test --suite unit --timeout 0 -C build\",\n", + "]\n", + "\n", + "# NOTE THIS EXECUTES ON THREADS IN PARALLEL, CHECK THE LOG FILES (NodeName_thread_log.log)\n", + "execute_commands_on_threads(slice.get_nodes(), commands)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "025be1fa-c926-4cf1-aec1-57c37e526f12", + "metadata": {}, + "outputs": [], + "source": [ + "#\n", + "# if you need to update already cloned repo\n", + "#\n", + "# update the code, compile and test\n", + "# note that most live tests only need the simplest URI - ejfats://token@ip:port/\n", + "# however the e2sar_reas_live_test requires data and sync addresses, and data address must\n", + "# be real (so we use loopback). Hence the long form of the URI for live tests \n", + "# (other tests simply ignore the parts of the URI they don't need.)\n", + "\n", + "commands = [\n", + " f\"cd E2SAR; GIT_SSH_COMMAND='ssh -i {vm_key_location} -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' git pull origin {e2sar_branch}\",\n", + " f\"cd E2SAR; BOOST_ROOT=/usr/local/ PATH=$HOME/.local/bin:/usr/local/bin:$PATH LD_LIBRARY_PATH=/usr/local/lib/ meson setup -Dpkg_config_path=/usr/local/lib/pkgconfig/:/usr/lib/lib64/pkgconfig/ --prefix {home_location}/e2sar-install build --wipe && sed -i 's/-std=c++11//g' build/build.ninja\",\n", + " f\"cd E2SAR/build; PATH=$HOME/.local/bin:/usr/local/bin:$PATH LD_LIBRARY_PATH=/usr/local/lib/ meson compile -j 8\",\n", + "# f\"cd E2SAR/build; PATH=$HOME/.local/bin:/usr/local/bin:$PATH LD_LIBRARY_PATH=/usr/local/lib/ meson test {e2sar_test_suite} --suite unit --timeout 0\",\n", + "]\n", + " \n", + "execute_commands(slice.get_nodes(), commands)" + ] + }, + { + "cell_type": "markdown", + "id": "3e19f887-0a99-488e-8b9e-b72d5247f425", + "metadata": {}, + "source": [ + "## Run Tests\n", + "\n", + "### Run simple single-threaded performance test\n", + "\n", + "Start Segmenter on Sender node and one Reassembler on Worker1 and test throughput without a real load balancer. Reassembler is told that LB header will be present and it ignores it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3409e1dd-d1fc-4a2e-ae48-46bd76881bdf", + "metadata": {}, + "outputs": [], + "source": [ + "sender = list(filter(lambda n: n.get_name()[0:6] == \"Sender\", slice.get_nodes()))[0]\n", + "recver = list(filter(lambda n: n.get_name()[0:7] == \"Worker1\", slice.get_nodes()))[0]\n", + "\n", + "sender_addr = sender.get_interface(network_name=make_net_name(sender.get_site())).get_ip_addr()\n", + "recver_addr = recver.get_interface(network_name=make_net_name(recver.get_site())).get_ip_addr()\n", + "print(f\"Sender sending from {sender_addr}, receiver receiving on {recver_addr}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a9a3dcd-c367-4240-9cea-f2bbb45b2ce0", + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "\n", + "# for e2sar_perf only the data= part of the query is meaningful. sync= must be present but is ignored\n", + "# same for gRPC token, address and port (and lb id)\n", + "e2sarPerfURI = f\"ejfat://useless@10.10.10.10:1234/lb/1?data={recver_addr}&sync=192.168.77.7:1234\"\n", + "recverDuration = 20\n", + "mtu = 9000\n", + "rate = 15 # Gbps\n", + "length = 1000000 # event length in bytes\n", + "numEvents = 10000 # number of events to send\n", + "bufSize = 300 * 1024 * 1024 # 100MB send and receive buffers\n", + "\n", + "recv_command = f\"cd E2SAR; PATH=$HOME/.local/bin:/usr/local/bin:$PATH LD_LIBRARY_PATH=/usr/local/lib/ ./build/bin/e2sar_perf -r -u '{e2sarPerfURI}' -d {recverDuration} -b {bufSize} --ip {recver_addr} --port 19522\"\n", + "send_command = f\"cd E2SAR; PATH=$HOME/.local/bin:/usr/local/bin:$PATH LD_LIBRARY_PATH=/usr/local/lib/ ./build/bin/e2sar_perf -s -u '{e2sarPerfURI}' --mtu {mtu} --rate {rate} --length {length} -n {numEvents} -b {bufSize} --ip {sender_addr}\"\n", + "\n", + "# start the receiver for 10 seconds and log its output\n", + "print(f'Executing command {recv_command} on receiver')\n", + "recver.execute_thread(recv_command, output_file=f\"{recver.get_name()}.perf.log\")\n", + "\n", + "# sleep 2 seconds to let receiver get going\n", + "time.sleep(2)\n", + "\n", + "# start the sender in the foreground\n", + "print(f'Executing command {send_command} on sender')\n", + "stdout_send, stderr_send = sender.execute(send_command, output_file=f\"{sender.get_name()}.perf.log\")\n", + "\n", + "print(f\"Inspect {recver.get_name()}.perf.log file in your Jupyter container to see the results\")" + ] + }, + { + "cell_type": "markdown", + "id": "46c04a61-eebd-4189-9848-1c787995919e", + "metadata": {}, + "source": [ + "### Run test on a live Load Balancer" + ] + }, + { + "cell_type": "markdown", + "id": "c60a87e0-0ecc-4e71-b728-fb358e349f04", + "metadata": {}, + "source": [ + "1. Reserve a new load balancer instance for a maximum 2 hours\n", + "2. Run the test (possibly multiple times)\n", + "3. Free the load balancer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "531e7e6b-dfd3-44b8-9d92-1df7b9887cae", + "metadata": {}, + "outputs": [], + "source": [ + "# Set the admin URI\n", + "ejfat_admin_uri = '' # cut and paste here\n", + "\n", + "lb_path = './E2SAR/build/bin'\n", + "ld_library_path = \"LD_LIBRARY_PATH=/usr/local/lib\"\n", + "bin_path = \"PATH=./E2SAR/build/bin:$PATH\"\n", + "# note we are forcing IPv4 here with -4 option - from FABRIC this is necessary\n", + "lbadm = f\"{ld_library_path} {bin_path} lbadm -4\"\n", + "e2sar_perf = f\"{ld_library_path} {bin_path} e2sar_perf\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "95678748-722e-4e95-8b2c-c2363dc523d1", + "metadata": {}, + "outputs": [], + "source": [ + "# run an overview command to see what is reserved\n", + "# we use sender node but any node can be used for admin commands\n", + "\n", + "command = f\"{lbadm} --overview -u {ejfat_admin_uri}\"\n", + "\n", + "execute_commands(sender, [command])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e96478f5-ca90-4ad7-9544-56330f59e8a4", + "metadata": {}, + "outputs": [], + "source": [ + "# Reserve the load balancer\n", + "lbname = 'e2sar-testlb'\n", + "duration = '02:00:00' # 2 hours\n", + "\n", + "command = f\"{lbadm} --reserve -l {lbname} -a '{sender_addr}' -d {duration} -u '{ejfat_admin_uri}'\"\n", + "execute_commands(sender, [command])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "acd72bab-8255-486d-a9cd-f753a8a707dd", + "metadata": {}, + "outputs": [], + "source": [ + "# copy the 'Updated URI after reserve with instance token' from the above result here:\n", + "instance_uri = ''" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b7ed79af-9715-4fe5-b416-b769bb69503d", + "metadata": {}, + "outputs": [], + "source": [ + "# get the status of the reserved LB (as a check)\n", + "command = f\"{lbadm} --status -u '{instance_uri}'\"\n", + "execute_commands(sender, [command])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a206ef22-bb58-48ed-a1e8-9673e0a7d3b5", + "metadata": {}, + "outputs": [], + "source": [ + "# run sender and receiver through the reserved LB\n", + "recverDuration = 40\n", + "mtu = 9000\n", + "rate = 15 # Gbps\n", + "length = 1000000 # event length in bytes\n", + "numEvents = 20000 # number of events to send\n", + "bufSize = 300 * 1024 * 1024 # 100MB send and receive buffers\n", + "recvThreads = 4\n", + "\n", + "# Given that in FABRIC ejfat-lb.es.net resolves to IP6 first and gRPC C++ library doesn't\n", + "# offer granular control over which resolved address is used, we use -4 option to tell the\n", + "# code to use the IPv4 address, but this also disables cert validation.\n", + "recv_command = f\"{e2sar_perf} -r -u '{instance_uri}' -d {recverDuration} -b {bufSize} --ip {recver_addr} --port 10000 --withcp -4 -t 4\"\n", + "send_command = f\"{e2sar_perf} -s -u '{instance_uri}' --mtu {mtu} --rate {rate} --length {length} -n {numEvents} -b {bufSize} --ip {sender_addr} --withcp -4\"\n", + "\n", + "# start the receiver for n seconds and log its output\n", + "print(f'Executing command {recv_command} on receiver')\n", + "recver.execute_thread(recv_command, output_file=f\"{recver.get_name()}.perf.log\")\n", + "\n", + "# sleep 5 seconds to let receiver get going\n", + "time.sleep(5)\n", + "\n", + "# start the sender in the foreground\n", + "print(f'Executing command {send_command} on sender')\n", + "stdout_send, stderr_send = sender.execute(send_command, output_file=f\"{sender.get_name()}.perf.log\")\n", + "\n", + "print(f\"Inspect {recver.get_name()}.perf.log file to see the results. It should indicate how many events were received and how many lost.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41716952-8d5c-4eae-a843-ef5ac0c3ff8a", + "metadata": {}, + "outputs": [], + "source": [ + "# free the load balancer\n", + "command = f\"{lbadm} --free -u '{instance_uri}'\"\n", + "\n", + "execute_commands(sender, [command])" + ] + }, + { + "cell_type": "markdown", + "id": "56c8d476-689a-4c4c-b5dc-ed0625410abc", + "metadata": {}, + "source": [ + "## Manage the slice\n", + "\n", + "### Extend by two weeks" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fbf9fe4f-b067-4d2c-94a5-9839d5a53b73", + "metadata": {}, + "outputs": [], + "source": [ + "# Set end host to now plus 14 days\n", + "end_date = (datetime.now(timezone.utc) + timedelta(days=14)).strftime(\"%Y-%m-%d %H:%M:%S %z\")\n", + "\n", + "try:\n", + " slice = fablib.get_slice(name=slice_name)\n", + "\n", + " slice.renew(end_date)\n", + "except Exception as e:\n", + " print(f\"Exception: {e}\")" + ] + }, + { + "cell_type": "markdown", + "id": "38eb476a-2b17-4064-9a0e-a7a98d865804", + "metadata": {}, + "source": [ + "### Delete" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "de3546bf-a013-4eaa-bab1-fb52f7d36c55", + "metadata": {}, + "outputs": [], + "source": [ + "slice = fablib.get_slice(slice_name)\n", + "slice.delete()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1737f112-95d3-4a78-9b15-f16ae98e7f12", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 12a915d15d1104ac45f2ec00d09541640f5a687f Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Tue, 24 Sep 2024 17:19:53 -0400 Subject: [PATCH 28/73] Modified docs/ --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 1e6195a6..8979dc67 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 1e6195a6ead60a573bb19c6bb96924a44e4773c4 +Subproject commit 8979dc67de51f8d0905df15860554e174e40a2eb From 96858846c21d449ec01dd47ecb9230a776e9ca96 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Wed, 25 Sep 2024 13:10:47 -0400 Subject: [PATCH 29/73] updated wiki --- wiki | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wiki b/wiki index 6d09ff0d..52a67848 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 6d09ff0dceb94c58e0af6e71eb45758e888ff04d +Subproject commit 52a6784808f7603ccbdb8ed6ab091a0ac805a4ff From cd584d508f5317ad4663f5004dba016f6db70f16 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Wed, 25 Sep 2024 13:32:39 -0400 Subject: [PATCH 30/73] Updating sync header version --- scripts/scapy/snifgen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/scapy/snifgen.py b/scripts/scapy/snifgen.py index f0053a85..e929fa4d 100755 --- a/scripts/scapy/snifgen.py +++ b/scripts/scapy/snifgen.py @@ -21,7 +21,7 @@ class SyncPacket(Packet): name = "SyncPacket" fields_desc = [ StrFixedLenField('preamble', 'LC', 2), - XByteField('version', 1), + XByteField('version', 2), XByteField('reserved', 0), IntField('eventSrcId', 0), LongField('eventNumber', 0), From 896799ef41f4cdda046830481dee347af12e7c32 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Thu, 26 Sep 2024 10:06:42 -0400 Subject: [PATCH 31/73] Added a way to show gRPC connect string --- bin/lbadm.cpp | 30 ++++++++++++++++++++---------- bin/lbmonitor.cpp | 6 ++++-- include/e2sarCP.hpp | 18 ++++++++++++++---- src/e2sarCP.cpp | 4 ++-- 4 files changed, 40 insertions(+), 18 deletions(-) diff --git a/bin/lbadm.cpp b/bin/lbadm.cpp index 6ddf8e9c..b4382eaf 100644 --- a/bin/lbadm.cpp +++ b/bin/lbadm.cpp @@ -51,7 +51,8 @@ result reserveLB(LBManager &lbman, if(!suppress) { std::cout << "Reserving a new load balancer " << std::endl; - std::cout << " Contacting: " << static_cast(lbman.get_URI()) << " on IP " << lbman.get_URI().get_cpAddr().value().first << std::endl; + std::cout << " Contacting: " << static_cast(lbman.get_URI()) << " using address: " << + lbman.get_AddrString() << std::endl; std::cout << " LB Name: " << lbname << std::endl; std::cout << " Allowed senders: "; std::for_each(senders.begin(), senders.end(), [](const std::string& s) { std::cout << s << ' '; }); @@ -85,7 +86,8 @@ result reserveLB(LBManager &lbman, result freeLB(LBManager &lbman, const std::string &lbid = "") { std::cout << "Freeing a load balancer " << std::endl; - std::cout << " Contacting: " << lbman.get_URI().to_string(EjfatURI::TokenType::admin) << " on IP " << lbman.get_URI().get_cpAddr().value().first << std::endl; + std::cout << " Contacting: " << lbman.get_URI().to_string(EjfatURI::TokenType::admin) << " using address: " << + lbman.get_AddrString() << std::endl; std::cout << " LB ID: " << (lbid.empty() ? lbman.get_URI().get_lbId() : lbid) << std::endl; result res{0}; @@ -116,7 +118,8 @@ result registerWorker(LBManager &lbman, const std::string &node_name, if(!suppress) { std::cout << "Registering a worker " << std::endl; - std::cout << " Contacting: " << lbman.get_URI().to_string(EjfatURI::TokenType::instance) << " on IP " << lbman.get_URI().get_cpAddr().value().first << std::endl; + std::cout << " Contacting: " << lbman.get_URI().to_string(EjfatURI::TokenType::instance) << " using address: " << + lbman.get_AddrString() << std::endl; std::cout << " Worker details: " << node_name << " at "s << node_ip << ":"s << node_port << std::endl; std::cout << " CP parameters: " << "w="s << weight << ", source_count="s << src_cnt << std::endl; @@ -148,7 +151,8 @@ result registerWorker(LBManager &lbman, const std::string &node_name, result deregisterWorker(LBManager &lbman) { std::cout << "De-Registering a worker " << std::endl; - std::cout << " Contacting: " << lbman.get_URI().to_string(EjfatURI::TokenType::session) << " on IP " << lbman.get_URI().get_cpAddr().value().first << std::endl; + std::cout << " Contacting: " << lbman.get_URI().to_string(EjfatURI::TokenType::session) << " using address: " << + lbman.get_AddrString() << std::endl; auto res = lbman.deregisterWorker(); @@ -167,7 +171,8 @@ result deregisterWorker(LBManager &lbman) result getLBStatus(LBManager &lbman, const std::string &lbid) { std::cout << "Getting LB Status " << std::endl; - std::cout << " Contacting: " << lbman.get_URI().to_string(EjfatURI::TokenType::session) << " on IP " << lbman.get_URI().get_cpAddr().value().first << std::endl; + std::cout << " Contacting: " << lbman.get_URI().to_string(EjfatURI::TokenType::session) << " using address: " << + lbman.get_AddrString() << std::endl; std::cout << " LB ID: " << (lbid.empty() ? lbman.get_URI().get_lbId() : lbid) << std::endl; auto res = lbman.getLBStatus(lbid); @@ -202,7 +207,8 @@ result getLBStatus(LBManager &lbman, const std::string &lbid) result overview(LBManager &lbman) { std::cout << "Getting Overview " << std::endl; - std::cout << " Contacting: " << lbman.get_URI().to_string(EjfatURI::TokenType::session) << " on IP " << lbman.get_URI().get_cpAddr().value().first << std::endl; + std::cout << " Contacting: " << lbman.get_URI().to_string(EjfatURI::TokenType::session) << " using address: " << + lbman.get_AddrString() << std::endl; auto res = lbman.overview(); @@ -243,7 +249,8 @@ result overview(LBManager &lbman) result sendState(LBManager &lbman, float fill_percent, float ctrl_signal, bool is_ready) { std::cout << "Sending Worker State " << std::endl; - std::cout << " Contacting: " << lbman.get_URI().to_string(EjfatURI::TokenType::session) << " on IP " << lbman.get_URI().get_cpAddr().value().first << std::endl; + std::cout << " Contacting: " << lbman.get_URI().to_string(EjfatURI::TokenType::session) << " using address: " << + lbman.get_AddrString() << std::endl; std::cout << " LB Name: " << (lbman.get_URI().get_lbName().empty() ? "not set"s : lbman.get_URI().get_lbId()) << std::endl; auto res = lbman.sendState(fill_percent, ctrl_signal, is_ready); @@ -264,7 +271,8 @@ result sendState(LBManager &lbman, float fill_percent, float ctrl_signal, b result removeSenders(LBManager &lbman, const std::vector& senders) { std::cout << "Removing senders to CP " << std::endl; - std::cout << " Contacting: " << lbman.get_URI().to_string(EjfatURI::TokenType::session) << " on IP " << lbman.get_URI().get_cpAddr().value().first << std::endl; + std::cout << " Contacting: " << lbman.get_URI().to_string(EjfatURI::TokenType::session) << " using address: " << + lbman.get_AddrString() << std::endl; std::cout << " LB Name: " << (lbman.get_URI().get_lbName().empty() ? "not set"s : lbman.get_URI().get_lbId()) << std::endl; std::cout << " Sender list: "; std::for_each(senders.begin(), senders.end(), [](const std::string& s) { std::cout << s << ' '; }); @@ -288,7 +296,8 @@ result removeSenders(LBManager &lbman, const std::vector& send result addSenders(LBManager &lbman, const std::vector& senders) { std::cout << "Adding senders to CP " << std::endl; - std::cout << " Contacting: " << lbman.get_URI().to_string(EjfatURI::TokenType::session) << " on IP " << lbman.get_URI().get_cpAddr().value().first << std::endl; + std::cout << " Contacting: " << lbman.get_URI().to_string(EjfatURI::TokenType::session) << " using address: " << + lbman.get_AddrString() << std::endl; std::cout << " LB Name: " << (lbman.get_URI().get_lbName().empty() ? "not set"s : lbman.get_URI().get_lbId()) << std::endl; std::cout << " Sender list: "; std::for_each(senders.begin(), senders.end(), [](const std::string& s) { std::cout << s << ' '; }); @@ -313,7 +322,8 @@ result version(LBManager &lbman) { std::cout << "Getting load balancer version " << std::endl; - std::cout << " Contacting: " << static_cast(lbman.get_URI()) << " on IP " << lbman.get_URI().get_cpAddr().value().first << std::endl; + std::cout << " Contacting: " << static_cast(lbman.get_URI()) << " using address: " << + lbman.get_AddrString() << std::endl; auto res = lbman.version(); diff --git a/bin/lbmonitor.cpp b/bin/lbmonitor.cpp index eff148d3..5052976e 100644 --- a/bin/lbmonitor.cpp +++ b/bin/lbmonitor.cpp @@ -13,7 +13,8 @@ using namespace e2sar; result getLBStatus(LBManager &lbman, const std::string &lbid) { std::cout << "Getting LB Status " << std::endl; - std::cout << " Contacting: " << lbman.get_URI().to_string(EjfatURI::TokenType::session) << " on IP " << lbman.get_URI().get_cpAddr().value().first << std::endl; + std::cout << " Contacting: " << lbman.get_URI().to_string(EjfatURI::TokenType::session) << " using address: " << + lbman.get_AddrString() << std::endl; std::cout << " LB Name: " << (lbman.get_URI().get_lbName().empty() ? "not set"s : lbman.get_URI().get_lbId()) << std::endl; std::cout << " LB ID: " << lbid << std::endl; @@ -49,7 +50,8 @@ result getLBStatus(LBManager &lbman, const std::string &lbid) result getLBOverview(LBManager &lbman) { std::cout << "Getting Overview " << std::endl; - std::cout << " Contacting: " << lbman.get_URI().to_string(EjfatURI::TokenType::session) << " on IP " << lbman.get_URI().get_cpAddr().value().first << std::endl; + std::cout << " Contacting: " << lbman.get_URI().to_string(EjfatURI::TokenType::session) << " using address: " << + lbman.get_AddrString() << std::endl; auto res = lbman.overview(); diff --git a/include/e2sarCP.hpp b/include/e2sarCP.hpp index 6fd79608..ee275e29 100644 --- a/include/e2sarCP.hpp +++ b/include/e2sarCP.hpp @@ -120,6 +120,7 @@ namespace e2sar { private: EjfatURI _cpuri; + std::string addr_string; std::unique_ptr _stub; std::shared_ptr _channel; @@ -141,7 +142,6 @@ namespace e2sar { auto cp_host_r = cpuri.get_cpHost(); - std::string addr_string; // using host address automatically disables cert validation if (useHostAddress) @@ -510,7 +510,7 @@ namespace e2sar */ static inline result makeSslOptions(const std::string &pem_root_certs, const std::string &pem_private_key, - const std::string &pem_cert_chain) + const std::string &pem_cert_chain) noexcept { return grpc::SslCredentialsOptions{std::move(pem_root_certs), std::move(pem_private_key), @@ -529,14 +529,24 @@ namespace e2sar static result makeSslOptionsFromFiles( std::string_view pem_root_certs, std::string_view pem_private_key, - std::string_view pem_cert_chain); + std::string_view pem_cert_chain) noexcept; /** * Generate gRPC-compliant custom SSL Options object with just the server root cert * @param pem_root_certs - The file name containing the PEM encoding of the server root certificate. */ static result makeSslOptionsFromFiles( - std::string_view pem_root_certs); + std::string_view pem_root_certs) noexcept; + + /** + * Return the address string used by gRPC to connect to control plane. Can be + * in the format of hostname:port or ipv4:///W.X.Y.Z:port or ipv6:///[XXXX::XX:XXXX]:port + * + * @return the string containing the address + */ + inline std::string get_AddrString() { + return addr_string; + } }; /** diff --git a/src/e2sarCP.cpp b/src/e2sarCP.cpp index 8f430f0e..4ae576cb 100644 --- a/src/e2sarCP.cpp +++ b/src/e2sarCP.cpp @@ -709,7 +709,7 @@ namespace e2sar result LBManager::makeSslOptionsFromFiles(std::string_view pem_root_certs, std::string_view pem_private_key, - std::string_view pem_cert_chain) + std::string_view pem_cert_chain) noexcept { auto root = read_file(pem_root_certs); auto priv = read_file(pem_private_key); @@ -726,7 +726,7 @@ namespace e2sar // just the server root cert (useful for self-signed) result LBManager::makeSslOptionsFromFiles( - std::string_view pem_root_certs) + std::string_view pem_root_certs) noexcept { auto root = read_file(pem_root_certs); From 08ba44f782127fe77c5571abcc58fc7b5484d568 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Thu, 26 Sep 2024 10:08:38 -0400 Subject: [PATCH 32/73] Added a way to show gRPC connect string --- src/pybind/py_e2sarCP.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pybind/py_e2sarCP.cpp b/src/pybind/py_e2sarCP.cpp index b7196544..4d229387 100644 --- a/src/pybind/py_e2sarCP.cpp +++ b/src/pybind/py_e2sarCP.cpp @@ -281,7 +281,7 @@ void init_e2sarCP(py::module_ &m) { /** * Return type containing result> */ - lb_manager.def( + lb_manager.def( "get_lb_overview", [](LBManager& self){ auto result = self.overview(); @@ -294,5 +294,8 @@ void init_e2sarCP(py::module_ &m) { // Return an EjfatURI object. lb_manager.def("get_uri", &LBManager::get_URI, py::return_value_policy::reference); + // return connect string + lb_manager.def("get_addrstring", &LBManager::get_AddrString, py::return_value_policy::reference); + /// NOTE: donot need to bind LBManager::makeSslOptionsFromFiles } From db91834b64ec05dd6fe317414bfae1b0a3db6923 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Thu, 26 Sep 2024 10:11:15 -0400 Subject: [PATCH 33/73] Updated docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 8979dc67..296501b2 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 8979dc67de51f8d0905df15860554e174e40a2eb +Subproject commit 296501b217b8ca7e3e7ed77d80b6a8da8738c5dc From ed4c294b9a9cca731a61cfe4c1946ba6065db122 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Thu, 26 Sep 2024 11:15:52 -0400 Subject: [PATCH 34/73] Setting conflicting options --- bin/lbadm.cpp | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/bin/lbadm.cpp b/bin/lbadm.cpp index b4382eaf..da596d9d 100644 --- a/bin/lbadm.cpp +++ b/bin/lbadm.cpp @@ -354,20 +354,20 @@ int main(int argc, char **argv) opts("lbname,l", po::value(), "specify name of the load balancer"); opts("lbid,i", po::value(), "override/provide id of the loadbalancer"); opts("address,a", po::value>()->multitoken(), "node IPv4/IPv6 address, can be used multiple times for 'reserve' call"); - opts("duration,d", po::value(), "specify duration as '[hh[:mm[:ss]]]'"); + opts("duration,d", po::value()->default_value("02:00:00"), "specify duration as '[hh[:mm[:ss]]]'"); opts("uri,u", po::value(), "specify EJFAT_URI on the command-line instead of the environment variable"); opts("name,n", po::value(), "specify node name for registration"); opts("port,p", po::value(), "node starting listening port number"); - opts("weight,w", po::value(), "node weight"); - opts("count,c", po::value(), "node source count"); + opts("weight,w", po::value()->default_value(1.0), "node weight"); + opts("count,c", po::value()->default_value(1), "node source count"); opts("session,s", po::value(), "override/provide session id"); - opts("queue,q", po::value(), "queue fill"); - opts("ctrl,t", po::value(), "control signal value"); - opts("ready,r", po::value(), "worker ready state (1 or 0)"); + opts("queue,q", po::value()->default_value(0.0), "queue fill"); + opts("ctrl,t", po::value()->default_value(0.0), "control signal value"); + opts("ready,r", po::value()->default_value(true), "worker ready state (1 or 0)"); opts("root,o", po::value(), "root cert for SSL communications"); opts("novalidate,v", "don't validate server certificate (conflicts with 'root')"); - opts("minfactor", po::value(), "node min factor, multiplied with the number of slots that would be assigned evenly to determine min number of slots for example, 4 nodes with a minFactor of 0.5 = (512 slots / 4) * 0.5 = min 64 slots"); - opts("maxfactor", po::value(), "multiplied with the number of slots that would be assigned evenly to determine max number of slots for example, 4 nodes with a maxFactor of 2 = (512 slots / 4) * 2 = max 256 slots set to 0 to specify no maximum"); + opts("minfactor", po::value()->default_value(0.5), "node min factor, multiplied with the number of slots that would be assigned evenly to determine min number of slots for example, 4 nodes with a minFactor of 0.5 = (512 slots / 4) * 0.5 = min 64 slots"); + opts("maxfactor", po::value()->default_value(2.0), "multiplied with the number of slots that would be assigned evenly to determine max number of slots for example, 4 nodes with a maxFactor of 2 = (512 slots / 4) * 2 = max 256 slots set to 0 to specify no maximum"); opts("ipv6,6", "force using IPv6 control plane address if URI specifies hostname (disables cert validation)"); opts("ipv4,4", "force using IPv4 control plane address if URI specifies hostname (disables cert validation)"); opts("export,e", "suppresses other messages and prints out 'export EJFAT_URI=' returned by the LB"); @@ -383,6 +383,8 @@ int main(int argc, char **argv) opts("addsenders","add 'safe' sender IP addresses to CP (one or more -a required). Uses instance token."); opts("removesenders","remove 'safe' sender IP addresses from CP (one or more -a required). Uses instance token."); + std::vector commands{"reserve", "free", "version", "register", + "deregister", "status", "state", "overview", "addsenders", "removesenders"}; po::variables_map vm; @@ -409,6 +411,15 @@ int main(int argc, char **argv) conflicting_options(vm, "ipv4", "ipv6"); option_dependency(vm,"addsenders", "address"); option_dependency(vm,"removesenders", "address"); + + for (auto c1: commands) + { + for (auto c2: commands) + { + if (c1.compare(c2)) + conflicting_options(vm, c1, c2); + } + } } catch (const std::logic_error &le) { From 8b4dc5c59b515c1cff243e532cae5989b5526190 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Thu, 26 Sep 2024 11:59:37 -0400 Subject: [PATCH 35/73] Adding sane defaults --- bin/lbadm.cpp | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/bin/lbadm.cpp b/bin/lbadm.cpp index da596d9d..e58d2a90 100644 --- a/bin/lbadm.cpp +++ b/bin/lbadm.cpp @@ -349,25 +349,30 @@ int main(int argc, char **argv) po::options_description od("Command-line options"); auto opts = od.add_options()("help,h", "show this help message"); + std::string duration; + float weight; + u_int16_t count; + float queue, ctrl, minfactor, maxfactor; + bool ready; // parameters opts("lbname,l", po::value(), "specify name of the load balancer"); opts("lbid,i", po::value(), "override/provide id of the loadbalancer"); opts("address,a", po::value>()->multitoken(), "node IPv4/IPv6 address, can be used multiple times for 'reserve' call"); - opts("duration,d", po::value()->default_value("02:00:00"), "specify duration as '[hh[:mm[:ss]]]'"); + opts("duration,d", po::value(&duration)->default_value("02:00:00"), "specify duration as '[hh[:mm[:ss]]]'"); opts("uri,u", po::value(), "specify EJFAT_URI on the command-line instead of the environment variable"); opts("name,n", po::value(), "specify node name for registration"); opts("port,p", po::value(), "node starting listening port number"); - opts("weight,w", po::value()->default_value(1.0), "node weight"); - opts("count,c", po::value()->default_value(1), "node source count"); + opts("weight,w", po::value(&weight)->default_value(1.0), "node weight"); + opts("count,c", po::value(&count)->default_value(1), "node source count"); opts("session,s", po::value(), "override/provide session id"); - opts("queue,q", po::value()->default_value(0.0), "queue fill"); - opts("ctrl,t", po::value()->default_value(0.0), "control signal value"); - opts("ready,r", po::value()->default_value(true), "worker ready state (1 or 0)"); + opts("queue,q", po::value(&queue)->default_value(0.0), "queue fill"); + opts("ctrl,t", po::value(&ctrl)->default_value(0.0), "control signal value"); + opts("ready,r", po::value(&ready)->default_value(true), "worker ready state (1 or 0)"); opts("root,o", po::value(), "root cert for SSL communications"); opts("novalidate,v", "don't validate server certificate (conflicts with 'root')"); - opts("minfactor", po::value()->default_value(0.5), "node min factor, multiplied with the number of slots that would be assigned evenly to determine min number of slots for example, 4 nodes with a minFactor of 0.5 = (512 slots / 4) * 0.5 = min 64 slots"); - opts("maxfactor", po::value()->default_value(2.0), "multiplied with the number of slots that would be assigned evenly to determine max number of slots for example, 4 nodes with a maxFactor of 2 = (512 slots / 4) * 2 = max 256 slots set to 0 to specify no maximum"); + opts("minfactor", po::value(&minfactor)->default_value(0.5), "node min factor, multiplied with the number of slots that would be assigned evenly to determine min number of slots for example, 4 nodes with a minFactor of 0.5 = (512 slots / 4) * 0.5 = min 64 slots"); + opts("maxfactor", po::value(&maxfactor)->default_value(2.0), "multiplied with the number of slots that would be assigned evenly to determine max number of slots for example, 4 nodes with a maxFactor of 2 = (512 slots / 4) * 2 = max 256 slots set to 0 to specify no maximum"); opts("ipv6,6", "force using IPv6 control plane address if URI specifies hostname (disables cert validation)"); opts("ipv4,4", "force using IPv4 control plane address if URI specifies hostname (disables cert validation)"); opts("export,e", "suppresses other messages and prints out 'export EJFAT_URI=' returned by the LB"); @@ -517,8 +522,7 @@ int main(int argc, char **argv) // execute command auto uri_r = reserveLB(lbman, vm["lbname"].as(), vm["address"].as>(), - vm["duration"].as(), - suppress); + duration, suppress); if (uri_r.has_error()) { std::cerr << "There was an error reserving LB: " << uri_r.error().message() << std::endl; @@ -552,10 +556,10 @@ int main(int argc, char **argv) vm["name"].as(), vm["address"].as>()[0], vm["port"].as(), - vm["weight"].as(), - vm["count"].as(), - vm["minfactor"].as(), - vm["maxfactor"].as(), + weight, + count, + minfactor, + maxfactor, suppress ); @@ -589,7 +593,7 @@ int main(int argc, char **argv) } else if (vm.count("state")) { - auto int_r = sendState(lbman, vm["queue"].as(), vm["ctrl"].as(), vm["ready"].as()); + auto int_r = sendState(lbman, queue, ctrl, ready); if (int_r.has_error()) { std::cerr << "There was an error getting sending worker state update: " << int_r.error().message() << std::endl; From 774330aba2fdc78a027690b4d89d7ac83171f3bb Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Thu, 26 Sep 2024 16:13:42 -0400 Subject: [PATCH 36/73] Update distro.yml - Updating pkgconfig files for dependencies and e2sar --- .github/workflows/distro.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/distro.yml b/.github/workflows/distro.yml index 95113e20..84ad7ee8 100644 --- a/.github/workflows/distro.yml +++ b/.github/workflows/distro.yml @@ -94,6 +94,15 @@ jobs: PATH=${IDIR}/bin:$PATH LD_LIBRARY_PATH=${IDIR}/lib meson test -C build --suite=unit --timeout=0 PATH=${IDIR}/bin:$PATH LD_LIBRARY_PATH=${IDIR}/lib meson install -C build + - name: Update pkgconfig prefix files for final installation + run: | + # update pkgconfig files for the install location. DESTDIR didn't work + export IDIR=`pwd`/package/usr/local + echo Updating pkgconfig files prefix from ${IDIR} to ${{ env.FINAL_INSTALL }} + for file in `ls -1 ${IDIR}/lib/pkgconfig`; do sed -i "s|${IDIR}|${{ env.FINAL_INSTALL }}|g" ${IDIR}/lib/pkgconfig/${file}; done + echo Updating E2SAR pkgconfig + sed -i "s|${IDIR}|${{ env.FINAL_INSTALL }}|g" ${IDIR}/lib/x86_64-linux-gnu/pkgconfig/e2sar.pc + - name: Package project into .deb or .rpm run: | pushd package/ && tar -zcf ../e2sar.tar.gz usr/ && popd From 664fbeee8991b6ea8d5a575c1ce8c238db6d90c4 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Thu, 26 Sep 2024 16:20:01 -0400 Subject: [PATCH 37/73] Marking libe2sar, libe2sar_t and liblbgrpc for install --- src/grpc/meson.build | 3 ++- src/meson.build | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/grpc/meson.build b/src/grpc/meson.build index 31f5ea68..ff854266 100644 --- a/src/grpc/meson.build +++ b/src/grpc/meson.build @@ -20,7 +20,8 @@ lb_pb_ch = custom_target( # build a static library from the stubs liblbgrpc = static_library('lbgrpc', [lb_grpc_ch, lb_pb_ch], - dependencies: [grpc_dep, protobuf_dep]) + dependencies: [grpc_dep, protobuf_dep], + install : true) # install generated headers meson.add_install_script('install.sh') diff --git a/src/meson.build b/src/meson.build index ac7542e2..4009468d 100644 --- a/src/meson.build +++ b/src/meson.build @@ -13,7 +13,8 @@ libe2sar_t = static_library('e2sar_t', # build one library from the E2SAR and gRPC library pieces libe2sar = static_library('e2sar', objects : [libe2sar_t.extract_all_objects(recursive: false), - liblbgrpc.extract_all_objects(recursive: false)]) + liblbgrpc.extract_all_objects(recursive: false)], + install : true) # The pybind subdir('pybind') From c3b289134d2e646fb3943ac823374ea43644db0d Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Thu, 26 Sep 2024 16:31:52 -0400 Subject: [PATCH 38/73] Update distro.yml - Forgot the FINAL_INSTALL variable --- .github/workflows/distro.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/distro.yml b/.github/workflows/distro.yml index 84ad7ee8..db9bac4e 100644 --- a/.github/workflows/distro.yml +++ b/.github/workflows/distro.yml @@ -5,6 +5,7 @@ on: env: E2SAR_VER: 0.1.2 E2SAR_DEP: boost-1.85.0-grpc-1.54.1 + FINAL_INSTALL: /usr/local jobs: build: From 2329f38bc49dd14ecc4a8138cfebc3e34bb86081 Mon Sep 17 00:00:00 2001 From: Srinivas Sivakumar Date: Fri, 27 Sep 2024 15:55:09 -0400 Subject: [PATCH 39/73] Update deploydebs.yml trying ref_name --- .github/workflows/deploydebs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploydebs.yml b/.github/workflows/deploydebs.yml index 7ce7ceb1..4740a2f6 100644 --- a/.github/workflows/deploydebs.yml +++ b/.github/workflows/deploydebs.yml @@ -49,7 +49,7 @@ jobs: - name: create GH release run: | cd E2SAR - gh release create E2SAR-${{ env.E2SAR_VER }}-${{ matrix.os }} ./*.deb + gh release create E2SAR-${{ github.ref_name }}-${{ env.E2SAR_VER }}-${{ matrix.os }} ./*.deb env: GITHUB_TOKEN: ${{ github.TOKEN }} shell: bash From 7bf7dbe020f4fb1c3c1619918d6551861969af03 Mon Sep 17 00:00:00 2001 From: Srinivas Sivakumar Date: Fri, 27 Sep 2024 16:03:03 -0400 Subject: [PATCH 40/73] Update distro.yml Added branch/tag name to package identifier --- .github/workflows/distro.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/distro.yml b/.github/workflows/distro.yml index db9bac4e..a5a6c130 100644 --- a/.github/workflows/distro.yml +++ b/.github/workflows/distro.yml @@ -116,7 +116,7 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: ${{ matrix.os }}-e2sar-${{ env.E2SAR_VER }}-package + name: ${{ matrix.os }}-e2sar-${{ github.ref_name }}-${{ env.E2SAR_VER }}-package path: | *.deb *.rpm From d89b2566e85b8af922e5c2172bb4625ed1af9aab Mon Sep 17 00:00:00 2001 From: Srinivas Sivakumar Date: Fri, 27 Sep 2024 16:04:56 -0400 Subject: [PATCH 41/73] Update deploydebs.yml added branch name --- .github/workflows/deploydebs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploydebs.yml b/.github/workflows/deploydebs.yml index 4740a2f6..978d0242 100644 --- a/.github/workflows/deploydebs.yml +++ b/.github/workflows/deploydebs.yml @@ -43,7 +43,7 @@ jobs: uses: dawidd6/action-download-artifact@v6 with: workflow: distro.yml - name: ${{ matrix.os }}-e2sar-${{ env.E2SAR_VER }}-package + name: ${{ matrix.os }}-e2sar-${{ github.ref_name }}-${{ env.E2SAR_VER }}-package path: E2SAR - name: create GH release From 407825d08ec7a008840d104d69ff92a56ab989c8 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Thu, 19 Sep 2024 11:20:55 -0400 Subject: [PATCH 42/73] Made weight, min and max_factor settable via flags; added sane defaults From e1d7a751aeff505294af7c0c066ac341be136e77 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Fri, 27 Sep 2024 16:46:18 -0400 Subject: [PATCH 43/73] Added unit tests for INI file reading of flags; modified meson.build to properly produce a unified e2sar.a --- src/meson.build | 9 +++++---- test/e2sar_reas_test.cpp | 37 +++++++++++++++++++++++++++++++++++++ test/e2sar_seg_test.cpp | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 4 deletions(-) diff --git a/src/meson.build b/src/meson.build index 4009468d..779701b6 100644 --- a/src/meson.build +++ b/src/meson.build @@ -11,10 +11,11 @@ libe2sar_t = static_library('e2sar_t', install : true) # build one library from the E2SAR and gRPC library pieces -libe2sar = static_library('e2sar', - objects : [libe2sar_t.extract_all_objects(recursive: false), - liblbgrpc.extract_all_objects(recursive: false)], - install : true) +#libe2sar = static_library('e2sar', +# objects : [libe2sar_t.extract_all_objects(recursive: false), +# liblbgrpc.extract_all_objects(recursive: false)], +# install : true) +libe2sar = static_library('e2sar', link_whole : [libe2sar_t, liblbgrpc], install : true) # The pybind subdir('pybind') diff --git a/test/e2sar_reas_test.cpp b/test/e2sar_reas_test.cpp index 2f27bd0d..ad39c03c 100644 --- a/test/e2sar_reas_test.cpp +++ b/test/e2sar_reas_test.cpp @@ -8,6 +8,9 @@ #include #include #include +#include +#include +#include #include "e2sar.hpp" @@ -634,4 +637,38 @@ BOOST_AUTO_TEST_CASE(DPReasTest4) } } +BOOST_AUTO_TEST_CASE(DPReasTest5) +{ + // test reading SegmenterFlags from INI files + // generate a file, read it in and compare expected values + boost::property_tree::ptree paramTree; + Reassembler::ReassemblerFlags rFlags; + std::string iniFileName = "/tmp/reassembler.ini"; + + // fill in the parameters + paramTree.put("general.useCP", false); + paramTree.put("control-plane.useHostAddress", true); + paramTree.put("data-plane.rcvSocketBufSize", 10000); + + try { + boost::property_tree::ini_parser::write_ini(iniFileName, paramTree); + } catch(boost::property_tree::ini_parser_error &ie) { + std::cout << "Unable to parse the segmenter flags configuration file "s + iniFileName << std::endl; + BOOST_CHECK(false); + } + + Reassembler::ReassemblerFlags segDefaults; + Reassembler::ReassemblerFlags readFlags; + auto res = Reassembler::ReassemblerFlags::getFromINI(iniFileName); + BOOST_CHECK(!res.has_error()); + readFlags = res.value(); + + BOOST_CHECK(readFlags.useCP == paramTree.get("general.useCP")); + BOOST_CHECK(readFlags.useHostAddress == paramTree.get("control-plane.useHostAddress")); + BOOST_CHECK(readFlags.validateCert == segDefaults.validateCert); + BOOST_CHECK(readFlags.rcvSocketBufSize == paramTree.get("data-plane.rcvSocketBufSize")); + + std::remove(iniFileName.c_str()); +} + BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/test/e2sar_seg_test.cpp b/test/e2sar_seg_test.cpp index 3c1033b5..a5184684 100644 --- a/test/e2sar_seg_test.cpp +++ b/test/e2sar_seg_test.cpp @@ -1,6 +1,9 @@ #define BOOST_TEST_MODULE DPSegTests #include #include +#include +#include +#include #include #include #include @@ -8,6 +11,9 @@ #include #include #include +#include +#include +#include #include "e2sar.hpp" @@ -332,4 +338,37 @@ BOOST_AUTO_TEST_CASE(DPSegTest4) // stop threads and exit } +BOOST_AUTO_TEST_CASE(DPSegTest5) +{ + // test reading SegmenterFlags from INI files + // generate a file, read it in and compare expected values + boost::property_tree::ptree paramTree; + Segmenter::SegmenterFlags sFlags; + std::string iniFileName = "/tmp/segmenter.ini"; + + // fill in the parameters + paramTree.put("general.useCP", false); + paramTree.put("data-plane.zeroCopy", true); + paramTree.put("data-plane.sndSocketBufSize", 10000); + + try { + boost::property_tree::ini_parser::write_ini(iniFileName, paramTree); + } catch(boost::property_tree::ini_parser_error &ie) { + std::cout << "Unable to parse the segmenter flags configuration file "s + iniFileName << std::endl; + BOOST_CHECK(false); + } + + Segmenter::SegmenterFlags segDefaults; + Segmenter::SegmenterFlags readFlags; + auto res = Segmenter::SegmenterFlags::getFromINI(iniFileName); + BOOST_CHECK(!res.has_error()); + readFlags = res.value(); + + BOOST_CHECK(readFlags.useCP == paramTree.get("general.useCP")); + BOOST_CHECK(readFlags.zeroCopy == paramTree.get("data-plane.zeroCopy")); + BOOST_CHECK(readFlags.dpV6 == segDefaults.dpV6); + BOOST_CHECK(readFlags.sndSocketBufSize == paramTree.get("data-plane.sndSocketBufSize")); + + std::remove(iniFileName.c_str()); +} BOOST_AUTO_TEST_SUITE_END() From 9137b88be1441c9f0f0638bb60c6e4267ea6d9cc Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Fri, 27 Sep 2024 17:55:01 -0400 Subject: [PATCH 44/73] More careful accounting of lost events (by logging them into thread-local set of event ids that have been declared lost) --- include/e2sarDPReassembler.hpp | 24 +++++++++++++++++++++--- src/e2sarDPReassembler.cpp | 15 +++++++++++---- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/include/e2sarDPReassembler.hpp b/include/e2sarDPReassembler.hpp index b6b9c3ac..46779579 100644 --- a/include/e2sarDPReassembler.hpp +++ b/include/e2sarDPReassembler.hpp @@ -14,6 +14,7 @@ #include #include #include +#include #include @@ -137,16 +138,20 @@ namespace e2sar boost::lockfree::queue eventQueue{QSIZE}; std::atomic eventQueueDepth{0}; - // push event on the event queue - inline void enqueue(EventQueueItem* item) noexcept + // push event on the common event queue + // return 1 if event is lost, 0 on success + inline int enqueue(EventQueueItem* item) noexcept { + int ret = 0; if (eventQueue.push(item)) eventQueueDepth++; else - recvStats.enqueueLoss++; // event lost, queue was full + ret = 1; // event lost, queue was full // queue is lock free so we don't lock recvThreadCond.notify_all(); + return ret; } + // pop event off the event queue inline EventQueueItem* dequeue() noexcept { @@ -209,6 +214,8 @@ namespace e2sar // segments are transmitted, they are guarangeed to go // to the same port boost::unordered_map, EventQueueItem*, pair_hash, pair_equal> eventsInProgress; + // thread local instance of events we lost + boost::container::flat_set> lostEvents; // CPU core ids std::vector cpuCoreList; @@ -228,6 +235,17 @@ namespace e2sar result _close(); // thread loop void _threadBody(); + + // log a lost event via a set of known lost + inline void logLostEvent(std::pair evt) + { + if (lostEvents.contains(evt)) + return; + // this is thread-local + lostEvents.insert(evt); + // this is atomic + reas.recvStats.enqueueLoss++; + } }; friend struct RecvThreadState; std::list recvThreadState; diff --git a/src/e2sarDPReassembler.cpp b/src/e2sarDPReassembler.cpp index c5c92f65..a5ed714a 100644 --- a/src/e2sarDPReassembler.cpp +++ b/src/e2sarDPReassembler.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include "portable_endian.h" @@ -136,8 +137,6 @@ namespace e2sar void Reassembler::RecvThreadState::_threadBody() { - std::set activeEventNumbers{}; - while(!reas.threadsStop) { fd_set curSet{fdSet}; @@ -153,13 +152,14 @@ namespace e2sar auto inWaiting = nowT - it->second->firstSegment; auto inWaiting_ms = boost::chrono::duration_cast(inWaiting); if (inWaiting_ms > boost::chrono::milliseconds(reas.eventTimeout_ms)) { + // check if this event number has been seen as lost + logLostEvent(it->first); // deallocate event (ood queue and event buffer) it->second->cleanup(recvBufferPool); delete it->second->event; // deallocate queue item delete it->second; it = eventsInProgress.erase(it); // erase returns the next element (or end()) - reas.recvStats.enqueueLoss++; } else { ++it; // Just advance the iterator if no deletion } @@ -298,7 +298,14 @@ namespace e2sar eventsInProgress.erase(std::make_pair(item->eventNum, item->dataId)); // queue it up for the user to receive - reas.enqueue(item); + auto ret = reas.enqueue(item); + // event lost on enqueuing + if (ret == 1) + { + logLostEvent(std::make_pair(item->eventNum, item->dataId)); + // free up the item + delete item; + } // update statistics reas.recvStats.eventSuccess++; From 11ee839e7a32eb166dff3221aec388fab0e8885e Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Fri, 27 Sep 2024 21:22:21 -0400 Subject: [PATCH 45/73] Adding debug printout of lost event numbers --- src/e2sarDPReassembler.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/e2sarDPReassembler.cpp b/src/e2sarDPReassembler.cpp index a5ed714a..dd5b9a2b 100644 --- a/src/e2sarDPReassembler.cpp +++ b/src/e2sarDPReassembler.cpp @@ -2,7 +2,6 @@ #include #include #include -#include #include #include "portable_endian.h" @@ -313,6 +312,13 @@ namespace e2sar } } + std::cout << "The following events were lost: "; + for (const auto& elem :lostEvents) + { + std::cout << "(" << elem.first << ", " << elem.second << ") "; + } + std::cout << std::endl; + // close on exit auto res = _close(); } From fe89f187578cc8ec8d8aa9017ef4b945e6fe5a9b Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Fri, 27 Sep 2024 23:08:48 -0400 Subject: [PATCH 46/73] Limiting the length of payload printed --- scripts/scapy/snifgen.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/scripts/scapy/snifgen.py b/scripts/scapy/snifgen.py index e929fa4d..7efa629f 100755 --- a/scripts/scapy/snifgen.py +++ b/scripts/scapy/snifgen.py @@ -43,6 +43,19 @@ class LBPacket(Packet): ] LBPacketLength = 2 + 1 + 1 + 2 + 2 + 8 +class TruncatedStrLenField(StrLenField): + + def __init__(self, name, default, length_from, truncate_to=10): + # Add a truncate_to parameter to control the number of characters to show + super().__init__(name, default, None, length_from) + self.truncate_to = truncate_to + + def i2repr(self, pkt, x): + if x is None: + return "" + # Truncate the string to `self.truncate_to` characters for display + return repr(x[:self.truncate_to] + ("..." if len(x) > self.truncate_to else "")) + # RE header itself class REPacket(Packet): name = "REPacket" @@ -54,7 +67,7 @@ class REPacket(Packet): IntField('bufferOffset', 0), IntField('bufferLength', 0), # note this is Event Length LongField('eventNumber', 0), - StrLenField('pld', '', length_from=lambda p: p.bufferLength) + TruncatedStrLenField('pld', '', length_from=lambda p: p.bufferLength, truncate_to=20) ] REPacketLength = 1 + 1 + 2 + 4 + 4 + 8 From 46831967e2019484810ce2a7c6a466c33a648189 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Fri, 27 Sep 2024 23:31:53 -0400 Subject: [PATCH 47/73] Updated truncated field --- scripts/scapy/snifgen.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/scapy/snifgen.py b/scripts/scapy/snifgen.py index 7efa629f..c49d7710 100755 --- a/scripts/scapy/snifgen.py +++ b/scripts/scapy/snifgen.py @@ -44,17 +44,18 @@ class LBPacket(Packet): LBPacketLength = 2 + 1 + 1 + 2 + 2 + 8 class TruncatedStrLenField(StrLenField): + __slots__ = ["trunc"] def __init__(self, name, default, length_from, truncate_to=10): # Add a truncate_to parameter to control the number of characters to show super().__init__(name, default, None, length_from) - self.truncate_to = truncate_to + self.trunc = truncate_to def i2repr(self, pkt, x): if x is None: return "" # Truncate the string to `self.truncate_to` characters for display - return repr(x[:self.truncate_to] + ("..." if len(x) > self.truncate_to else "")) + return repr(x[:self.trunc] + ("..." if len(x) > self.trunc else "")) # RE header itself class REPacket(Packet): From 17cba6e11d864755ae79e5b3c905f282dba558f5 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Sat, 28 Sep 2024 00:19:21 -0400 Subject: [PATCH 48/73] Removing double-print --- scripts/scapy/snifgen.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/scripts/scapy/snifgen.py b/scripts/scapy/snifgen.py index c49d7710..46ea5140 100755 --- a/scripts/scapy/snifgen.py +++ b/scripts/scapy/snifgen.py @@ -55,7 +55,7 @@ def i2repr(self, pkt, x): if x is None: return "" # Truncate the string to `self.truncate_to` characters for display - return repr(x[:self.trunc] + ("..." if len(x) > self.trunc else "")) + return repr(x[:self.trunc].decode("utf-8") + ("..." if len(x) > self.trunc else "")) # RE header itself class REPacket(Packet): @@ -172,14 +172,13 @@ def packet_callback(packet): elif LBPacket in packet: valid, error_msg = validate_lb_packet(packet[LBPacket]) if valid: - packet[IP].show() - else: - print(f"LB packet validation error: {error_msg}") - valid, error_msg = validate_re_packet(packet[REPacket]) - if valid: - packet[IP].show() + valid, error_msg = validate_re_packet(packet[REPacket]) + if valid: + packet[IP].show() + else: + print(f"RE packet validation error: {error_msg}") else: - print(f"RE packet validation error: {error_msg}") + print(f"LB packet validation error: {error_msg}") elif REPacket in packet: valid, error_msg = validate_re_packet(packet[REPacket]) if valid: From 176b1441cade7505db0ae20b337f2fdce197dc1e Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Sat, 28 Sep 2024 00:19:44 -0400 Subject: [PATCH 49/73] Adding small delay after enqueing send packets to allow them to leave --- bin/e2sar_perf.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/e2sar_perf.cpp b/bin/e2sar_perf.cpp index d64cc9d1..c89dd137 100644 --- a/bin/e2sar_perf.cpp +++ b/bin/e2sar_perf.cpp @@ -148,6 +148,10 @@ result sendEvents(Segmenter &s, EventNum_t startEventNum, size_t numEvents, evtBufferPool->free(item); boost::this_thread::sleep_until(until); } + // sleep to allow small number of frames to leave + boost::chrono::seconds duration(1); + boost::this_thread::sleep_for(duration); + auto stats = s.getSendStats(); evtBufferPool->purge_memory(); From 0fcaf5d9805416e172350ff742cc53e9a8e4ae90 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Sat, 28 Sep 2024 00:27:55 -0400 Subject: [PATCH 50/73] Removing Raw --- scripts/scapy/snifgen.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/scapy/snifgen.py b/scripts/scapy/snifgen.py index 46ea5140..f4bcd644 100755 --- a/scripts/scapy/snifgen.py +++ b/scripts/scapy/snifgen.py @@ -163,6 +163,9 @@ def validate_re_packet(packet): # generic callback for all headers def packet_callback(packet): + if Raw in packet: + del packet[Raw] + if SyncPacket in packet: valid, error_msg = validate_sync_packet(packet[SyncPacket]) if valid: From d08a091d663e5fd391559a6798d6ddbe1a2a5f10 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Sat, 28 Sep 2024 00:43:26 -0400 Subject: [PATCH 51/73] Correcting super() --- scripts/scapy/snifgen.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/scapy/snifgen.py b/scripts/scapy/snifgen.py index f4bcd644..b2437f10 100755 --- a/scripts/scapy/snifgen.py +++ b/scripts/scapy/snifgen.py @@ -48,7 +48,7 @@ class TruncatedStrLenField(StrLenField): def __init__(self, name, default, length_from, truncate_to=10): # Add a truncate_to parameter to control the number of characters to show - super().__init__(name, default, None, length_from) + super().__init__(name, default, length_from=length_from) self.trunc = truncate_to def i2repr(self, pkt, x): @@ -165,7 +165,7 @@ def validate_re_packet(packet): def packet_callback(packet): if Raw in packet: del packet[Raw] - + if SyncPacket in packet: valid, error_msg = validate_sync_packet(packet[SyncPacket]) if valid: From c2ecf7e11daa6022c38def67f35edd9ab64a1142 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Sat, 28 Sep 2024 01:21:28 -0400 Subject: [PATCH 52/73] Adding numSendSockets option --- bin/e2sar_perf.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/e2sar_perf.cpp b/bin/e2sar_perf.cpp index c89dd137..3a4e0503 100644 --- a/bin/e2sar_perf.cpp +++ b/bin/e2sar_perf.cpp @@ -272,7 +272,7 @@ int main(int argc, char **argv) u_int16_t mtu; u_int32_t eventSourceId; u_int16_t dataId; - size_t numThreads; + size_t numThreads, numSockets; float rateGbps; int sockBufSize; int durationSec; @@ -293,6 +293,7 @@ int main(int argc, char **argv) opts("src", po::value(&eventSourceId)->default_value(1234), "Event source (default 1234) [s]"); opts("dataid", po::value(&dataId)->default_value(4321), "Data id (default 4321) [s]"); opts("threads,t", po::value(&numThreads)->default_value(1), "number of receive threads (defaults to 1) [r]"); + opts("sockets,s", po::value(&numSockets)->default_value(4), "number of send sockets (defaults to 4) [r]"); opts("rate", po::value(&rateGbps)->default_value(1.0), "send rate in Gbps (defaults to 1.0)"); opts("period,p", po::value(&reportThreadSleepMs)->default_value(1000), "receive side reporting thread sleep period in ms (defaults to 1000) [r]"); opts("bufsize,b", po::value(&sockBufSize)->default_value(1024*1024*3), "send or receive socket buffer size (default to 3MB)"); @@ -415,6 +416,7 @@ int main(int argc, char **argv) sflags.useCP = withCP; sflags.mtu = mtu; sflags.sndSocketBufSize = sockBufSize; + sflags.numSendSockets = numSockets; } std::cout << "Control plane will be " << (sflags.useCP ? "ON" : "OFF") << std::endl; std::cout << (sflags.useCP ? "*** Make sure the LB has been reserved and the URI reflects the reserved instance information." : From 209c7e6403700cc148c024b29b73a1f26983e6f7 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Sat, 28 Sep 2024 01:30:09 -0400 Subject: [PATCH 53/73] Disambiguating options --- bin/e2sar_perf.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/e2sar_perf.cpp b/bin/e2sar_perf.cpp index 3a4e0503..df466cf8 100644 --- a/bin/e2sar_perf.cpp +++ b/bin/e2sar_perf.cpp @@ -292,8 +292,8 @@ int main(int argc, char **argv) opts("mtu,m", po::value(&mtu)->default_value(1500), "MTU (default 1500) [s]"); opts("src", po::value(&eventSourceId)->default_value(1234), "Event source (default 1234) [s]"); opts("dataid", po::value(&dataId)->default_value(4321), "Data id (default 4321) [s]"); - opts("threads,t", po::value(&numThreads)->default_value(1), "number of receive threads (defaults to 1) [r]"); - opts("sockets,s", po::value(&numSockets)->default_value(4), "number of send sockets (defaults to 4) [r]"); + opts("threads", po::value(&numThreads)->default_value(1), "number of receive threads (defaults to 1) [r]"); + opts("sockets", po::value(&numSockets)->default_value(4), "number of send sockets (defaults to 4) [r]"); opts("rate", po::value(&rateGbps)->default_value(1.0), "send rate in Gbps (defaults to 1.0)"); opts("period,p", po::value(&reportThreadSleepMs)->default_value(1000), "receive side reporting thread sleep period in ms (defaults to 1000) [r]"); opts("bufsize,b", po::value(&sockBufSize)->default_value(1024*1024*3), "send or receive socket buffer size (default to 3MB)"); From 498642f633e8bc4e1ac8f353d34b13a0f7ccfcec Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Sat, 28 Sep 2024 18:30:38 -0400 Subject: [PATCH 54/73] Added ability to parse pcap files --- scripts/scapy/snifgen.py | 41 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/scripts/scapy/snifgen.py b/scripts/scapy/snifgen.py index b2437f10..988f1aed 100755 --- a/scripts/scapy/snifgen.py +++ b/scripts/scapy/snifgen.py @@ -12,10 +12,13 @@ from typing import List +from datetime import datetime + from scapy.all import * from scapy.packet import Packet, bind_layers from scapy.fields import ShortField, StrLenField + # Sync header class SyncPacket(Packet): name = "SyncPacket" @@ -197,8 +200,10 @@ def packet_callback(packet): operations = parser.add_mutually_exclusive_group(required=True) operations.add_argument("-l", "--listen", action="store_true", help="listen for incoming packets and try to parse and validate them") operations.add_argument("-g", "--generate", action="store_true", help="generate new packets of specific types") + operations.add_argument("-a", "--parse", action="store_true", help="parse a pcap file") parser.add_argument("-p", "--port", action="store", help="UDP port (for -l and -g)", default=19522, type=int) - parser.add_argument("-c", "--count", action="store", help="number of packet streams (if pld larger than mtu, otherwise packets) to generate or expect", default=10, type=int) + parser.add_argument("-n", "--nports", action="store", type=int, default=1, help="number of ports starting with -p to listen on") + parser.add_argument("-c", "--count", action="store", help="number of packet streams (if pld larger than mtu, otherwise packets) to generate or expect or parse", default=10, type=int) parser.add_argument("--ip", action="store", help="IP address to which to send the packet(s) or listen from") parser.add_argument("--show", action="store_true", default=False, help="only show the packet without sending it (with -g)") parser.add_argument("--entropy", action="store", default=0, help="entropy value for LB+RE packet", type=int) @@ -209,6 +214,7 @@ def packet_callback(packet): parser.add_argument("--mtu", action="store", type=int, default=1500, help="set the MTU length, so LB+RE and RE packets can be fragmented.") parser.add_argument("--pld", action="store", help="payload for LB+RE or RE packets. May be broken up if MTU size insufficient") parser.add_argument("--iface", action="store", default="all", help="which interface should we listen on (defaults to all)") + parser.add_argument("-f", "--file", action="store", help="pcap file name to parse", default="./e2sar.pcap") packet_types = parser.add_mutually_exclusive_group(required=True) packet_types.add_argument("--sync", action="store_true", help="listen for or generate sync packets") packet_types.add_argument("--lbre", action="store_true", help="listen for or generate packets with LB+RE header") @@ -245,10 +251,12 @@ def packet_callback(packet): send(p) elif args.listen: # craft a filter + listeningPorts = [x + args.port for x in range(0, args.nports)] + portFilter = "or".join([f" dst port {port} " for port in listeningPorts]) if args.ip: - filter = f'ip dst host {args.ip} and udp dst port {args.port}' + filter = f'udp and dst host {args.ip} and \\( {portFilter} \\)' else: - filter=f'udp dst port {args.port}' + filter=f'udp {portFilter}' if args.sync: # sync packets @@ -272,5 +280,32 @@ def packet_callback(packet): # Start sniffing for packets sniff(iface=interfaces, filter=filter, prn=packet_callback, count=args.count) + + elif args.parse: + + if args.sync: + # sync packets + bind_sync_hdr(args.port) + packet_type = 'Sync' + elif args.lbre: + # lb+re packets + bind_lb_hdr(args.port) + packet_type = 'LB+RE' + elif args.re: + bind_re_hdr(args.port) + packet_type = 'RE' + + print(f'Looking for {packet_type} packets in PCAP file {args.file}') + try: + packets = rdpcap(args.file, count=args.count) + + for packet in packets: + packet_time = packet.time + if packet_time is not None: + human_readable_time = datetime.fromtimestamp(float(packet.time)) + print(f"Timestamp: {human_readable_time}") + packet_callback(packet) + except Exception as e: + print(f'Unable to parse file {args.file} due to exception: {e}') print('Finished') From e9af398d9e505aca3f8f8d6450b78cc5ef3b1ceb Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Sat, 28 Sep 2024 18:35:41 -0400 Subject: [PATCH 55/73] Clarified help message --- scripts/scapy/snifgen.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/scapy/snifgen.py b/scripts/scapy/snifgen.py index 988f1aed..06c2ba58 100755 --- a/scripts/scapy/snifgen.py +++ b/scripts/scapy/snifgen.py @@ -197,10 +197,11 @@ def packet_callback(packet): if __name__ == "__main__": parser = argparse.ArgumentParser() + operations = parser.add_mutually_exclusive_group(required=True) operations.add_argument("-l", "--listen", action="store_true", help="listen for incoming packets and try to parse and validate them") operations.add_argument("-g", "--generate", action="store_true", help="generate new packets of specific types") - operations.add_argument("-a", "--parse", action="store_true", help="parse a pcap file") + operations.add_argument("-a", "--parse", action="store_true", help="parse a pcap file. The recommended way to capture is something like this 'sudo tcpdump -s 200 -tttt -i enp7s0 udp \\( dst port 19522 or dst port 19523 \\) -w e2sar.pcap'") parser.add_argument("-p", "--port", action="store", help="UDP port (for -l and -g)", default=19522, type=int) parser.add_argument("-n", "--nports", action="store", type=int, default=1, help="number of ports starting with -p to listen on") parser.add_argument("-c", "--count", action="store", help="number of packet streams (if pld larger than mtu, otherwise packets) to generate or expect or parse", default=10, type=int) From ad6e63fe8bcf008e505ac46104a5ae38dbc25681 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Sat, 28 Sep 2024 23:31:28 -0400 Subject: [PATCH 56/73] Avoiding conversion of port numbers to service names; Adding ability to bind RE header to multiple ports in listen and parse modes --- scripts/scapy/snifgen.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/scripts/scapy/snifgen.py b/scripts/scapy/snifgen.py index 06c2ba58..89fc638d 100755 --- a/scripts/scapy/snifgen.py +++ b/scripts/scapy/snifgen.py @@ -85,9 +85,13 @@ def bind_lb_hdr(dport): bind_layers(UDP, LBPacket, dport=dport) bind_layers(LBPacket, REPacket) -# bind RE header to specified port +# bind RE header to specified ports def bind_re_hdr(dport): - bind_layers(UDP, REPacket, dport=dport) + if (isinstance(dport, list)): + for p in dport: + bind_layers(UDP, REPacket, dport=p) + else: + bind_layers(UDP, REPacket, dport=dport) # generate a packet with specific field values def genSyncPkt(ip_addr: str, udp_dport: int, eventSrcId: int, eventNumber: int, avgEventRateHz: int) -> Packet: @@ -223,6 +227,10 @@ def packet_callback(packet): args = parser.parse_args() + # equivalent to -n - not to resolve port numbers to service names + conf.noenum.add(UDP.sport) + conf.noenum.add(UDP.dport) + if args.generate: if not args.ip: print(f'--ip option is required (use dotted notation to specify IPv4 address)') @@ -268,7 +276,7 @@ def packet_callback(packet): bind_lb_hdr(args.port) packet_type = 'LB+RE' elif args.re: - bind_re_hdr(args.port) + bind_re_hdr([x + args.port for x in range(0, args.nports)]) packet_type = 'RE' # Linux supports "any" shortcut interface specification, but other OSs may not, so we just list @@ -293,12 +301,16 @@ def packet_callback(packet): bind_lb_hdr(args.port) packet_type = 'LB+RE' elif args.re: - bind_re_hdr(args.port) + #bind_re_hdr(args.port) + bind_re_hdr([x + args.port for x in range(0, args.nports)]) packet_type = 'RE' print(f'Looking for {packet_type} packets in PCAP file {args.file}') try: - packets = rdpcap(args.file, count=args.count) + if (args.count != 0): + packets = rdpcap(args.file, count=args.count) + else: + packets = rdpcap(args.file) for packet in packets: packet_time = packet.time From 60ae4567618e442eff5d4492f2747bc9406f0dd6 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Sun, 29 Sep 2024 22:55:00 -0400 Subject: [PATCH 57/73] Cleaned up options; added --lbresync combined LBRE+Sync packets listen and parse; disambiguated syncports and data ports --- scripts/scapy/snifgen.py | 90 ++++++++++++++++++++++++++-------------- 1 file changed, 60 insertions(+), 30 deletions(-) diff --git a/scripts/scapy/snifgen.py b/scripts/scapy/snifgen.py index 89fc638d..a9f003a1 100755 --- a/scripts/scapy/snifgen.py +++ b/scripts/scapy/snifgen.py @@ -106,7 +106,7 @@ def genLBREPkt(ip_addr: str, udp_port: int, entropy: int, dataId: int, eventNumb max_segment_len = mtu - LBPacketLength - REPacketLength if max_segment_len < 0: print(f'MTU of {mtu} too short to accommodate LB header {LBPacketLength} and RE header {REPacketLength}') - sys.exit() + sys.exit(-1) segment_offset = 0 segment_end = len(payload) if len(payload) < max_segment_len else max_segment_len while segment_offset < len(payload): @@ -129,7 +129,7 @@ def genREPkt(ip_addr: str, udp_port: int, dataId: int, eventNumber: int, payload max_segment_len = mtu - REPacketLength if max_segment_len < 0: print(f'MTU of {mtu} too short to accommodate RE header {REPacketLength}') - sys.exit() + sys.exit(-1) segment_offset = 0 segment_end = len(payload) if len(payload) < max_segment_len else max_segment_len while segment_offset < len(payload): @@ -148,8 +148,8 @@ def genREPkt(ip_addr: str, udp_port: int, dataId: int, eventNumber: int, payload def validate_sync_packet(packet): if not (packet.preamble == b'LC'): return False, f"Preamble must be 'LC' instead of {packet.preamble}" - if not (packet.version == 1): - return False, f"Expected version number 1, not {packet.version}" + if not (packet.version == 2): + return False, f"Expected version number 2, not {packet.version}" return True, "" # validate LB packet @@ -201,29 +201,30 @@ def packet_callback(packet): if __name__ == "__main__": parser = argparse.ArgumentParser() - operations = parser.add_mutually_exclusive_group(required=True) operations.add_argument("-l", "--listen", action="store_true", help="listen for incoming packets and try to parse and validate them") operations.add_argument("-g", "--generate", action="store_true", help="generate new packets of specific types") operations.add_argument("-a", "--parse", action="store_true", help="parse a pcap file. The recommended way to capture is something like this 'sudo tcpdump -s 200 -tttt -i enp7s0 udp \\( dst port 19522 or dst port 19523 \\) -w e2sar.pcap'") - parser.add_argument("-p", "--port", action="store", help="UDP port (for -l and -g)", default=19522, type=int) - parser.add_argument("-n", "--nports", action="store", type=int, default=1, help="number of ports starting with -p to listen on") - parser.add_argument("-c", "--count", action="store", help="number of packet streams (if pld larger than mtu, otherwise packets) to generate or expect or parse", default=10, type=int) + parser.add_argument("-p", "--port", action="store", default=19522, type=int, help="UDP data port (only port for --lbre, starting port for --re)") + parser.add_argument("-y", "--syncport", action="store", help="UDP sync port", default=19010, type=int) + parser.add_argument("-n", "--nports", action="store", type=int, default=1, help="number of ports starting with -p to listen on for --re") + parser.add_argument("-c", "--count", action="store", default=10, type=int, help="number of events (if pld larger than mtu, otherwise packets) to generate or expect or parse") parser.add_argument("--ip", action="store", help="IP address to which to send the packet(s) or listen from") parser.add_argument("--show", action="store_true", default=False, help="only show the packet without sending it (with -g)") - parser.add_argument("--entropy", action="store", default=0, help="entropy value for LB+RE packet", type=int) - parser.add_argument("--event", action="store", default=0, help="event number for sync, LB+RE and RE packet", type=int) + parser.add_argument("--entropy", action="store", default=0, type=int, help="entropy value for LB+RE packet") + parser.add_argument("--event", action="store", default=0, type=int, help="event number for sync, LB+RE and RE packet") parser.add_argument("--rate", action="store", default=100, type=int, help="event rate in Hz for Sync packet") parser.add_argument("--dataid", action="store", default=1, type=int, help="data id for RE packet") parser.add_argument("--srcid", action="store", default=1, type=int, help="source id for Sync packet") parser.add_argument("--mtu", action="store", type=int, default=1500, help="set the MTU length, so LB+RE and RE packets can be fragmented.") - parser.add_argument("--pld", action="store", help="payload for LB+RE or RE packets. May be broken up if MTU size insufficient") + parser.add_argument("--pld", action="store", help="payload for LB+RE or RE packets. May be broken up if MTU size insufficient", default="This is a default payload.") parser.add_argument("--iface", action="store", default="all", help="which interface should we listen on (defaults to all)") parser.add_argument("-f", "--file", action="store", help="pcap file name to parse", default="./e2sar.pcap") packet_types = parser.add_mutually_exclusive_group(required=True) - packet_types.add_argument("--sync", action="store_true", help="listen for or generate sync packets") - packet_types.add_argument("--lbre", action="store_true", help="listen for or generate packets with LB+RE header") - packet_types.add_argument("--re", action="store_true", help="listen for or generate packets with just the RE header") + packet_types.add_argument("--sync", action="store_true", help="listen for, parse or generate Sync packets") + packet_types.add_argument("--lbre", action="store_true", help="listen for, parse or generate packets with LB+RE header") + packet_types.add_argument("--re", action="store_true", help="listen for, parse or generate packets with just the RE header") + packet_types.add_argument("--lbresync", action="store_true", help="listen for or parse LB+RE and Sync packets") args = parser.parse_args() @@ -234,21 +235,35 @@ def packet_callback(packet): if args.generate: if not args.ip: print(f'--ip option is required (use dotted notation to specify IPv4 address)') - sys.exit() + sys.exit(-1) # generate and show a packet + if args.lbresync: + print('Invalid option combination -g/--generate and --lbresync - multiple layers are bound, pick one') + sys.exit(-1) + + if args.sync: + # sync packets + bind_sync_hdr(args.syncport) + print(f'Generating Sync(port {args.syncport}) packets for mtu {args.mtu}') + elif args.lbre: + # lb+re packets + bind_lb_hdr(args.port) + print(f'Generating LBRE(port {args.port}) packets for mtu {args.mtu}') + elif args.re: + # re packets (only one port) + bind_re_hdr(args.port) + print(f'Generating RE(port {args.port}) packets for mtu {args.mtu}') + packets = list() for i in range(args.count): if args.sync: # sync packets - bind_sync_hdr(args.port) packets.append(genSyncPkt(args.ip, args.port, args.srcid, args.event + i, args.rate)) elif args.lbre: # lb+re packets - bind_lb_hdr(args.port) packets.extend(genLBREPkt(args.ip, args.port, args.entropy, args.dataid, args.event, args.pld, args.mtu)) elif args.re: - # re packets - bind_re_hdr(args.port) + # re packets (only one port) packets.extend(genREPkt(args.ip, args.port, args.dataid, args.event, args.pld, args.mtu)) if args.show: @@ -269,15 +284,23 @@ def packet_callback(packet): if args.sync: # sync packets - bind_sync_hdr(args.port) - packet_type = 'Sync' + bind_sync_hdr(args.syncport) + packet_type = f'Sync(port {args.syncport})' elif args.lbre: # lb+re packets bind_lb_hdr(args.port) - packet_type = 'LB+RE' + packet_type = f'LB+RE(port {args.port})' elif args.re: - bind_re_hdr([x + args.port for x in range(0, args.nports)]) - packet_type = 'RE' + # multiple ports possible + portList = [x + args.port for x in range(0, args.nports)] + bind_re_hdr(portList) + packet_type = f'RE(ports {portList})' + elif args.lbresync: + # sync packets + bind_sync_hdr(args.syncport) + # lb+re packets on a single port + bind_lb_hdr(args.port) + packet_type = f'LB+RE(port {args.port}) + Sync(port {args.syncport})' # Linux supports "any" shortcut interface specification, but other OSs may not, so we just list # all the interfaces first @@ -294,16 +317,23 @@ def packet_callback(packet): if args.sync: # sync packets - bind_sync_hdr(args.port) - packet_type = 'Sync' + bind_sync_hdr(args.syncport) + packet_type = f'Sync(port {args.syncport})' elif args.lbre: # lb+re packets bind_lb_hdr(args.port) - packet_type = 'LB+RE' + packet_type = f'LB+RE(port {args.port})' elif args.re: - #bind_re_hdr(args.port) - bind_re_hdr([x + args.port for x in range(0, args.nports)]) - packet_type = 'RE' + # multiple ports possible + portList = [x + args.port for x in range(0, args.nports)] + bind_re_hdr(portList) + packet_type = f'RE(ports {portList})' + elif args.lbresync: + # sync packets + bind_sync_hdr(args.syncport) + # lb+re packets on a single port + bind_lb_hdr(args.port) + packet_type = f'LB+RE(port {args.port}) + Sync(port {args.syncport})' print(f'Looking for {packet_type} packets in PCAP file {args.file}') try: From e155bbd36aae46c0ce6f9306fe064ceea69d9830 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Mon, 30 Sep 2024 10:15:40 -0400 Subject: [PATCH 58/73] Fixing typos --- scripts/scapy/snifgen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/scapy/snifgen.py b/scripts/scapy/snifgen.py index a9f003a1..a61e7fde 100755 --- a/scripts/scapy/snifgen.py +++ b/scripts/scapy/snifgen.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# this is a script for sniffing and generating various UDP packet formats withing EJ-FAT: +# this is a script for sniffing, parsing PCAP and generating various UDP packet formats within EJ-FAT: # - sync packets # - data packets with LB+RE header (pre-load-balancer) # - data packets with RE header only (post-load-balancer) From ab719094ded1c382f94b5667366c5ee5efe05ae8 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Mon, 30 Sep 2024 16:49:54 -0400 Subject: [PATCH 59/73] Added a lockfree queue to collect lost event IDs for reporting --- bin/e2sar_perf.cpp | 14 ++++++++++++++ include/e2sarDPReassembler.hpp | 23 +++++++++++++++++++++++ src/e2sarDPReassembler.cpp | 7 ------- test/e2sar_reas_live_test.cpp | 7 +++++++ test/e2sar_reas_test.cpp | 21 +++++++++++++++++++++ 5 files changed, 65 insertions(+), 7 deletions(-) diff --git a/bin/e2sar_perf.cpp b/bin/e2sar_perf.cpp index df466cf8..9a4bb328 100644 --- a/bin/e2sar_perf.cpp +++ b/bin/e2sar_perf.cpp @@ -232,11 +232,18 @@ result recvEvents(Reassembler &r, int durationSec) { void recvStatsThread(Reassembler *r) { + std::vector> lostEvents; + while(threadsRunning) { auto nowT = boost::chrono::high_resolution_clock::now(); auto stats = r->getStats(); + + for(auto res = r->get_LostEvent(); !res.has_error();) + { + lostEvents.push_back(res.value()); + } /* * - EventNum_t enqueueLoss; // number of events received and lost on enqueue * - EventNum_t eventSuccess; // events successfully processed @@ -255,6 +262,13 @@ void recvStatsThread(Reassembler *r) if (stats.get<5>() != E2SARErrorc::NoError) std::cout << "\tLast E2SARError code: " << stats.get<5>() << std::endl; + std::cout << "\tEvents lost so far: "; + for(auto evt: lostEvents) + { + std::cout << evt.first << ":" << evt.second << " "; + } + std::cout << std::endl; + auto until = nowT + boost::chrono::milliseconds(reportThreadSleepMs); boost::this_thread::sleep_until(until); } diff --git a/include/e2sarDPReassembler.hpp b/include/e2sarDPReassembler.hpp index 46779579..0c38e9e5 100644 --- a/include/e2sarDPReassembler.hpp +++ b/include/e2sarDPReassembler.hpp @@ -130,6 +130,8 @@ namespace e2sar std::atomic dataErrCnt{0}; // last e2sar error std::atomic lastE2SARError{E2SARErrorc::NoError}; + // a limited queue to push lost event numbers to + boost::lockfree::queue*> lostEventsQueue{20}; }; AtomicStats recvStats; @@ -237,12 +239,16 @@ namespace e2sar void _threadBody(); // log a lost event via a set of known lost + // and add to lost queue for external inspection inline void logLostEvent(std::pair evt) { if (lostEvents.contains(evt)) return; // this is thread-local lostEvents.insert(evt); + // lockfree queue (only takes trivial types) + std::pair *evtPtr = new std::pair(evt.first, evt.second); + reas.recvStats.lostEventsQueue.push(evtPtr); // this is atomic reas.recvStats.enqueueLoss++; } @@ -487,6 +493,23 @@ namespace e2sar recvStats.lastE2SARError); } + /** + * Try to pop an event number of a lost event from the queue that stores them + * @return result with either (eventNumber,dataId) or E2SARErrorc::NotFound if queue is empty + */ + inline result> get_LostEvent() noexcept + { + std::pair *res; + if (recvStats.lostEventsQueue.pop(res)) + { + auto ret = *res; + delete res; + return ret; + } + else + return E2SARErrorInfo{E2SARErrorc::NotFound, "Lost event queue is empty"}; + } + /** * Get the number of threads this Reassembler is using */ diff --git a/src/e2sarDPReassembler.cpp b/src/e2sarDPReassembler.cpp index dd5b9a2b..39537b05 100644 --- a/src/e2sarDPReassembler.cpp +++ b/src/e2sarDPReassembler.cpp @@ -312,13 +312,6 @@ namespace e2sar } } - std::cout << "The following events were lost: "; - for (const auto& elem :lostEvents) - { - std::cout << "(" << elem.first << ", " << elem.second << ") "; - } - std::cout << std::endl; - // close on exit auto res = _close(); } diff --git a/test/e2sar_reas_live_test.cpp b/test/e2sar_reas_live_test.cpp index f7eb6a5b..9513eabf 100644 --- a/test/e2sar_reas_live_test.cpp +++ b/test/e2sar_reas_live_test.cpp @@ -64,6 +64,13 @@ BOOST_AUTO_TEST_CASE(DPReasTest1) // data error count BOOST_CHECK(recvStats.get<4>() == 0); + auto lostEvent = reas.get_LostEvent(); + if (lostEvent.has_error()) + std::cout << "NO EVENT LOSS " << std::endl; + else + std::cout << "LOST EVENT " << lostEvent.value().first << ":" << lostEvent.value().second << std::endl; + BOOST_CHECK(lostEvent.has_error() && lostEvent.error().code() == E2SARErrorc::NotFound); + // stop threads and exit } diff --git a/test/e2sar_reas_test.cpp b/test/e2sar_reas_test.cpp index ad39c03c..32587f2b 100644 --- a/test/e2sar_reas_test.cpp +++ b/test/e2sar_reas_test.cpp @@ -136,6 +136,13 @@ BOOST_AUTO_TEST_CASE(DPReasTest1) BOOST_CHECK(recvStats.get<4>() == 0); // no data errors BOOST_CHECK(recvStats.get<5>() == E2SARErrorc::NoError); // no error + auto lostEvent = reas.get_LostEvent(); + if (lostEvent.has_error()) + std::cout << "NO EVENT LOSS " << std::endl; + else + std::cout << "LOST EVENT " << lostEvent.value().first << ":" << lostEvent.value().second << std::endl; + BOOST_CHECK(lostEvent.has_error() && lostEvent.error().code() == E2SARErrorc::NotFound); + // stop threads and exit } catch (E2SARException &ee) { @@ -276,6 +283,13 @@ BOOST_AUTO_TEST_CASE(DPReasTest2) BOOST_CHECK(recvStats.get<4>() == 0); // no data errors BOOST_CHECK(recvStats.get<5>() == E2SARErrorc::NoError); // no error + auto lostEvent = reas.get_LostEvent(); + if (lostEvent.has_error()) + std::cout << "NO EVENT LOSS " << std::endl; + else + std::cout << "LOST EVENT " << lostEvent.value().first << ":" << lostEvent.value().second << std::endl; + BOOST_CHECK(lostEvent.has_error() && lostEvent.error().code() == E2SARErrorc::NotFound); + // stop threads and exit } catch (E2SARException &ee) { @@ -613,6 +627,13 @@ BOOST_AUTO_TEST_CASE(DPReasTest4) BOOST_CHECK(recvStats.get<4>() == 0); // no data errors BOOST_CHECK(recvStats.get<5>() == E2SARErrorc::NoError); // no error + auto lostEvent = reas.get_LostEvent(); + if (lostEvent.has_error()) + std::cout << "NO EVENT LOSS " << std::endl; + else + std::cout << "LOST EVENT " << lostEvent.value().first << ":" << lostEvent.value().second << std::endl; + BOOST_CHECK(lostEvent.has_error() && lostEvent.error().code() == E2SARErrorc::NotFound); + // stop threads and exit } catch (E2SARException &ee) { From b0b8791d4a8dff9fe56d5c31e936436fdde7f77d Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Mon, 30 Sep 2024 17:33:29 -0400 Subject: [PATCH 60/73] Temporarily commenting out lost event stats --- bin/e2sar_perf.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/e2sar_perf.cpp b/bin/e2sar_perf.cpp index 9a4bb328..e2623574 100644 --- a/bin/e2sar_perf.cpp +++ b/bin/e2sar_perf.cpp @@ -240,10 +240,10 @@ void recvStatsThread(Reassembler *r) auto stats = r->getStats(); - for(auto res = r->get_LostEvent(); !res.has_error();) - { - lostEvents.push_back(res.value()); - } + //for(auto res = r->get_LostEvent(); !res.has_error();) + //{ + // lostEvents.push_back(res.value()); + //} /* * - EventNum_t enqueueLoss; // number of events received and lost on enqueue * - EventNum_t eventSuccess; // events successfully processed From 9d0e21baaa08d21c6a79b738732e13579c3fe35d Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Mon, 30 Sep 2024 18:08:12 -0400 Subject: [PATCH 61/73] Trying to resolve memory issue --- include/e2sarDPReassembler.hpp | 4 ++-- test/boost_test.cpp | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/include/e2sarDPReassembler.hpp b/include/e2sarDPReassembler.hpp index 0c38e9e5..7ea217c6 100644 --- a/include/e2sarDPReassembler.hpp +++ b/include/e2sarDPReassembler.hpp @@ -499,10 +499,10 @@ namespace e2sar */ inline result> get_LostEvent() noexcept { - std::pair *res; + std::pair *res = nullptr; if (recvStats.lostEventsQueue.pop(res)) { - auto ret = *res; + auto ret{*res}; delete res; return ret; } diff --git a/test/boost_test.cpp b/test/boost_test.cpp index 4650349f..9ea8342f 100644 --- a/test/boost_test.cpp +++ b/test/boost_test.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include @@ -330,5 +331,20 @@ int main() std::cout << "Head of buffer " << pidSampleBuffer.front() << std::endl; std::cout << "Tail of buffer " << pidSampleBuffer.back() << std::endl; + + std::cout << "Test allocate deallocate" << std::endl; + + boost::lockfree::queue*> lostEventsQueue{20}; + + std::pair *evtPtr = new std::pair(10, 120); + lostEventsQueue.push(evtPtr); + + std::pair *res; + while(lostEventsQueue.pop(res)) + { + auto ret = *res; + std::cout << "Retrieved " << ret.first << ":" << ret.second << std::endl; + delete res; + } } From a1e67cd6a4e59545db7fb0f4e497648fdc019aac Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Mon, 30 Sep 2024 18:17:15 -0400 Subject: [PATCH 62/73] Uncommenting the use of lost events --- bin/e2sar_perf.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/e2sar_perf.cpp b/bin/e2sar_perf.cpp index e2623574..9a4bb328 100644 --- a/bin/e2sar_perf.cpp +++ b/bin/e2sar_perf.cpp @@ -240,10 +240,10 @@ void recvStatsThread(Reassembler *r) auto stats = r->getStats(); - //for(auto res = r->get_LostEvent(); !res.has_error();) - //{ - // lostEvents.push_back(res.value()); - //} + for(auto res = r->get_LostEvent(); !res.has_error();) + { + lostEvents.push_back(res.value()); + } /* * - EventNum_t enqueueLoss; // number of events received and lost on enqueue * - EventNum_t eventSuccess; // events successfully processed From 67c9a78fef980f1ae91416c58f09cd92b6b02f6d Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Mon, 30 Sep 2024 21:27:56 -0400 Subject: [PATCH 63/73] Straightening out the memory bug --- bin/e2sar_perf.cpp | 5 ++++- test/boost_test.cpp | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/bin/e2sar_perf.cpp b/bin/e2sar_perf.cpp index 9a4bb328..7063a510 100644 --- a/bin/e2sar_perf.cpp +++ b/bin/e2sar_perf.cpp @@ -240,8 +240,11 @@ void recvStatsThread(Reassembler *r) auto stats = r->getStats(); - for(auto res = r->get_LostEvent(); !res.has_error();) + while(true) { + auto res = r->get_LostEvent(); + if (res.has_error()) + break; lostEvents.push_back(res.value()); } /* diff --git a/test/boost_test.cpp b/test/boost_test.cpp index 9ea8342f..3bfe7614 100644 --- a/test/boost_test.cpp +++ b/test/boost_test.cpp @@ -336,8 +336,10 @@ int main() boost::lockfree::queue*> lostEventsQueue{20}; - std::pair *evtPtr = new std::pair(10, 120); - lostEventsQueue.push(evtPtr); + for(int i=0; i<5; i++) + { + lostEventsQueue.push(new std::pair(i, i*10)); + } std::pair *res; while(lostEventsQueue.pop(res)) From df635129476d14f9ef76819a10fd953cbbb91924 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Mon, 30 Sep 2024 21:49:23 -0400 Subject: [PATCH 64/73] Cosmetic change --- bin/e2sar_perf.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/e2sar_perf.cpp b/bin/e2sar_perf.cpp index 7063a510..df0f802e 100644 --- a/bin/e2sar_perf.cpp +++ b/bin/e2sar_perf.cpp @@ -268,7 +268,7 @@ void recvStatsThread(Reassembler *r) std::cout << "\tEvents lost so far: "; for(auto evt: lostEvents) { - std::cout << evt.first << ":" << evt.second << " "; + std::cout << "<" << evt.first << ":" << evt.second << "> "; } std::cout << std::endl; From 787782e1ab3ae3b845b0cac509a1c15963e67575 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Mon, 30 Sep 2024 21:49:45 -0400 Subject: [PATCH 65/73] Updated live lb notebook for latest testing strategy --- .../EJFAT/E2SAR-live-lb-tester.ipynb | 130 +++++++++++++++--- 1 file changed, 110 insertions(+), 20 deletions(-) diff --git a/scripts/notebooks/EJFAT/E2SAR-live-lb-tester.ipynb b/scripts/notebooks/EJFAT/E2SAR-live-lb-tester.ipynb index d5d8afc2..1987d815 100644 --- a/scripts/notebooks/EJFAT/E2SAR-live-lb-tester.ipynb +++ b/scripts/notebooks/EJFAT/E2SAR-live-lb-tester.ipynb @@ -54,7 +54,7 @@ "#site_list_override = ['SRI', 'UCSD', 'CLEM']\n", "\n", "# (super)core sites - should be low loss\n", - "site_list_override = ['NEWY', 'WASH', 'LOSA', 'DALL', 'ATLA']\n", + "site_list_override = ['STAR', 'SALT', 'KANS', 'NEWY', 'WASH', 'LOSA', 'DALL', 'ATLA']\n", "\n", "# grouped around STAR with optical connections to the backbone - should be low loss\n", "#site_list_override = ['STAR', 'INDI', 'NCSA', 'MICH']\n", @@ -131,6 +131,7 @@ "#\n", "# Preamble\n", "import json\n", + "import time\n", "from datetime import datetime\n", "from datetime import timezone\n", "from datetime import timedelta\n", @@ -891,7 +892,7 @@ "\n", "### Run simple single-threaded performance test\n", "\n", - "Start Segmenter on Sender node and one Reassembler on Worker1 and test throughput without a real load balancer. Reassembler is told that LB header will be present and it ignores it." + "Start Segmenter on Sender node and one Reassembler on Worker1 and test throughput **without a real load balancer**. Reassembler is told that LB header will be present and it ignores it." ] }, { @@ -916,8 +917,6 @@ "metadata": {}, "outputs": [], "source": [ - "import time\n", - "\n", "# for e2sar_perf only the data= part of the query is meaningful. sync= must be present but is ignored\n", "# same for gRPC token, address and port (and lb id)\n", "e2sarPerfURI = f\"ejfat://useless@10.10.10.10:1234/lb/1?data={recver_addr}&sync=192.168.77.7:1234\"\n", @@ -950,17 +949,7 @@ "id": "46c04a61-eebd-4189-9848-1c787995919e", "metadata": {}, "source": [ - "### Run test on a live Load Balancer" - ] - }, - { - "cell_type": "markdown", - "id": "c60a87e0-0ecc-4e71-b728-fb358e349f04", - "metadata": {}, - "source": [ - "1. Reserve a new load balancer instance for a maximum 2 hours\n", - "2. Run the test (possibly multiple times)\n", - "3. Free the load balancer" + "### Reserve the Load Balancer" ] }, { @@ -971,7 +960,7 @@ "outputs": [], "source": [ "# Set the admin URI\n", - "ejfat_admin_uri = '' # cut and paste here\n", + "ejfat_admin_uri = 'ejfats://replace_me' # cut and paste here\n", "\n", "lb_path = './E2SAR/build/bin'\n", "ld_library_path = \"LD_LIBRARY_PATH=/usr/local/lib\"\n", @@ -1007,7 +996,7 @@ "lbname = 'e2sar-testlb'\n", "duration = '02:00:00' # 2 hours\n", "\n", - "command = f\"{lbadm} --reserve -l {lbname} -a '{sender_addr}' -d {duration} -u '{ejfat_admin_uri}'\"\n", + "command = f\"{lbadm} --reserve -l {lbname} -a '{sender_addr}' -d {duration} -u '{ejfat_admin_uri}' -e\"\n", "execute_commands(sender, [command])" ] }, @@ -1019,7 +1008,7 @@ "outputs": [], "source": [ "# copy the 'Updated URI after reserve with instance token' from the above result here:\n", - "instance_uri = ''" + "instance_uri = 'ejfats://replace_me'" ] }, { @@ -1034,12 +1023,38 @@ "execute_commands(sender, [command])" ] }, + { + "cell_type": "markdown", + "id": "95e0c89c-73e0-4d2c-a963-9ba405ba255b", + "metadata": {}, + "source": [ + "### Run a test with real load balancer and single sender node and single receiver node" + ] + }, { "cell_type": "code", "execution_count": null, - "id": "a206ef22-bb58-48ed-a1e8-9673e0a7d3b5", + "id": "1a0e762e-3945-4b74-8a88-91d49b16eaee", "metadata": {}, "outputs": [], + "source": [ + "# select sender and receiver\n", + "sender = list(filter(lambda n: n.get_name()[0:6] == \"Sender\", slice.get_nodes()))[0]\n", + "recver = list(filter(lambda n: n.get_name()[0:7] == \"Worker1\", slice.get_nodes()))[0]\n", + "\n", + "sender_addr = sender.get_interface(network_name=make_net_name(sender.get_site())).get_ip_addr()\n", + "recver_addr = recver.get_interface(network_name=make_net_name(recver.get_site())).get_ip_addr()\n", + "print(f\"Sender sending from {sender_addr}, receiver receiving on {recver_addr}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a206ef22-bb58-48ed-a1e8-9673e0a7d3b5", + "metadata": { + "scrolled": true + }, + "outputs": [], "source": [ "# run sender and receiver through the reserved LB\n", "recverDuration = 40\n", @@ -1053,7 +1068,7 @@ "# Given that in FABRIC ejfat-lb.es.net resolves to IP6 first and gRPC C++ library doesn't\n", "# offer granular control over which resolved address is used, we use -4 option to tell the\n", "# code to use the IPv4 address, but this also disables cert validation.\n", - "recv_command = f\"{e2sar_perf} -r -u '{instance_uri}' -d {recverDuration} -b {bufSize} --ip {recver_addr} --port 10000 --withcp -4 -t 4\"\n", + "recv_command = f\"{e2sar_perf} -r -u '{instance_uri}' -d {recverDuration} -b {bufSize} --ip {recver_addr} --port 10000 --withcp -4 --threads 4\"\n", "send_command = f\"{e2sar_perf} -s -u '{instance_uri}' --mtu {mtu} --rate {rate} --length {length} -n {numEvents} -b {bufSize} --ip {sender_addr} --withcp -4\"\n", "\n", "# start the receiver for n seconds and log its output\n", @@ -1070,6 +1085,81 @@ "print(f\"Inspect {recver.get_name()}.perf.log file to see the results. It should indicate how many events were received and how many lost.\")" ] }, + { + "cell_type": "markdown", + "id": "2842c38c-baac-4ad9-8caa-9fb994a92b6d", + "metadata": {}, + "source": [ + "### Run the test with a single sender node and multiple receiver nodes" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7477f90a-864b-43ef-ab66-16c379fb26de", + "metadata": {}, + "outputs": [], + "source": [ + "# select sender and receivers\n", + "sender = list(filter(lambda n: n.get_name()[0:6] == \"Sender\", slice.get_nodes()))[0]\n", + "\n", + "worker_index = 1\n", + "recvers = list()\n", + "recver_addrs = list()\n", + "while True:\n", + " matches = list(filter(lambda n: n.get_name()[0:7] == f\"Worker{worker_index}\", slice.get_nodes()))\n", + " if len(matches) == 0:\n", + " break\n", + " recver = matches[0]\n", + " recvers.append(recver)\n", + " recver_addr = recver.get_interface(network_name=make_net_name(recver.get_site())).get_ip_addr()\n", + " recver_addrs.append(recver_addr)\n", + " worker_index += 1\n", + " \n", + "sender_addr = sender.get_interface(network_name=make_net_name(sender.get_site())).get_ip_addr()\n", + "print(f\"Sender sending from {sender_addr}, receivers receiving on:\")\n", + "for recver, recver_addr in zip(recvers, recver_addrs):\n", + " print(\"\\t\" + recver.get_name() + \": \" + recver_addr)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8266a685-437f-4e13-8603-1bd550e900c4", + "metadata": {}, + "outputs": [], + "source": [ + "# run sender and receivers through the reserved LB\n", + "recverDuration = 60\n", + "mtu = 9000\n", + "rate = 15 # Gbps\n", + "length = 1000000 # event length in bytes\n", + "numEvents = 20000 # number of events to send\n", + "bufSize = 300 * 1024 * 1024 # 100MB send and receive buffers\n", + "recvThreads = 4 # on each receiver\n", + "numSocks = 4 # number send sockets in sender\n", + "startPort = 10000 # receiver starting port\n", + "\n", + "# Given that in FABRIC ejfat-lb.es.net resolves to IP6 first and gRPC C++ library doesn't\n", + "# offer granular control over which resolved address is used, we use -4 option to tell the\n", + "# code to use the IPv4 address, but this also disables cert validation.\n", + "send_command = f\"{e2sar_perf} -s -u '{instance_uri}' --mtu {mtu} --rate {rate} --length {length} -n {numEvents} -b {bufSize} --ip {sender_addr} --sockets {numSocks} --withcp -4\"\n", + "\n", + "for recver, recver_addr in zip(recvers, recver_addrs):\n", + " recv_command = f\"{e2sar_perf} -r -u '{instance_uri}' -d {recverDuration} -b {bufSize} --ip {recver_addr} --port {startPort} --withcp -4 --threads {recvThreads}\"\n", + " print(f'Executing command {recv_command} on receiver {recver.get_name()}')\n", + " recver.execute_thread(recv_command, output_file=f\"{recver.get_name()}.perf.log\")\n", + "\n", + "# sleep 5 seconds to let receivers get going\n", + "time.sleep(5)\n", + "\n", + "# start the sender in the foreground\n", + "print(f'Executing command {send_command} on sender')\n", + "stdout_send, stderr_send = sender.execute(send_command, output_file=f\"{sender.get_name()}.perf.log\")\n", + "\n", + "print(f\"Inspect WorkerX.perf.log files to see the results. They should indicate how many events were received and how many lost.\")" + ] + }, { "cell_type": "code", "execution_count": null, From 1a98f393fa57b3f3f53fcf02bb01c9c0e843c929 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Mon, 30 Sep 2024 21:50:57 -0400 Subject: [PATCH 66/73] Updated docs --- docs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs b/docs index 296501b2..cc336b83 160000 --- a/docs +++ b/docs @@ -1 +1 @@ -Subproject commit 296501b217b8ca7e3e7ed77d80b6a8da8738c5dc +Subproject commit cc336b8346707de63fb93017ecfd43abb500789b From 7020959a893da263fc105d3180eb5f388575b466 Mon Sep 17 00:00:00 2001 From: Ilya Baldin Date: Mon, 30 Sep 2024 22:02:25 -0400 Subject: [PATCH 67/73] Updated wiki --- wiki | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wiki b/wiki index 52a67848..e34dc9a5 160000 --- a/wiki +++ b/wiki @@ -1 +1 @@ -Subproject commit 52a6784808f7603ccbdb8ed6ab091a0ac805a4ff +Subproject commit e34dc9a5167c301d3ad3efc9d800a4b1b8693c08 From 78c8c6f3db76fb751a32b06a8895b0b5857930c8 Mon Sep 17 00:00:00 2001 From: Xinxin Mei Date: Tue, 1 Oct 2024 16:44:19 +0000 Subject: [PATCH 68/73] More Pybind updates --- src/pybind/py_e2sarCP.cpp | 4 ++-- src/pybind/py_e2sarDP.cpp | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/pybind/py_e2sarCP.cpp b/src/pybind/py_e2sarCP.cpp index 4d229387..5c802d6f 100644 --- a/src/pybind/py_e2sarCP.cpp +++ b/src/pybind/py_e2sarCP.cpp @@ -294,8 +294,8 @@ void init_e2sarCP(py::module_ &m) { // Return an EjfatURI object. lb_manager.def("get_uri", &LBManager::get_URI, py::return_value_policy::reference); - // return connect string - lb_manager.def("get_addrstring", &LBManager::get_AddrString, py::return_value_policy::reference); + // Return connect string. + lb_manager.def("get_addr_string", &LBManager::get_AddrString); /// NOTE: donot need to bind LBManager::makeSslOptionsFromFiles } diff --git a/src/pybind/py_e2sarDP.cpp b/src/pybind/py_e2sarDP.cpp index c99a672a..38e4f088 100644 --- a/src/pybind/py_e2sarDP.cpp +++ b/src/pybind/py_e2sarDP.cpp @@ -31,7 +31,7 @@ void print_type(const T& param) { namespace py = pybind11; using namespace e2sar; -// Has to have a wrapper because of the callback function. +// Must have a wrapper because of the callback function. result addToSendQueueWrapper(Segmenter& seg, uint8_t *event, size_t bytes, int64_t _eventNum, uint16_t _dataId, uint16_t entropy, std::function callback, @@ -182,13 +182,23 @@ void init_e2sarDP_reassembler(py::module_ &m) reas.def( py::init(), "Init the Reassembler object with number of recv threads.", - py::arg("uri"), // must-have arg when init + py::arg("uri"), // must-have args when init py::arg("data_ip"), py::arg("starting_port"), py::arg("num_recv_threads") = (size_t)1, py::arg("rflags") = Reassembler::ReassemblerFlags()); - // Recv events part. return py::tuple. + // Constructor with CPU core list. + reas.def( + py::init, const Reassembler::ReassemblerFlags &>(), + "Init the Reassembler object with a list of CPU cores.", + py::arg("uri"), // must-have args when init + py::arg("data_ip"), + py::arg("starting_port"), + py::arg("cpu_core_list"), + py::arg("rflags") = Reassembler::ReassemblerFlags()); + + // Recv events part. Return py::tuple. reas.def("getEvent", [](Reassembler& self, /* py::list is mutable */ py::list& recv_bytes ) -> py::tuple { From fd6c842c9a330a8efbb3e9ffc70963641f4ece18 Mon Sep 17 00:00:00 2001 From: Xinxin Mei Date: Tue, 1 Oct 2024 18:35:55 +0000 Subject: [PATCH 69/73] Bind new members of Reassembler --- src/pybind/py_e2sar.cpp | 3 ++- src/pybind/py_e2sarDP.cpp | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/pybind/py_e2sar.cpp b/src/pybind/py_e2sar.cpp index 78326e65..8a4aa218 100644 --- a/src/pybind/py_e2sar.cpp +++ b/src/pybind/py_e2sar.cpp @@ -129,9 +129,10 @@ void init_e2sarResultTypes(py::module_ &m) bind_result(m, "E2SARResultString"); bind_result(m, "E2SARResultEjfatURI"); bind_result(m, "E2SARResultSslCredentialsOptions"); - bind_result(m, "E2SARResultUInit32"); + bind_result(m, "E2SARResultUInt32"); bind_result>(m, "E2SARResultPairIP"); bind_result>(m, "E2SARResultPairString"); + bind_result>(m, "E2SARResultPairUInt64"); bind_result(m, "E2SARResultReassemblerFlags"); bind_result(m, "E2SARResultSegmenterFlags"); } diff --git a/src/pybind/py_e2sarDP.cpp b/src/pybind/py_e2sarDP.cpp index 38e4f088..ab330282 100644 --- a/src/pybind/py_e2sarDP.cpp +++ b/src/pybind/py_e2sarDP.cpp @@ -170,6 +170,9 @@ void init_e2sarDP_reassembler(py::module_ &m) .def_readwrite("Ki", &Reassembler::ReassemblerFlags::Ki) .def_readwrite("Kp", &Reassembler::ReassemblerFlags::Kp) .def_readwrite("Kd", &Reassembler::ReassemblerFlags::Kd) + .def_readwrite("weight", &Reassembler::ReassemblerFlags::weight) + .def_readwrite("min_factor", &Reassembler::ReassemblerFlags::min_factor) + .def_readwrite("max_factor", &Reassembler::ReassemblerFlags::max_factor) .def_readwrite("setPoint", &Reassembler::ReassemblerFlags::setPoint) .def_readwrite("epoch_ms", &Reassembler::ReassemblerFlags::epoch_ms) .def_readwrite("portRange", &Reassembler::ReassemblerFlags::portRange) @@ -268,10 +271,12 @@ void init_e2sarDP_reassembler(py::module_ &m) // Return type of result reas.def("OpenAndStart", &Reassembler::openAndStart); - /// TODO: to be test reas.def("registerWorker", &Reassembler::registerWorker); reas.def("deregisterWorker", &Reassembler::deregisterWorker); + // Return type of resultresult> + reas.def("get_LostEvent", &Reassembler::get_LostEvent); + // Return type of boost::tuple<>: convert to std::tuple reas.def("getStats", [](const Reassembler& reasObj) { auto stats = reasObj.getStats(); From 4934820b667658b7eb5faf29caf6fc0308d0f846 Mon Sep 17 00:00:00 2001 From: Xinxin Mei Date: Tue, 1 Oct 2024 18:58:10 +0000 Subject: [PATCH 70/73] Update the first DP Python example --- .../pybind11_examples/example_DataPlane.ipynb | 366 ++++-------------- 1 file changed, 71 insertions(+), 295 deletions(-) diff --git a/scripts/notebooks/pybind11_examples/example_DataPlane.ipynb b/scripts/notebooks/pybind11_examples/example_DataPlane.ipynb index 3177e0fe..e7ecbdc9 100644 --- a/scripts/notebooks/pybind11_examples/example_DataPlane.ipynb +++ b/scripts/notebooks/pybind11_examples/example_DataPlane.ipynb @@ -45,8 +45,8 @@ "source": [ "DP_IPV4_ADDR = \"127.0.0.1\"\n", "DP_IPV4_PORT = 10000\n", - "data_id = 0x0505 # decimal value: 1085\n", - "eventSrc_id = 0x11223344 # decimal value: 287454020" + "DATA_ID = 0x0505 # decimal value: 1085\n", + "EVENTSRC_ID = 0x11223344 # decimal value: 287454020" ] }, { @@ -107,7 +107,7 @@ "outputs": [], "source": [ "# Init segmenter object\n", - "seg = e2sar_py.DataPlane.Segmenter(seg_uri, data_id, eventSrc_id, sflags)" + "seg = e2sar_py.DataPlane.Segmenter(seg_uri, DATA_ID, EVENTSRC_ID, sflags)" ] }, { @@ -164,7 +164,7 @@ " epoch_ms = 1000\n", " setPoint = 0.0\n", " [Kp, Ki, Kd] = [0.0, 0.0, 0.0]\n", - " cpV6 = False\n", + " [weight, min_factor, max_factor] = [1.0, 0.5, 2.0]\n", " portRange = -1\n", " withLBHeader = True\n" ] @@ -204,7 +204,7 @@ "print(f\" epoch_ms = {rflags.epoch_ms}\")\n", "print(f\" setPoint = {rflags.setPoint}\")\n", "print(f\" [Kp, Ki, Kd] = [{rflags.Kp}, {rflags.Ki}, {rflags.Kd}]\")\n", - "print(f\" cpV6 = {rflags.cpV6}\")\n", + "print(f\" [weight, min_factor, max_factor] = [{rflags.weight}, {rflags.min_factor}, {rflags.max_factor}]\")\n", "print(f\" portRange = {rflags.portRange}\")\n", "print(f\" withLBHeader = {rflags.withLBHeader}\")" ] @@ -363,38 +363,70 @@ " continue\n", " recv = recv_bytes_list[0].decode('utf-8')\n", " print(f\" recv_buf:\\t {recv}\")\n", - " assert(recv_data_id == data_id)\n", + " assert(recv_data_id == DATA_ID)\n", " print(f\" bufLen:\\t {recv_event_len}\")\n", " print(f\" eventNum:\\t {recv_event_num}\")\n", " print(f\" dataId:\\t {recv_data_id}\")\n" ] }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "reas_status = reas.getStats()\n", + "assert(reas_status[0] == 0) # no losses\n", + "assert(reas_status[1] == 5) # all succeeded\n", + "assert(reas_status[2] == 0) # no errno\n", + "assert(reas_status[3] == 0) # no grpc errors\n", + "assert(reas_status[4] == 0) # no data errors\n", + "assert(reas_status[5] == e2sar_py.E2SARErrorc.NoError) # no error" + ] + }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## DPReasTest2\n", - "\n", - "Test segmentation and reassembly on local host with no control plane (basic segmentation) using small MTU." + "Check if there is any lost event." ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Segmenter flags:\n", - " syncPeriodMs=1000\n", - " useCP=False\n", - " mtu=80\n", - " syncPeriods=5\n" + "NO EVENT LOSS\n" ] } ], + "source": [ + "if reas.get_LostEvent().has_error():\n", + " print(\"NO EVENT LOSS\")\n", + "else:\n", + " print(f\"LOST EVENT: ({reas.get_LostEvent().value()[0]}, {reas.get_LostEvent().value()[1]})\")\n", + "\n", + "assert(reas.get_LostEvent().error().code == e2sar_py.E2SARErrorc.NotFound)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DPReasTest2\n", + "\n", + "Test segmentation and reassembly on local host with no control plane (basic segmentation) using small MTU." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# Set up sflags\n", "sflags2 = e2sar_py.DataPlane.SegmenterFlags()\n", @@ -445,22 +477,9 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Sending data... Rd 0\n", - "Sending data... Rd 1\n", - "Sending data... Rd 2\n", - "Sending data... Rd 3\n", - "Sending data... Rd 4\n", - "Sent 25 data frames\n" - ] - } - ], + "outputs": [], "source": [ "# Send data with the small MTU\n", "for i in range(5):\n", @@ -486,41 +505,9 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Receiving data... Rd 0\n", - " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE WE WANT TO SEND EVERY 1 SECOND.\n", - " bufLen:\t 65\n", - " eventNum:\t 0\n", - " dataId:\t 1285\n", - "Receiving data... Rd 1\n", - " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE WE WANT TO SEND EVERY 1 SECOND.\n", - " bufLen:\t 65\n", - " eventNum:\t 1\n", - " dataId:\t 1285\n", - "Receiving data... Rd 2\n", - " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE WE WANT TO SEND EVERY 1 SECOND.\n", - " bufLen:\t 65\n", - " eventNum:\t 2\n", - " dataId:\t 1285\n", - "Receiving data... Rd 3\n", - " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE WE WANT TO SEND EVERY 1 SECOND.\n", - " bufLen:\t 65\n", - " eventNum:\t 3\n", - " dataId:\t 1285\n", - "Receiving data... Rd 4\n", - " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE WE WANT TO SEND EVERY 1 SECOND.\n", - " bufLen:\t 65\n", - " eventNum:\t 4\n", - " dataId:\t 1285\n" - ] - } - ], + "outputs": [], "source": [ "# Prepare Python list to hold the output data\n", "recv_bytes_list = [None] # Placeholder for the byte buffer\n", @@ -551,20 +538,9 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "This reassembler has\n", - " 1 threads;\n", - " listening on ports 19522:19522;\n", - " using portRange: 0.\n" - ] - } - ], + "outputs": [], "source": [ "# Create reassembler with 1 recv thread\n", "\n", @@ -584,20 +560,9 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "This reassembler has\n", - " 4 threads;\n", - " listening on ports 19522:19525;\n", - " using portRange: 2.\n" - ] - } - ], + "outputs": [], "source": [ "# Create reassembler with 4 recv threads\n", "reas_b = e2sar_py.DataPlane.Reassembler(\n", @@ -615,20 +580,9 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "This reassembler has\n", - " 7 threads;\n", - " listening on ports 19522:19529;\n", - " using portRange: 3.\n" - ] - } - ], + "outputs": [], "source": [ "# Create reassembler with 7 recv threads\n", "reas_c = e2sar_py.DataPlane.Reassembler(\n", @@ -646,20 +600,9 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "This reassembler has\n", - " 4 threads;\n", - " listening on ports 19522:20545;\n", - " using portRange: 10.\n" - ] - } - ], + "outputs": [], "source": [ "# 4 threads with portRange override\n", "rflags_m.portRange = 10\n", @@ -679,20 +622,9 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "This reassembler has\n", - " 4 threads;\n", - " listening on ports 19522:19523;\n", - " using portRange: 1.\n" - ] - } - ], + "outputs": [], "source": [ "# 4 threads with low portRange override\n", "rflags_m.portRange = 1\n", @@ -721,24 +653,9 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Creating Segmenter with uri:\n", - " ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data=127.0.0.1:19522\n", - "Creating Segmenter with uri:\n", - " ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data=127.0.0.1:19523\n", - "Creating Segmenter with uri:\n", - " ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data=127.0.0.1:19524\n", - "Creating Segmenter with uri:\n", - " ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data=127.0.0.1:19525\n" - ] - } - ], + "outputs": [], "source": [ "# Create 4 segmenter objects\n", "\n", @@ -773,20 +690,9 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "This reassembler has\n", - " 1 threads;\n", - " listening on ports 19522:19525;\n", - " using portRange: 2.\n" - ] - } - ], + "outputs": [], "source": [ "# Create reassembler with no control plane. 1 recv thread listening on 4 ports.\n", "\n", @@ -805,20 +711,9 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Seg1.openAndStart()\n", - "Seg2.openAndStart()\n", - "Seg3.openAndStart()\n", - "Seg4.openAndStart()\n" - ] - } - ], + "outputs": [], "source": [ "# Start the 4 segmenters\n", "for i in range(4):\n", @@ -857,20 +752,9 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Segmenter 1 sent 5 data frames\n", - "Segmenter 2 sent 5 data frames\n", - "Segmenter 3 sent 5 data frames\n", - "Segmenter 4 sent 5 data frames\n" - ] - } - ], + "outputs": [], "source": [ "def get_send_str(idx):\n", " prefix_str = \"THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER\"\n", @@ -905,117 +789,9 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Receiving data... Rd 0\n", - " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER0 WE WANT TO SEND EVERY 1 MILLISECOND.\n", - " bufLen:\t 86\n", - " eventNum:\t 1000\n", - " dataId:\t 1285\n", - "Receiving data... Rd 1\n", - " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER1 WE WANT TO SEND EVERY 1 MILLISECOND.\n", - " bufLen:\t 86\n", - " eventNum:\t 2000\n", - " dataId:\t 1285\n", - "Receiving data... Rd 2\n", - " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER2 WE WANT TO SEND EVERY 1 MILLISECOND.\n", - " bufLen:\t 86\n", - " eventNum:\t 3000\n", - " dataId:\t 1285\n", - "Receiving data... Rd 3\n", - " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER3 WE WANT TO SEND EVERY 1 MILLISECOND.\n", - " bufLen:\t 86\n", - " eventNum:\t 4000\n", - " dataId:\t 1285\n", - "Receiving data... Rd 4\n", - " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER0 WE WANT TO SEND EVERY 1 MILLISECOND.\n", - " bufLen:\t 86\n", - " eventNum:\t 1001\n", - " dataId:\t 1285\n", - "Receiving data... Rd 5\n", - " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER1 WE WANT TO SEND EVERY 1 MILLISECOND.\n", - " bufLen:\t 86\n", - " eventNum:\t 2001\n", - " dataId:\t 1285\n", - "Receiving data... Rd 6\n", - " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER2 WE WANT TO SEND EVERY 1 MILLISECOND.\n", - " bufLen:\t 86\n", - " eventNum:\t 3001\n", - " dataId:\t 1285\n", - "Receiving data... Rd 7\n", - " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER3 WE WANT TO SEND EVERY 1 MILLISECOND.\n", - " bufLen:\t 86\n", - " eventNum:\t 4001\n", - " dataId:\t 1285\n", - "Receiving data... Rd 8\n", - " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER0 WE WANT TO SEND EVERY 1 MILLISECOND.\n", - " bufLen:\t 86\n", - " eventNum:\t 1002\n", - " dataId:\t 1285\n", - "Receiving data... Rd 9\n", - " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER1 WE WANT TO SEND EVERY 1 MILLISECOND.\n", - " bufLen:\t 86\n", - " eventNum:\t 2002\n", - " dataId:\t 1285\n", - "Receiving data... Rd 10\n", - " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER2 WE WANT TO SEND EVERY 1 MILLISECOND.\n", - " bufLen:\t 86\n", - " eventNum:\t 3002\n", - " dataId:\t 1285\n", - "Receiving data... Rd 11\n", - " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER3 WE WANT TO SEND EVERY 1 MILLISECOND.\n", - " bufLen:\t 86\n", - " eventNum:\t 4002\n", - " dataId:\t 1285\n", - "Receiving data... Rd 12\n", - " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER0 WE WANT TO SEND EVERY 1 MILLISECOND.\n", - " bufLen:\t 86\n", - " eventNum:\t 1003\n", - " dataId:\t 1285\n", - "Receiving data... Rd 13\n", - " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER1 WE WANT TO SEND EVERY 1 MILLISECOND.\n", - " bufLen:\t 86\n", - " eventNum:\t 2003\n", - " dataId:\t 1285\n", - "Receiving data... Rd 14\n", - " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER2 WE WANT TO SEND EVERY 1 MILLISECOND.\n", - " bufLen:\t 86\n", - " eventNum:\t 3003\n", - " dataId:\t 1285\n", - "Receiving data... Rd 15\n", - " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER3 WE WANT TO SEND EVERY 1 MILLISECOND.\n", - " bufLen:\t 86\n", - " eventNum:\t 4003\n", - " dataId:\t 1285\n", - "Receiving data... Rd 16\n", - " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER0 WE WANT TO SEND EVERY 1 MILLISECOND.\n", - " bufLen:\t 86\n", - " eventNum:\t 1004\n", - " dataId:\t 1285\n", - "Receiving data... Rd 17\n", - " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER1 WE WANT TO SEND EVERY 1 MILLISECOND.\n", - " bufLen:\t 86\n", - " eventNum:\t 2004\n", - " dataId:\t 1285\n", - "Receiving data... Rd 18\n", - " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER2 WE WANT TO SEND EVERY 1 MILLISECOND.\n", - " bufLen:\t 86\n", - " eventNum:\t 3004\n", - " dataId:\t 1285\n", - "Receiving data... Rd 19\n", - " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER3 WE WANT TO SEND EVERY 1 MILLISECOND.\n", - " bufLen:\t 86\n", - " eventNum:\t 4004\n", - " dataId:\t 1285\n", - "Reassembler stats: (0, 20, 0, 0, 0, )\n" - ] - } - ], + "outputs": [], "source": [ "# Prepare Python list to hold the output data\n", "recv_bytes_list = [None] # Placeholder for the byte buffer\n", From 75c01f414aff96d5d164e4b61938310a016ce7b8 Mon Sep 17 00:00:00 2001 From: Xinxin Mei Date: Tue, 1 Oct 2024 19:07:35 +0000 Subject: [PATCH 71/73] More updates to the DP Python example --- .../pybind11_examples/example_DataPlane.ipynb | 112 ++++++++++++++++-- 1 file changed, 102 insertions(+), 10 deletions(-) diff --git a/scripts/notebooks/pybind11_examples/example_DataPlane.ipynb b/scripts/notebooks/pybind11_examples/example_DataPlane.ipynb index e7ecbdc9..d3eb2fc4 100644 --- a/scripts/notebooks/pybind11_examples/example_DataPlane.ipynb +++ b/scripts/notebooks/pybind11_examples/example_DataPlane.ipynb @@ -424,9 +424,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Segmenter flags:\n", + " syncPeriodMs=1000\n", + " useCP=False\n", + " mtu=80\n", + " syncPeriods=5\n" + ] + } + ], "source": [ "# Set up sflags\n", "sflags2 = e2sar_py.DataPlane.SegmenterFlags()\n", @@ -449,7 +461,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -459,12 +471,12 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ "# Initialize a new Segmenter objects\n", - "seg2 = e2sar_py.DataPlane.Segmenter(seg_uri2, data_id, eventSrc_id, sflags2)\n", + "seg2 = e2sar_py.DataPlane.Segmenter(seg_uri2, DATA_ID, EVENTSRC_ID, sflags2)\n", "\n", "res = seg2.OpenAndStart()\n", "assert(res.value() == 0)\n", @@ -477,9 +489,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Sending data... Rd 0\n", + "Sending data... Rd 1\n", + "Sending data... Rd 2\n", + "Sending data... Rd 3\n", + "Sending data... Rd 4\n", + "Sent 25 data frames\n" + ] + } + ], "source": [ "# Send data with the small MTU\n", "for i in range(5):\n", @@ -505,9 +530,41 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Receiving data... Rd 0\n", + " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE WE WANT TO SEND EVERY 1 SECOND.\n", + " bufLen:\t 65\n", + " eventNum:\t 0\n", + " dataId:\t 1285\n", + "Receiving data... Rd 1\n", + " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE WE WANT TO SEND EVERY 1 SECOND.\n", + " bufLen:\t 65\n", + " eventNum:\t 1\n", + " dataId:\t 1285\n", + "Receiving data... Rd 2\n", + " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE WE WANT TO SEND EVERY 1 SECOND.\n", + " bufLen:\t 65\n", + " eventNum:\t 2\n", + " dataId:\t 1285\n", + "Receiving data... Rd 3\n", + " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE WE WANT TO SEND EVERY 1 SECOND.\n", + " bufLen:\t 65\n", + " eventNum:\t 3\n", + " dataId:\t 1285\n", + "Receiving data... Rd 4\n", + " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE WE WANT TO SEND EVERY 1 SECOND.\n", + " bufLen:\t 65\n", + " eventNum:\t 4\n", + " dataId:\t 1285\n" + ] + } + ], "source": [ "# Prepare Python list to hold the output data\n", "recv_bytes_list = [None] # Placeholder for the byte buffer\n", @@ -521,12 +578,47 @@ " continue\n", " recv = recv_bytes_list[0].decode('utf-8')\n", " print(f\" recv_buf:\\t {recv}\")\n", - " assert(recv_data_id == data_id)\n", + " assert(recv_data_id == DATA_ID)\n", " print(f\" bufLen:\\t {recv_event_len}\")\n", " print(f\" eventNum:\\t {recv_event_num}\")\n", " print(f\" dataId:\\t {recv_data_id}\")" ] }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "(0, 10, 0, 0, 0, )\n", + "NO EVENT LOSS\n" + ] + } + ], + "source": [ + "# Validation\n", + "reas_status = reas.getStats()\n", + "print(reas_status)\n", + "\n", + "assert(reas_status[0] == 0) # no losses\n", + "# assert(reas_status[1] == 5) # NOTE: this number is accumulated. So it's not always 5. \n", + "assert(reas_status[2] == 0) # no errno\n", + "assert(reas_status[3] == 0) # no grpc errors\n", + "assert(reas_status[4] == 0) # no data errors\n", + "assert(reas_status[5] == e2sar_py.E2SARErrorc.NoError) # no error\n", + "\n", + "\n", + "if reas.get_LostEvent().has_error():\n", + " print(\"NO EVENT LOSS\")\n", + "else:\n", + " print(f\"LOST EVENT: ({reas.get_LostEvent().value()[0]}, {reas.get_LostEvent().value()[1]})\")\n", + "\n", + "assert(reas.get_LostEvent().error().code == e2sar_py.E2SARErrorc.NotFound)" + ] + }, { "cell_type": "markdown", "metadata": {}, From 23326b7a7bb21c1bfa0680f51efde772d07672ef Mon Sep 17 00:00:00 2001 From: Xinxin Mei Date: Tue, 1 Oct 2024 21:27:43 +0000 Subject: [PATCH 72/73] DP Python example works --- .../pybind11_examples/example_DataPlane.ipynb | 388 ++++++++++++++++-- 1 file changed, 364 insertions(+), 24 deletions(-) diff --git a/scripts/notebooks/pybind11_examples/example_DataPlane.ipynb b/scripts/notebooks/pybind11_examples/example_DataPlane.ipynb index d3eb2fc4..ab1ce69c 100644 --- a/scripts/notebooks/pybind11_examples/example_DataPlane.ipynb +++ b/scripts/notebooks/pybind11_examples/example_DataPlane.ipynb @@ -586,7 +586,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -630,9 +630,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "This reassembler has\n", + " 1 threads;\n", + " listening on ports 19522:19522;\n", + " using portRange: 0.\n" + ] + } + ], "source": [ "# Create reassembler with 1 recv thread\n", "\n", @@ -652,9 +663,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "This reassembler has\n", + " 4 threads;\n", + " listening on ports 19522:19525;\n", + " using portRange: 2.\n" + ] + } + ], "source": [ "# Create reassembler with 4 recv threads\n", "reas_b = e2sar_py.DataPlane.Reassembler(\n", @@ -672,9 +694,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "This reassembler has\n", + " 7 threads;\n", + " listening on ports 19522:19529;\n", + " using portRange: 3.\n" + ] + } + ], "source": [ "# Create reassembler with 7 recv threads\n", "reas_c = e2sar_py.DataPlane.Reassembler(\n", @@ -692,9 +725,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "This reassembler has\n", + " 4 threads;\n", + " listening on ports 19522:20545;\n", + " using portRange: 10.\n" + ] + } + ], "source": [ "# 4 threads with portRange override\n", "rflags_m.portRange = 10\n", @@ -714,9 +758,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "This reassembler has\n", + " 4 threads;\n", + " listening on ports 19522:19523;\n", + " using portRange: 1.\n" + ] + } + ], "source": [ "# 4 threads with low portRange override\n", "rflags_m.portRange = 1\n", @@ -745,9 +800,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating Segmenter with uri:\n", + " ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data=127.0.0.1:19522\n", + "Creating Segmenter with uri:\n", + " ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data=127.0.0.1:19523\n", + "Creating Segmenter with uri:\n", + " ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data=127.0.0.1:19524\n", + "Creating Segmenter with uri:\n", + " ejfat://useless@192.168.100.1:9876/lb/1?sync=192.168.0.1:12345&data=127.0.0.1:19525\n" + ] + } + ], "source": [ "# Create 4 segmenter objects\n", "\n", @@ -775,16 +845,27 @@ "seg_obj_list = []\n", "for i in range(4):\n", " seg_obj_list.append(create_seg_obj(\n", - " i, data_id, eventSrc_id, sflags))\n", + " i, DATA_ID, EVENTSRC_ID, sflags))\n", "assert len(seg_obj_list) == 4\n", "\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "This reassembler has\n", + " 1 threads;\n", + " listening on ports 19522:19525;\n", + " using portRange: 2.\n" + ] + } + ], "source": [ "# Create reassembler with no control plane. 1 recv thread listening on 4 ports.\n", "\n", @@ -803,9 +884,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Seg1.openAndStart()\n", + "Seg2.openAndStart()\n", + "Seg3.openAndStart()\n", + "Seg4.openAndStart()\n" + ] + } + ], "source": [ "# Start the 4 segmenters\n", "for i in range(4):\n", @@ -818,7 +910,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ @@ -831,7 +923,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -844,9 +936,20 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Segmenter 1 sent 5 data frames\n", + "Segmenter 2 sent 5 data frames\n", + "Segmenter 3 sent 5 data frames\n", + "Segmenter 4 sent 5 data frames\n" + ] + } + ], "source": [ "def get_send_str(idx):\n", " prefix_str = \"THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER\"\n", @@ -881,9 +984,117 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Receiving data... Rd 0\n", + " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER0 WE WANT TO SEND EVERY 1 MILLISECOND.\n", + " bufLen:\t 86\n", + " eventNum:\t 1000\n", + " dataId:\t 1285\n", + "Receiving data... Rd 1\n", + " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER1 WE WANT TO SEND EVERY 1 MILLISECOND.\n", + " bufLen:\t 86\n", + " eventNum:\t 2000\n", + " dataId:\t 1285\n", + "Receiving data... Rd 2\n", + " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER2 WE WANT TO SEND EVERY 1 MILLISECOND.\n", + " bufLen:\t 86\n", + " eventNum:\t 3000\n", + " dataId:\t 1285\n", + "Receiving data... Rd 3\n", + " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER3 WE WANT TO SEND EVERY 1 MILLISECOND.\n", + " bufLen:\t 86\n", + " eventNum:\t 4000\n", + " dataId:\t 1285\n", + "Receiving data... Rd 4\n", + " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER0 WE WANT TO SEND EVERY 1 MILLISECOND.\n", + " bufLen:\t 86\n", + " eventNum:\t 1001\n", + " dataId:\t 1285\n", + "Receiving data... Rd 5\n", + " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER1 WE WANT TO SEND EVERY 1 MILLISECOND.\n", + " bufLen:\t 86\n", + " eventNum:\t 2001\n", + " dataId:\t 1285\n", + "Receiving data... Rd 6\n", + " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER2 WE WANT TO SEND EVERY 1 MILLISECOND.\n", + " bufLen:\t 86\n", + " eventNum:\t 3001\n", + " dataId:\t 1285\n", + "Receiving data... Rd 7\n", + " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER3 WE WANT TO SEND EVERY 1 MILLISECOND.\n", + " bufLen:\t 86\n", + " eventNum:\t 4001\n", + " dataId:\t 1285\n", + "Receiving data... Rd 8\n", + " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER0 WE WANT TO SEND EVERY 1 MILLISECOND.\n", + " bufLen:\t 86\n", + " eventNum:\t 1002\n", + " dataId:\t 1285\n", + "Receiving data... Rd 9\n", + " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER1 WE WANT TO SEND EVERY 1 MILLISECOND.\n", + " bufLen:\t 86\n", + " eventNum:\t 2002\n", + " dataId:\t 1285\n", + "Receiving data... Rd 10\n", + " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER2 WE WANT TO SEND EVERY 1 MILLISECOND.\n", + " bufLen:\t 86\n", + " eventNum:\t 3002\n", + " dataId:\t 1285\n", + "Receiving data... Rd 11\n", + " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER3 WE WANT TO SEND EVERY 1 MILLISECOND.\n", + " bufLen:\t 86\n", + " eventNum:\t 4002\n", + " dataId:\t 1285\n", + "Receiving data... Rd 12\n", + " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER0 WE WANT TO SEND EVERY 1 MILLISECOND.\n", + " bufLen:\t 86\n", + " eventNum:\t 1003\n", + " dataId:\t 1285\n", + "Receiving data... Rd 13\n", + " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER1 WE WANT TO SEND EVERY 1 MILLISECOND.\n", + " bufLen:\t 86\n", + " eventNum:\t 2003\n", + " dataId:\t 1285\n", + "Receiving data... Rd 14\n", + " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER2 WE WANT TO SEND EVERY 1 MILLISECOND.\n", + " bufLen:\t 86\n", + " eventNum:\t 3003\n", + " dataId:\t 1285\n", + "Receiving data... Rd 15\n", + " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER3 WE WANT TO SEND EVERY 1 MILLISECOND.\n", + " bufLen:\t 86\n", + " eventNum:\t 4003\n", + " dataId:\t 1285\n", + "Receiving data... Rd 16\n", + " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER0 WE WANT TO SEND EVERY 1 MILLISECOND.\n", + " bufLen:\t 86\n", + " eventNum:\t 1004\n", + " dataId:\t 1285\n", + "Receiving data... Rd 17\n", + " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER1 WE WANT TO SEND EVERY 1 MILLISECOND.\n", + " bufLen:\t 86\n", + " eventNum:\t 2004\n", + " dataId:\t 1285\n", + "Receiving data... Rd 18\n", + " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER2 WE WANT TO SEND EVERY 1 MILLISECOND.\n", + " bufLen:\t 86\n", + " eventNum:\t 3004\n", + " dataId:\t 1285\n", + "Receiving data... Rd 19\n", + " recv_buf:\t THIS IS A VERY LONG EVENT MESSAGE FROM SEGMENTER3 WE WANT TO SEND EVERY 1 MILLISECOND.\n", + " bufLen:\t 86\n", + " eventNum:\t 4004\n", + " dataId:\t 1285\n", + "Reassembler stats: (0, 20, 0, 0, 0, )\n" + ] + } + ], "source": [ "# Prepare Python list to hold the output data\n", "recv_bytes_list = [None] # Placeholder for the byte buffer\n", @@ -907,6 +1118,135 @@ "assert(res[0] == 0) # no losses\n", "# assert(res[1] == 20) # hold for the 1st try" ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NO EVENT LOSS\n" + ] + } + ], + "source": [ + "# Validation\n", + "if reas.get_LostEvent().has_error():\n", + " print(\"NO EVENT LOSS\")\n", + "else:\n", + " print(f\"LOST EVENT: ({reas.get_LostEvent().value()[0]}, {reas.get_LostEvent().value()[1]})\")\n", + "\n", + "assert(reas.get_LostEvent().error().code == e2sar_py.E2SARErrorc.NotFound)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## DPReasTest5\n", + "\n", + "Examples of reading SegmenterFlags from INI files." + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Init ReassemblerFlag successfully!\n" + ] + } + ], + "source": [ + "# Reassembler\n", + "res_flag = e2sar_py.DataPlane.ReassemblerFlags.getFromINI(\n", + " '/home/ubuntu/dev-e2sar/reassembler_config.ini'\n", + ")\n", + "\n", + "if res_flag.has_error():\n", + " print(f\"Read from ini file failed: {res_flag.error()}\")\n", + "else:\n", + " print(\"Init ReassemblerFlag successfully!\")" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Reassembler flags:\n", + " period_ms=100\n", + " useCP=True\n", + " validateCert = True\n", + " epoch_ms = 1000\n", + " setPoint = 0.0\n", + " [Kp, Ki, Kd] = [0.0, 0.0, 2.0]\n", + " [weight, min_factor, max_factor] = [1.0, 0.5, 2.0]\n", + " portRange = -1\n", + " withLBHeader = False\n" + ] + } + ], + "source": [ + "# Validation\n", + "rflags = res_flag.value()\n", + "\n", + "assert(rflags.useCP == True)\n", + "assert(rflags.validateCert == True)\n", + "assert(rflags.portRange == -1)\n", + "\n", + "print(\"Reassembler flags:\")\n", + "print(f\" period_ms={rflags.period_ms}\") # should be 100 according to the C++ constructor\n", + "print(f\" useCP={rflags.useCP}\")\n", + "print(f\" validateCert = {rflags.validateCert}\")\n", + "print(f\" epoch_ms = {rflags.epoch_ms}\")\n", + "print(f\" setPoint = {rflags.setPoint}\")\n", + "print(f\" [Kp, Ki, Kd] = [{rflags.Kp}, {rflags.Ki}, {rflags.Kd}]\")\n", + "print(f\" [weight, min_factor, max_factor] = [{rflags.weight}, {rflags.min_factor}, {rflags.max_factor}]\")\n", + "print(f\" portRange = {rflags.portRange}\")\n", + "print(f\" withLBHeader = {rflags.withLBHeader}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Init SegmenterFlag successfully!\n" + ] + } + ], + "source": [ + "# Segmenter\n", + "res_flag = e2sar_py.DataPlane.SegmenterFlags.getFromINI(\n", + " '/home/ubuntu/dev-e2sar/segmenter_config.ini'\n", + ")\n", + "\n", + "if res_flag.has_error():\n", + " print(f\"Read from ini file failed: {res_flag.error()}\")\n", + "else:\n", + " print(\"Init SegmenterFlag successfully!\")\n", + "\n", + "sflags = res_flag.value()\n", + "assert(sflags.useCP == True)\n", + "assert(sflags.syncPeriodMs == 1000)\n", + "assert(sflags.syncPeriods == 2)" + ] } ], "metadata": { From 1959b6e206fb27c85e8f7a290896d8536146758f Mon Sep 17 00:00:00 2001 From: Xinxin Mei Date: Wed, 2 Oct 2024 04:19:25 +0000 Subject: [PATCH 73/73] All other Python examples work --- .../example_ControlPlane.ipynb | 50 +++++++++---------- .../pybind11_examples/example_EjfatURI.ipynb | 12 +---- 2 files changed, 27 insertions(+), 35 deletions(-) diff --git a/scripts/notebooks/pybind11_examples/example_ControlPlane.ipynb b/scripts/notebooks/pybind11_examples/example_ControlPlane.ipynb index f7cc61b2..c123da96 100644 --- a/scripts/notebooks/pybind11_examples/example_ControlPlane.ipynb +++ b/scripts/notebooks/pybind11_examples/example_ControlPlane.ipynb @@ -64,7 +64,7 @@ "Defaulting to user installation because normal site-packages is not writeable\n", "Requirement already satisfied: protobuf in /home/ubuntu/.local/lib/python3.10/site-packages (5.28.1)\n", "Note: you may need to restart the kernel to use updated packages.\n", - "Timestamp: 2024-09-18T22:24:39.348087Z, seconds = 1726698279, nanos = 348087000\n" + "Timestamp: 2024-10-02T04:14:25.698545Z, seconds = 1727842465, nanos = 698545000\n" ] } ], @@ -111,7 +111,7 @@ "Worker fill percent: 75.5\n", "Worker control signal: 0.8999999761581421\n", "Worker slots assigned: 10\n", - "Worker last updated: 2024-09-18T22:24:39.348087Z\n" + "Worker last updated: 2024-10-02T04:14:25.698545Z\n" ] } ], @@ -144,8 +144,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "Timestamp: 2024-09-18T22:24:39.348087Z\n", - "Expires at: 2024-09-18T23:24:39.348087Z\n" + "Timestamp: 2024-10-02T04:14:25.698545Z\n", + "Expires at: 2024-10-02T05:14:25.698545Z\n" ] } ], @@ -328,7 +328,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Instance token: af539d34168742fcb7885b500ce36c69bd78c53ad7ce42e6ae8bc050e9a67929\n" + "Instance token: 955d427b68d84c56e0f197dbcb80ff5217b21c7ff13e5d4334ec33dd31fe177f\n" ] } ], @@ -399,7 +399,7 @@ "output_type": "stream", "text": [ "Sync IPv4 addr is: 192.168.0.3\n", - "LB id is: 32\n" + "LB id is: 2\n" ] } ], @@ -555,7 +555,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "LB ID is: 33\n" + "LB ID is: 3\n" ] } ], @@ -678,8 +678,8 @@ "\n", "Send state succeeded\n", "\n", - "Session id is: 16\n", - "Session token is: 5eda10833ab10f3f322f8026615bb6da872052993a2dfc0ce36bb08b14216b50\n", + "Session id is: 1\n", + "Session token is: 9f5dbfd36e86f520eea8a808faf79129a709f896df3a1123f8bed664c400c1d4\n", "\n", "Deregister worker succeeded\n" ] @@ -796,7 +796,7 @@ "\n", "Send state for 25 times\n", "\n", - "Get LB status succeeded: \n" + "Get LB status succeeded: \n" ] } ], @@ -851,7 +851,7 @@ " fill_percent: 0.800000011920929\n", " control_signal: 1.0\n", " slots_assigned: 512\n", - " last_updated: 2024-09-18T22:24:42.738418623Z\n" + " last_updated: 2024-10-02T04:18:05.888130752Z\n" ] } ], @@ -895,11 +895,11 @@ "name": "stdout", "output_type": "stream", "text": [ - "Timestamp: 2024-09-18T22:24:42.840649988Z\n", - "expiresAt: 2024-09-18T23:24:40Z\n", + "Timestamp: 2024-10-02T04:18:05.990769793Z\n", + "expiresAt: 2024-10-02T05:18:03Z\n", "currentEpoch: 3\n", "currentPredictedEventNumber: 9223372036854775808\n", - "workerStatusList: []\n" + "workerStatusList: []\n" ] } ], @@ -974,16 +974,16 @@ "Create LBManager obj succeeded: 0\n", "Register worker succeeded\n", "\n", - "Session id is: 18\n", - "Session token is: 6367c9176ad83fbeaf43d4a53e36b0f01695a8d80789ecf1cd2e0c435e145369\n", + "Session id is: 3\n", + "Session token is: 68cb8d9db7a4b367d4d57572a785c1d68aca22c3331fa52d4fdd1e2304c5c9ce\n", "\n", "Send state for 25 times\n", "Add senders succeeded.\n", "\n", - "Get LB status succeeded: \n", + "Get LB status succeeded: \n", "Remove senders succeeded.\n", "\n", - "Get LB status succeeded: \n", + "Get LB status succeeded: \n", "\n", "Deregister worker succeeded\n", "Free LB succeeded!\n" @@ -1138,22 +1138,22 @@ "Create LBManager obj succeeded: 0\n", "Register worker succeeded\n", "\n", - "Session id is: 19\n", - "Session token is: b91e53badc06c7547526b6a2a1e3c45283c7033e575749a276d2aca27dbad63a\n", + "Session id is: 4\n", + "Session token is: 8800e7562f0a9a4b78bff20d8a20a9047201faaee9658c65e41e57b8e3fe41af\n", "\n", "Send state for 25 times\n", "\n", - "Get LB status succeeded: \n", + "Get LB status succeeded: \n", "Sender addresses: ['192.168.20.1', '192.168.20.2']\n", "Current Epoch: 2\n", "Current predicted Event No: 9223372036854775808\n", - "Workers: []\n", - "Timestamp: 2024-09-18T22:24:48.319546597Z\n", - "expiresAt: 2024-09-18T23:24:45Z\n", + "Workers: []\n", + "Timestamp: 2024-10-02T04:18:11.402769466Z\n", + "expiresAt: 2024-10-02T05:18:08Z\n", "\n", "Send state for 25 times\n", "\n", - "LB id: 37\n", + "LB id: 7\n", "1\n" ] } diff --git a/scripts/notebooks/pybind11_examples/example_EjfatURI.ipynb b/scripts/notebooks/pybind11_examples/example_EjfatURI.ipynb index b27ee6e2..745fb4b6 100644 --- a/scripts/notebooks/pybind11_examples/example_EjfatURI.ipynb +++ b/scripts/notebooks/pybind11_examples/example_EjfatURI.ipynb @@ -13,15 +13,7 @@ "cell_type": "code", "execution_count": 1, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0.1.1\n" - ] - } - ], + "outputs": [], "source": [ "import sys\n", "\n", @@ -279,7 +271,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\n", + "\n", "Instance Token: token123\n" ] }