nDPId daemon status event: serialize nDPI API version + Size/Flow

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

TODO.md

@@ -17,3 +17,4 @@ no release plan:
  * improve UDP/TCP timeout handling by reading netfilter conntrack timeouts from /proc (or just read conntrack table entries)
  * detect interface / timeout changes and apply them to nDPId
  * switch to MIT or BSD License
+ * libdaq integration

nDPId.c

@@ -2288,6 +2288,8 @@ static void jsonize_daemon(struct nDPId_reader_thread * const reader_thread, enu
     ndpi_serialize_string_string(&workflow->ndpi_serializer, "version", PKG_VERSION);
 #endif
     ndpi_serialize_string_string(&workflow->ndpi_serializer, "ndpi_version", ndpi_revision());
+    ndpi_serialize_string_uint32(&workflow->ndpi_serializer, "ndpi_api_version", ndpi_get_api_version());
+    ndpi_serialize_string_uint64(&workflow->ndpi_serializer, "size_per_flow", (uint64_t)(sizeof(struct nDPId_flow) + sizeof(struct nDPId_detection_data)));
 
     switch (event)
     {
@@ -5357,13 +5359,19 @@ static void nDPId_print_deps_version(FILE * const out)
 {
     fprintf(out,
             "-------------------------------------------------------\n"
+            "package version: %s\n"
 #ifdef LIBNDPI_STATIC
             "nDPI version...: %s (statically linked)\n"
 #else
             "nDPI version...: %s\n"
 #endif
             " API version...: %u\n"
             "pcap version...: %s\n",
+#ifndef PKG_VERSION
+            "unknown",
+#else
+            PKG_VERSION,
+#endif
             ndpi_revision(),
             ndpi_get_api_version(),
             pcap_lib_version() + strlen("libpcap version "));
@@ -5962,9 +5970,7 @@ int main(int argc, char ** argv)
     }
 
 #ifdef ENABLE_MEMORY_PROFILING
-    logger_early(0, "size/workflow....: %zu bytes", sizeof(struct nDPId_workflow));
     logger_early(0, "size/flow........: %zu bytes", sizeof(struct nDPId_flow) + sizeof(struct nDPId_detection_data));
-    logger_early(0, "size/flow-analyse: %zu bytes", sizeof(struct nDPId_flow_analysis));
 #endif
 
     global_context = ndpi_global_init();

schema/daemon_event_schema.json

@@ -9,7 +9,9 @@
         "daemon_event_name",
         "global_ts_usec",
         "version",
-        "ndpi_version"
+        "ndpi_version",
+        "ndpi_api_version",
+        "size_per_flow"
     ],
     "if": {
         "properties": { "daemon_event_name": { "enum": [ "init", "reconnect" ] } }
@@ -63,6 +65,15 @@
         "ndpi_version": {
             "type": "string"
         },
+        "ndpi_api_version": {
+            "type": "number",
+            "minimum": 11500
+        },
+        "size_per_flow": {
+            "type": "number",
+            "minimum": 1400,
+            "maximum": 1500
+        },
 
         "max-flows-per-thread": {
             "type": "number"

test/results/caches_cfg/teams.pcap.out

@@ -1,5 +1,5 @@
-00610{"daemon_event_id":1,"daemon_event_name":"init","thread_id":0,"packet_id":0,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","uuid":"00000000-dead-c0de-0000-123456789abc","version":"1.7.0","ndpi_version":"4.11.0-4976-59ee1fe","max-flows-per-thread":2048,"max-idle-flows-per-thread":64,"reader-thread-count":1,"flow-scan-interval":10000000,"generic-max-idle-time":600000000,"icmp-max-idle-time":120000000,"udp-max-idle-time":180000000,"tcp-max-idle-time":7560000000,"max-packets-per-flow-to-send":5,"max-packets-per-flow-to-process":32,"max-packets-per-flow-to-analyse":32,"global_ts_usec":0}
-00834{"daemon_event_id":4,"daemon_event_name":"status","thread_id":0,"packet_id":1,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","uuid":"00000000-dead-c0de-0000-123456789abc","version":"1.7.0","ndpi_version":"4.11.0-4976-59ee1fe","packets-captured":1,"packets-processed":0,"pfring_active":false,"pfring_recv":0,"pfring_drop":0,"pfring_shunt":0,"total-skipped-flows":0,"total-l4-payload-len":0,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":0,"total-detection-updates":0,"total-updates":0,"current-active-flows":0,"total-active-flows":0,"total-idle-flows":0,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"global-alloc-count":0,"global-free-count":0,"global-alloc-bytes":0,"global-free-bytes":0,"total-events-serialized":2,"global_ts_usec":1587041672419153}
+00656{"daemon_event_id":1,"daemon_event_name":"init","thread_id":0,"packet_id":0,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","uuid":"00000000-dead-c0de-0000-123456789abc","version":"1.7.0","ndpi_version":"4.11.0-4976-59ee1fe","ndpi_api_version":11619,"size_per_flow":1408,"max-flows-per-thread":2048,"max-idle-flows-per-thread":64,"reader-thread-count":1,"flow-scan-interval":10000000,"generic-max-idle-time":600000000,"icmp-max-idle-time":120000000,"udp-max-idle-time":180000000,"tcp-max-idle-time":7560000000,"max-packets-per-flow-to-send":5,"max-packets-per-flow-to-process":32,"max-packets-per-flow-to-analyse":32,"global_ts_usec":0}
+00880{"daemon_event_id":4,"daemon_event_name":"status","thread_id":0,"packet_id":1,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","uuid":"00000000-dead-c0de-0000-123456789abc","version":"1.7.0","ndpi_version":"4.11.0-4976-59ee1fe","ndpi_api_version":11619,"size_per_flow":1408,"packets-captured":1,"packets-processed":0,"pfring_active":false,"pfring_recv":0,"pfring_drop":0,"pfring_shunt":0,"total-skipped-flows":0,"total-l4-payload-len":0,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":0,"total-detection-updates":0,"total-updates":0,"current-active-flows":0,"total-active-flows":0,"total-idle-flows":0,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"global-alloc-count":0,"global-free-count":0,"global-alloc-bytes":0,"global-free-bytes":0,"total-events-serialized":2,"global_ts_usec":1587041672419153}
 00822{"flow_event_id":1,"flow_event_name":"new","thread_id":0,"packet_id":1,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","uuid":"00000000-dead-c0de-0000-123456789abc","flow_id":1,"flow_state":"info","flow_src_packets_processed":1,"flow_dst_packets_processed":0,"flow_first_seen":1587041672419153,"flow_src_last_pkt_time":1587041672419153,"flow_dst_last_pkt_time":1587041672419153,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":279,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":279,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":279,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1587041672419153,"l3_proto":"ip4","src_ip":"192.168.0.1","dst_ip":"255.255.255.255","src_port":68,"dst_port":67,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":5}
 00943{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":1,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","uuid":"00000000-dead-c0de-0000-123456789abc","flow_id":1,"flow_packet_id":1,"flow_src_last_pkt_time":1587041672419153,"flow_dst_last_pkt_time":1587041672419153,"flow_idle_time":200000000,"pkt_datalink":1,"pkt_caplen":321,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":321,"pkt_l4_len":287,"thread_ts_usec":1587041672419153,"pkt":"\/\/\/\/\/\/\/\/2A0X1lWrCABFAAEzES1AAEARZ+TAqAAB\/\/\/\/\/wBEAEMBHwAAAQEGABgr52AAAIAAAAAAAAAAAAAAAAAAAAAAANgNF9ZVqwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABjglNjNQEBNwIBAwwJVEwtU0cxMTZFPAlUTC1TRzExNkU9BwHYDRfWVav\/"}
 01074{"flow_event_id":7,"flow_event_name":"detected","thread_id":0,"packet_id":1,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","uuid":"00000000-dead-c0de-0000-123456789abc","flow_id":1,"flow_state":"info","flow_src_packets_processed":1,"flow_dst_packets_processed":0,"flow_first_seen":1587041672419153,"flow_src_last_pkt_time":1587041672419153,"flow_dst_last_pkt_time":1587041672419153,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":279,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":279,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":279,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1587041672419153,"l3_proto":"ip4","src_ip":"192.168.0.1","dst_ip":"255.255.255.255","src_port":68,"dst_port":67,"l4_proto":"udp","ndpi": {"confidence": {"6":"DPI"},"proto":"DHCP","proto_id":"18","proto_by_ip":"Unknown","proto_by_ip_id":0,"encrypted":0,"breed":"Acceptable","category_id":14,"category":"Network","hostname":"tl-sg116e","domainame":"tl-sg116e","dhcp": {"fingerprint":"1,3","class_ident":"TL-SG116E"}}}
@@ -665,7 +665,7 @@
 01070{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":1540,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","uuid":"00000000-dead-c0de-0000-123456789abc","flow_id":75,"flow_state":"finished","flow_src_packets_processed":1,"flow_dst_packets_processed":1,"flow_first_seen":1587041694221137,"flow_src_last_pkt_time":1587041694221137,"flow_dst_last_pkt_time":1587041694234511,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":58,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":58,"flow_dst_max_l4_payload_len":134,"flow_src_tot_l4_payload_len":58,"flow_dst_tot_l4_payload_len":134,"midstream":0,"thread_ts_usec":1587041698021081,"l3_proto":"ip4","src_ip":"192.168.1.6","dst_ip":"192.168.1.1","src_port":60837,"dst_port":53,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":5,"ndpi": {"confidence": {"6":"DPI"},"proto":"DNS.Teams","proto_id":"5.250","proto_by_ip":"Unknown","proto_by_ip_id":0,"encrypted":0,"breed":"Safe","category_id":14,"category":"Network","hostname":"c-flightproxy-euno-01-teams.cloudapp.net"}}
 01051{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":1540,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","uuid":"00000000-dead-c0de-0000-123456789abc","flow_id":53,"flow_state":"finished","flow_src_packets_processed":19,"flow_dst_packets_processed":13,"flow_first_seen":1587041687436782,"flow_src_last_pkt_time":1587041687725655,"flow_dst_last_pkt_time":1587041687725568,"flow_idle_time":7580000000,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":1313,"flow_dst_max_l4_payload_len":1440,"flow_src_tot_l4_payload_len":2206,"flow_dst_tot_l4_payload_len":7143,"midstream":0,"thread_ts_usec":1587041698021081,"l3_proto":"ip4","src_ip":"192.168.1.6","dst_ip":"104.40.187.151","src_port":60562,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":5,"ndpi": {"confidence": {"6":"DPI"},"proto":"TLS","proto_id":"91","proto_by_ip":"Azure","proto_by_ip_id":276,"encrypted":1,"breed":"Safe","category_id":5,"category":"Web","hostname":"api.microsoftstream.com"}}
 01065{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":1540,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","uuid":"00000000-dead-c0de-0000-123456789abc","flow_id":39,"flow_state":"finished","flow_src_packets_processed":1,"flow_dst_packets_processed":1,"flow_first_seen":1587041685093044,"flow_src_last_pkt_time":1587041685093044,"flow_dst_last_pkt_time":1587041685127636,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":53,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":53,"flow_dst_max_l4_payload_len":174,"flow_src_tot_l4_payload_len":53,"flow_dst_tot_l4_payload_len":174,"midstream":0,"thread_ts_usec":1587041698021081,"l3_proto":"ip4","src_ip":"192.168.1.6","dst_ip":"192.168.1.1","src_port":50653,"dst_port":53,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":5,"ndpi": {"confidence": {"6":"DPI"},"proto":"DNS.Teams","proto_id":"5.250","proto_by_ip":"Unknown","proto_by_ip_id":0,"encrypted":0,"breed":"Safe","category_id":14,"category":"Network","hostname":"api.flightproxy.teams.microsoft.com"}}
-00856{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":1540,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","uuid":"00000000-dead-c0de-0000-123456789abc","version":"1.7.0","ndpi_version":"4.11.0-4976-59ee1fe","packets-captured":1540,"packets-processed":1498,"pfring_active":false,"pfring_recv":0,"pfring_drop":0,"pfring_shunt":0,"total-skipped-flows":0,"total-l4-payload-len":587095,"total-not-detected-flows":1,"total-guessed-flows":2,"total-detected-flows":80,"total-detection-updates":51,"total-updates":0,"current-active-flows":0,"total-active-flows":83,"total-idle-flows":83,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"global-alloc-count":0,"global-free-count":0,"global-alloc-bytes":0,"global-free-bytes":0,"total-events-serialized":668,"global_ts_usec":1587041698021081}
+00902{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":1540,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","uuid":"00000000-dead-c0de-0000-123456789abc","version":"1.7.0","ndpi_version":"4.11.0-4976-59ee1fe","ndpi_api_version":11619,"size_per_flow":1408,"packets-captured":1540,"packets-processed":1498,"pfring_active":false,"pfring_recv":0,"pfring_drop":0,"pfring_shunt":0,"total-skipped-flows":0,"total-l4-payload-len":587095,"total-not-detected-flows":1,"total-guessed-flows":2,"total-detected-flows":80,"total-detection-updates":51,"total-updates":0,"current-active-flows":0,"total-active-flows":83,"total-idle-flows":83,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"global-alloc-count":0,"global-free-count":0,"global-alloc-bytes":0,"global-free-bytes":0,"total-events-serialized":668,"global_ts_usec":1587041698021081}
 ~~~~~~~~~~~~~~~~~~~~ SUMMARY ~~~~~~~~~~~~~~~~~~~~
 ~~ packets captured/processed: 1540/1498
 ~~ skipped flows.............: 0