Initial drop of the JSON store/sampler

configure.ac

@@ -61,6 +61,7 @@ AC_C_FLEXIBLE_ARRAY_MEMBER
 OVIS_PKGLIBDIR
 
 AC_LIB_HAVE_LINKFLAGS([jansson], [], [#include <jansson.h>])
+AM_CONDITIONAL([HAVE_LIBJANSSON], [test "x$HAVE_LIBJANSSON" = xyes])
 
 dnl change sharedstatedir default
 test "$sharedstatedir" = '${prefix}/com' && sharedstatedir='${prefix}/var/lib'
@@ -1047,6 +1048,7 @@ ldms/src/sampler/syspapi/Makefile
 ldms/src/sampler/app_sampler/Makefile
 ldms/src/sampler/slingshot_metrics/Makefile
 ldms/src/sampler/slingshot_info/Makefile
+ldms/src/sampler/json/Makefile
 ldms/src/contrib/sampler/Makefile
 ldms/src/contrib/sampler/daos/Makefile
 ldms/src/contrib/sampler/daos/test/Makefile

ldms/src/sampler/Makefile.am

@@ -34,6 +34,7 @@ SUBDIRS += lustre_client
 SUBDIRS += lustre_mdt
 SUBDIRS += lustre_ost
 SUBDIRS += lustre_mdc
+SUBDIRS += json
 
 if HAVE_DCGM
 SUBDIRS += dcgm_sampler

ldms/src/sampler/json/Makefile.am

@@ -0,0 +1,18 @@
+pkglib_LTLIBRARIES =
+dist_man7_MANS=
+
+AM_CPPFLAGS = @OVIS_INCLUDE_ABS@
+AM_LDFLAGS = @OVIS_LIB_ABS@
+COMMON_LIBADD = $(top_builddir)/ldms/src/sampler/libsampler_base.la \
+		$(top_builddir)/ldms/src/core/libldms.la \
+		$(top_builddir)/ldms/src/ldmsd/libldmsd_stream.la \
+		@LDFLAGS_GETTIME@ \
+		$(top_builddir)/lib/src/ovis_util/libovis_util.la \
+		$(top_builddir)/lib/src/coll/libcoll.la \
+		$(top_builddir)/lib/src/ovis_json/libovis_json.la \
+		-lpthread
+
+dist_man7_MANS += Plugin_json_stream_sampler.man
+libjson_stream_sampler_la_SOURCES = json_stream_sampler.c
+libjson_stream_sampler_la_LIBADD = $(COMMON_LIBADD)
+pkglib_LTLIBRARIES += libjson_stream_sampler.la

ldms/src/sampler/json/Plugin_json_stream_sampler.man

@@ -0,0 +1,245 @@
+.\" Manpage for json_stream_sampler
+.\" Contact ovis-help@ca.sandia.gov to correct errors or typos.
+.TH man 7 "5 Aug 2023" "v4" "LDMSD Plugin JSON Stream Sampler man page"
+
+.SH NAME
+Plugin_json_stream_sampler - man page for the LDMSD json_stream_sampler plugin
+
+.SH SYNOPSIS
+
+Within ldmsd_controller or a configuration file:
+
+.SY config
+.BR name=\fBjson_stream_sampler\fR
+.BI producer=\fIPRODUCER\fR
+.BI instance=\fIINSTANCE\fR
+.OP component_id=\fICOMP_ID\fR
+.OP stream=\fINAME\fR
+.OP uid=\fIUID\fR
+.OP gid=\fIGID\fR
+.OP perm=\fIPERM\fR
+.OP heap_szperm=\fIBYTES\fR
+.YS
+
+.SH DESCRIPTION
+.P
+The \fBjson_stream_store\fR monitors JSON object data presented on a configured
+set of streams. JSON object data is encoded in LDMS Metric Sets; the
+intention of which is to store these metric sets using decomposition
+through a storage plugin.
+.P
+When publishing JSON dictionary data to \fBjson_stream_plugin\fR,
+there are fields in the JSON dictionary that have special
+meaning. These fields are shown in the table below:
+.P
+.TS
+tab(@) box;
+l l l .
+\fBAttribute Name\fR @ \fBData Type\fR @ \fBDescription\fR
+_
+schema @ string @ The name of a Metric Set schema for JSON dictionaries received on this stream.
+\fINAME\fR_max_len @ integer @ For a list or array named \fINAME\fR, this is maximum length of the list or array.
+.TE
+.SS "Schema Management"
+The value of the \fIschema\fR attribute in the top-level JSON
+dictionary is maintained in a tree. The first time the schema name is
+seen, an LDMS Schema is created based on the value of the JSON
+dictionary. Once created, the schema is used to create the metric
+set. Each time a stream message is received, the metric set is
+updated.
+.PP
+The \fIschema\fR attribute is mandatory. If it not present in the
+top-level JSON dictionary, an error is logged and the message is ignored.
+
+.SS "Encoding Types"
+Primitive types are encoded as attributes in the LDMS metric set with
+their associated LDMS type. The table below shows how the JSON
+attributes are mapped to LDMS metric types.
+.TS
+tab(@) box;
+l l l .
+\fBJSON Type\fR @ \fBLDMS Type\fR @ \fBExample JSON Value\fR
+_
+Integer @ LDMS_V_S64 @ 45
+Floating Point @ LDMS_V_D64 @ 3.1415
+String @ LDMS_V_BYTE_ARRAY @ "hello", 'world'
+List @ LDMS_V_LIST @ [ 1, 2, 3 ]
+Dictionary @ LDMS_V_RECORD @ { "attr1" : 1, "attr2" : 2, "attr3" : 3 }
+.TE
+.PP
+The encoding of all JSON types except strings, dictionaries and lists is
+straightfoward. The coding of Strings, Lists and Dictionaries have additional
+limitations as described below.
+
+.SS "Encoding Strings"
+Strings are encoded as LDMS_V_BYTE_ARRAY. By default, the length of
+the array is 255 unless an attribute with the name \fINAME\fR_max_len
+is present in the dictionary along with the string value, its value is
+used to size the string array.
+.PP
+For example:
+.PP
+.RS 4
+.nf
+{ "my_string" : "this is a string", "my_string_max_len" : 4096 }
+.fi
+.RE
+.PP
+will result in an LDMS metric with the name "my_string", type
+LDMS_V_BYTE_ARRAY, and length of 4096 being created in the metric set.
+
+.SS "Encoding Arrays"
+Any list present in the top-level dictionary is encoded as a list,
+however, lists present in a 2nd-level dictionary are encoded as
+arrays. This is because LDMS_V_LIST inside an LDMS_V_RECORD is not
+supported. The length of the array is determined by the initial value
+of the array in the record; but can be overridden with the
+\fINAME\fR_max_len attribute as described above for strings.
+
+.SS "Encoding Dictionaries"
+The attributes in the top-level JSON dictionary are encoded in the metric
+set directly. For example the JSON dictionary:
+.PP
+.RS 4
+.nf
+{
+  "schema" : "example",
+  "component_id", 10001,
+  "job_id" : 2048,
+  "seq" : [ 1, 2, 3 ]
+}
+.fi
+.RE
+.PP
+results in a metric set as follows:
+.fi
+.RS 4
+.nf
+$ ldms_ls -h localhost -p 10411 -a munge -E example -l
+ovs-5416_example: consistent, last update: Sat Aug 05 11:38:26 2023 -0500 [281178us]
+D s64        component_id                               10001
+D s64        job_id                                     2048
+D list<>     seq                                        [1,2,3]
+D char[]     schema                                     "example"
+.fi
+.RE
+.PP
+Dictionaries inside the top-level dictionary are encoded as
+LDMS_V_RECORD inside a single element LDMS_V_RECORD_ARRAY.  This
+limitation is because an LDMS_V_RECORD is only allowed inside an
+LDMS_V_LIST or LDMS_V_ARRAY.
+.PP
+The JSON below:
+.RS 4
+.nf
+{
+  "schema" : "dictionary",
+  "a_dict" : { "attr_1" : 1, "attr_2" : 2 },
+  "b_dict" : { "attr_3" : 3, "attr_4" : 4 }
+}
+.fi
+.RE
+.PP
+results in the following LDMS metric set.
+.PP
+.RS 4
+.nf
+ovs-5416_dict: consistent, last update: Sat Aug 05 21:14:38 2023 -0500 [839029us]
+M record_type  a_dict_record                             LDMS_V_RECORD_TYPE
+D record[]     a_dict
+  attr_2 attr_1
+       2      1
+M record_type  b_dict_record                             LDMS_V_RECORD_TYPE
+D record[]     b_dict
+  attr_4 attr_3
+       4      3
+D char[]     schema                                     "dict"
+.fi
+.RE
+.PP
+Lists of JSON dictionaries results in each dictionary being encoded as
+an element in an LDMS_V_LIST. Note that all elements in the list must
+be the same type.
+.PP
+The JSON below:
+.PP
+.RS 4
+.nf
+{ "schema" : "dict_list",
+  "a_dict_list" : [
+    { "attr_1" : 1, "attr_2" : 2 },
+    { "attr_1" : 3, "attr_2" : 4 }
+  ]
+}
+.fi
+.RE
+.PP
+results in the following LDMS metric set.
+.PP
+.RS 4
+.nf
+ovs-5416_dict_list: consistent, last update: Sat Aug 05 21:23:11 2023 -0500 [52659us]
+M record_type a_dict_list_record                         LDMS_V_RECORD_TYPE
+D list<>      a_dict_list
+  attr_2 attr_1
+       2      1
+       4      3
+D char[]     schema                                     "dict_list"
+.fi
+.RE
+.SH "CONFIG OPTIONS"
+
+.TP
+.BR name=json_stream_sampler
+This must be json_stream_sampler (the name of the plugin).
+.TP
+.BI producer=\fINAME\fR
+The \fINAME\fR of the data producer (e.g. hostname).
+.TP
+.BI instance=\fINAME\fR
+The \fINAME\fR of the set produced by this plugin. This option is required.
+.TP
+.BI component_id=\fIINT\fR
+An integer identifying the component (default: \fI0\fR).
+.TP
+.BI stream=\fINAME\fR
+The name of the LDMSD stream to register for JSON object data.
+.TP
+.BI uid=\fIUID\fR
+The user-id to assign to the metric set.
+.TP
+.BI gid=\fIGID\fR
+The group-id to assign to the metric set.
+.TP
+.BI perm=\fIOCTAL\fR
+An octal number specifying the read-write permissions for the metric
+set. See open(3).
+.TP
+.BI heap_sz=\fIBYTES\fR
+The number of bytes to reserve for the metric set heap.
+.TP
+
+.SH BUGS
+.PP
+Not all JSON objects can be encoded as metric sets. Support for
+records nested inside other records is accomplished by encoding the
+nested records as strings.
+
+.SH EXAMPLES
+
+Plugin configuration example:
+
+.RS
+.EX
+load name=json_stream_sampler
+config name=json_stream_sampler producer=${HOSTNAME} instance=${HOSTNAME}/slurm \\
+       component_id=2 stream=darshan_data heap_sz=1024
+start name=json_stream_sampler interval=1000000
+.EE
+.RE
+
+.SH SEE ALSO
+.nh
+.BR ldmsd (8),
+.BR ldmsd_controller (8),
+.BR Plugin_store_avro_kakfa (8)