Skip to content

Commit

Permalink
Add support for custom event writer (#388)
Browse files Browse the repository at this point in the history 
* Pass json serializer options through converters

* Update file generator

* Add unit test

* Add integration tests

* Fix tests

* Fix test code style

* Add type-specialized property converter write methods

* Use cached DateTimeOffset JsonConverter when default JsonSerializerOptions

* Use cached DateTimeOffset JsonConverter when default JsonSerializerOptions

* Consolidate JsonConverter<DateTimeOffset> getter
  • Loading branch information
@sergiojrdotnet
sergiojrdotnet authored May 29, 2024
1 parent c128995 commit 6ce0252
Show file tree
Hide file tree
Showing 12 changed files with 396 additions and 239 deletions.
136 changes: 69 additions & 67 deletions src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.Generated.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ private static bool ReadProperties(
TBase ecsEvent,
ref DateTimeOffset? timestamp,
ref string loglevel,
ref string ecsVersion
ref string ecsVersion,
JsonSerializerOptions options
)
{
var propertyName = reader.GetString();
Expand All @@ -33,14 +34,14 @@ ref string ecsVersion
{
"log.level" => ReadString(ref reader, ref loglevel),
"ecs.version" => ReadString(ref reader, ref ecsVersion),
"metadata" => ReadProp<MetadataDictionary>(ref reader, "metadata", ecsEvent, (b, v) => b.Metadata = v),
"@timestamp" => ReadDateTime(ref reader, ref @timestamp),
"message" => ReadProp<string>(ref reader, "message", ecsEvent, (b, v) => b.Message = v),
"tags" => ReadProp<string[]>(ref reader, "tags", ecsEvent, (b, v) => b.Tags = v),
"span.id" => ReadProp<string>(ref reader, "span.id", ecsEvent, (b, v) => b.SpanId = v),
"trace.id" => ReadProp<string>(ref reader, "trace.id", ecsEvent, (b, v) => b.TraceId = v),
"transaction.id" => ReadProp<string>(ref reader, "transaction.id", ecsEvent, (b, v) => b.TransactionId = v),
"labels" => ReadProp<Labels>(ref reader, "labels", ecsEvent, (b, v) => b.Labels = v),
"metadata" => ReadProp<MetadataDictionary>(ref reader, "metadata", ecsEvent, (b, v) => b.Metadata = v, options),
"@timestamp" => ReadDateTime(ref reader, ref @timestamp, options),
"message" => ReadProp<string>(ref reader, "message", ecsEvent, (b, v) => b.Message = v, options),
"tags" => ReadProp<string[]>(ref reader, "tags", ecsEvent, (b, v) => b.Tags = v, options),
"span.id" => ReadProp<string>(ref reader, "span.id", ecsEvent, (b, v) => b.SpanId = v, options),
"trace.id" => ReadProp<string>(ref reader, "trace.id", ecsEvent, (b, v) => b.TraceId = v, options),
"transaction.id" => ReadProp<string>(ref reader, "transaction.id", ecsEvent, (b, v) => b.TransactionId = v, options),
"labels" => ReadProp<Labels>(ref reader, "labels", ecsEvent, (b, v) => b.Labels = v, options),
"agent" => ReadProp<Agent>(ref reader, "agent", EcsJsonContext.Default.Agent, ecsEvent, (b, v) => b.Agent = v),
"as" => ReadProp<As>(ref reader, "as", EcsJsonContext.Default.As, ecsEvent, (b, v) => b.As = v),
"client" => ReadProp<Client>(ref reader, "client", EcsJsonContext.Default.Client, ecsEvent, (b, v) => b.Client = v),
Expand Down Expand Up @@ -94,7 +95,7 @@ ref string ecsVersion
typeof(EcsDocument) == ecsEvent.GetType()
? false
: ecsEvent.TryRead(propertyName, out var t)
? ecsEvent.ReceiveProperty(propertyName, ReadPropDeserialize(ref reader, t))
? ecsEvent.ReceiveProperty(propertyName, ReadPropDeserialize(ref reader, t, options))
: false
};
}
Expand All @@ -109,71 +110,72 @@ public override void Write(Utf8JsonWriter writer, TBase value, JsonSerializerOpt
}
writer.WriteStartObject();

WriteTimestamp(writer, value);
WriteTimestamp(writer, value, options);
WriteLogLevel(writer, value);
WriteMessage(writer, value);
WriteEcsVersion(writer, value);
WriteLogEntity(writer, value.Log);
WriteEcsEntity(writer, value.Ecs);
WriteLogEntity(writer, value.Log, options);
WriteEcsEntity(writer, value.Ecs, options);

// Base fields
WriteProp(writer, "tags", value.Tags);
WriteProp(writer, "span.id", value.SpanId);
WriteProp(writer, "trace.id", value.TraceId);
WriteProp(writer, "transaction.id", value.TransactionId);
WriteProp(writer, "labels", value.Labels);
WriteProp(writer, "tags", value.Tags, options);
WriteProp(writer, "span.id", value.SpanId, options);
WriteProp(writer, "trace.id", value.TraceId, options);
WriteProp(writer, "transaction.id", value.TransactionId, options);
WriteProp(writer, "labels", value.Labels, options);

// Complex types
WriteProp(writer, "agent", value.Agent, EcsJsonContext.Default.Agent);
WriteProp(writer, "as", value.As, EcsJsonContext.Default.As);
WriteProp(writer, "client", value.Client, EcsJsonContext.Default.Client);
WriteProp(writer, "cloud", value.Cloud, EcsJsonContext.Default.Cloud);
WriteProp(writer, "code_signature", value.CodeSignature, EcsJsonContext.Default.CodeSignature);
WriteProp(writer, "container", value.Container, EcsJsonContext.Default.Container);
WriteProp(writer, "data_stream", value.DataStream, EcsJsonContext.Default.DataStream);
WriteProp(writer, "destination", value.Destination, EcsJsonContext.Default.Destination);
WriteProp(writer, "device", value.Device, EcsJsonContext.Default.Device);
WriteProp(writer, "dll", value.Dll, EcsJsonContext.Default.Dll);
WriteProp(writer, "dns", value.Dns, EcsJsonContext.Default.Dns);
WriteProp(writer, "elf", value.Elf, EcsJsonContext.Default.Elf);
WriteProp(writer, "email", value.Email, EcsJsonContext.Default.Email);
WriteProp(writer, "error", value.Error, EcsJsonContext.Default.Error);
WriteProp(writer, "event", value.Event, EcsJsonContext.Default.Event);
WriteProp(writer, "faas", value.Faas, EcsJsonContext.Default.Faas);
WriteProp(writer, "file", value.File, EcsJsonContext.Default.File);
WriteProp(writer, "geo", value.Geo, EcsJsonContext.Default.Geo);
WriteProp(writer, "group", value.Group, EcsJsonContext.Default.Group);
WriteProp(writer, "hash", value.Hash, EcsJsonContext.Default.Hash);
WriteProp(writer, "host", value.Host, EcsJsonContext.Default.Host);
WriteProp(writer, "http", value.Http, EcsJsonContext.Default.Http);
WriteProp(writer, "interface", value.Interface, EcsJsonContext.Default.Interface);
WriteProp(writer, "macho", value.Macho, EcsJsonContext.Default.Macho);
WriteProp(writer, "network", value.Network, EcsJsonContext.Default.Network);
WriteProp(writer, "observer", value.Observer, EcsJsonContext.Default.Observer);
WriteProp(writer, "orchestrator", value.Orchestrator, EcsJsonContext.Default.Orchestrator);
WriteProp(writer, "organization", value.Organization, EcsJsonContext.Default.Organization);
WriteProp(writer, "os", value.Os, EcsJsonContext.Default.Os);
WriteProp(writer, "package", value.Package, EcsJsonContext.Default.Package);
WriteProp(writer, "pe", value.Pe, EcsJsonContext.Default.Pe);
WriteProp(writer, "process", value.Process, EcsJsonContext.Default.Process);
WriteProp(writer, "registry", value.Registry, EcsJsonContext.Default.Registry);
WriteProp(writer, "related", value.Related, EcsJsonContext.Default.Related);
WriteProp(writer, "risk", value.Risk, EcsJsonContext.Default.Risk);
WriteProp(writer, "rule", value.Rule, EcsJsonContext.Default.Rule);
WriteProp(writer, "server", value.Server, EcsJsonContext.Default.Server);
WriteProp(writer, "service", value.Service, EcsJsonContext.Default.Service);
WriteProp(writer, "source", value.Source, EcsJsonContext.Default.Source);
WriteProp(writer, "threat", value.Threat, EcsJsonContext.Default.Threat);
WriteProp(writer, "tls", value.Tls, EcsJsonContext.Default.Tls);
WriteProp(writer, "url", value.Url, EcsJsonContext.Default.Url);
WriteProp(writer, "user", value.User, EcsJsonContext.Default.User);
WriteProp(writer, "user_agent", value.UserAgent, EcsJsonContext.Default.UserAgent);
WriteProp(writer, "vlan", value.Vlan, EcsJsonContext.Default.Vlan);
WriteProp(writer, "vulnerability", value.Vulnerability, EcsJsonContext.Default.Vulnerability);
WriteProp(writer, "x509", value.X509, EcsJsonContext.Default.X509);
WriteProp(writer, "metadata", value.Metadata);
WriteProp(writer, "agent", value.Agent, EcsJsonContext.Default.Agent, options);
WriteProp(writer, "as", value.As, EcsJsonContext.Default.As, options);
WriteProp(writer, "client", value.Client, EcsJsonContext.Default.Client, options);
WriteProp(writer, "cloud", value.Cloud, EcsJsonContext.Default.Cloud, options);
WriteProp(writer, "code_signature", value.CodeSignature, EcsJsonContext.Default.CodeSignature, options);
WriteProp(writer, "container", value.Container, EcsJsonContext.Default.Container, options);
WriteProp(writer, "data_stream", value.DataStream, EcsJsonContext.Default.DataStream, options);
WriteProp(writer, "destination", value.Destination, EcsJsonContext.Default.Destination, options);
WriteProp(writer, "device", value.Device, EcsJsonContext.Default.Device, options);
WriteProp(writer, "dll", value.Dll, EcsJsonContext.Default.Dll, options);
WriteProp(writer, "dns", value.Dns, EcsJsonContext.Default.Dns, options);
WriteProp(writer, "elf", value.Elf, EcsJsonContext.Default.Elf, options);
WriteProp(writer, "email", value.Email, EcsJsonContext.Default.Email, options);
WriteProp(writer, "error", value.Error, EcsJsonContext.Default.Error, options);
WriteProp(writer, "event", value.Event, EcsJsonContext.Default.Event, options);
WriteProp(writer, "faas", value.Faas, EcsJsonContext.Default.Faas, options);
WriteProp(writer, "file", value.File, EcsJsonContext.Default.File, options);
WriteProp(writer, "geo", value.Geo, EcsJsonContext.Default.Geo, options);
WriteProp(writer, "group", value.Group, EcsJsonContext.Default.Group, options);
WriteProp(writer, "hash", value.Hash, EcsJsonContext.Default.Hash, options);
WriteProp(writer, "host", value.Host, EcsJsonContext.Default.Host, options);
WriteProp(writer, "http", value.Http, EcsJsonContext.Default.Http, options);
WriteProp(writer, "interface", value.Interface, EcsJsonContext.Default.Interface, options);
WriteProp(writer, "macho", value.Macho, EcsJsonContext.Default.Macho, options);
WriteProp(writer, "network", value.Network, EcsJsonContext.Default.Network, options);
WriteProp(writer, "observer", value.Observer, EcsJsonContext.Default.Observer, options);
WriteProp(writer, "orchestrator", value.Orchestrator, EcsJsonContext.Default.Orchestrator, options);
WriteProp(writer, "organization", value.Organization, EcsJsonContext.Default.Organization, options);
WriteProp(writer, "os", value.Os, EcsJsonContext.Default.Os, options);
WriteProp(writer, "package", value.Package, EcsJsonContext.Default.Package, options);
WriteProp(writer, "pe", value.Pe, EcsJsonContext.Default.Pe, options);
WriteProp(writer, "process", value.Process, EcsJsonContext.Default.Process, options);
WriteProp(writer, "registry", value.Registry, EcsJsonContext.Default.Registry, options);
WriteProp(writer, "related", value.Related, EcsJsonContext.Default.Related, options);
WriteProp(writer, "risk", value.Risk, EcsJsonContext.Default.Risk, options);
WriteProp(writer, "rule", value.Rule, EcsJsonContext.Default.Rule, options);
WriteProp(writer, "server", value.Server, EcsJsonContext.Default.Server, options);
WriteProp(writer, "service", value.Service, EcsJsonContext.Default.Service, options);
WriteProp(writer, "source", value.Source, EcsJsonContext.Default.Source, options);
WriteProp(writer, "threat", value.Threat, EcsJsonContext.Default.Threat, options);
WriteProp(writer, "tls", value.Tls, EcsJsonContext.Default.Tls, options);
WriteProp(writer, "url", value.Url, EcsJsonContext.Default.Url, options);
WriteProp(writer, "user", value.User, EcsJsonContext.Default.User, options);
WriteProp(writer, "user_agent", value.UserAgent, EcsJsonContext.Default.UserAgent, options);
WriteProp(writer, "vlan", value.Vlan, EcsJsonContext.Default.Vlan, options);
WriteProp(writer, "vulnerability", value.Vulnerability, EcsJsonContext.Default.Vulnerability, options);
WriteProp(writer, "x509", value.X509, EcsJsonContext.Default.X509, options);
WriteProp(writer, "metadata", value.Metadata, options);

if (typeof(EcsDocument) != value.GetType())
value.WriteAdditionalProperties((k, v) => WriteProp(writer, k, v));
value.WriteAdditionalProperties((k, v) => WriteProp(writer, k, v, options));
writer.WriteEndObject();
}
}
Expand Down
15 changes: 8 additions & 7 deletions src/Elastic.CommonSchema/Serialization/EcsDocumentJsonConverter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ namespace Elastic.CommonSchema.Serialization
if (reader.TokenType != JsonTokenType.PropertyName)
throw new JsonException();

var _ = ReadProperties(ref reader, ecsEvent, ref timestamp, ref loglevel, ref ecsVersion);
var _ = ReadProperties(ref reader, ecsEvent, ref timestamp, ref loglevel, ref ecsVersion, options);
}
if (!string.IsNullOrEmpty(loglevel))
{
Expand All @@ -65,11 +65,11 @@ private static void WriteMessage(Utf8JsonWriter writer, EcsDocument value)
writer.WriteString("message", value.Message);
}

private static void WriteLogEntity(Utf8JsonWriter writer, Log? value) {
private static void WriteLogEntity(Utf8JsonWriter writer, Log? value, JsonSerializerOptions options) {
if (value == null) return;
if (!value.ShouldSerialize) return;

WriteProp(writer, "log", value, EcsJsonContext.Default.Log);
WriteProp(writer, "log", value, EcsJsonContext.Default.Log, options);
}

private static void WriteLogLevel(Utf8JsonWriter writer, EcsDocument value)
Expand All @@ -78,23 +78,24 @@ private static void WriteLogLevel(Utf8JsonWriter writer, EcsDocument value)
writer.WriteString("log.level", value.Log?.Level);
}

private static void WriteEcsEntity(Utf8JsonWriter writer, Ecs? value)
private static void WriteEcsEntity(Utf8JsonWriter writer, Ecs? value, JsonSerializerOptions options)
{
if (value == null) return;
if (!value.ShouldSerialize) return;

WriteProp(writer, "ecs", value, EcsJsonContext.Default.Ecs);
WriteProp(writer, "ecs", value, EcsJsonContext.Default.Ecs, options);
}

private static void WriteEcsVersion(Utf8JsonWriter writer, EcsDocument value) =>
writer.WriteString("ecs.version", value.Ecs?.Version ?? EcsDocument.Version);

private static void WriteTimestamp(Utf8JsonWriter writer, BaseFieldSet value)
private static void WriteTimestamp(Utf8JsonWriter writer, BaseFieldSet value, JsonSerializerOptions options)
{
if (!value.Timestamp.HasValue) return;

writer.WritePropertyName("@timestamp");
EcsJsonConfiguration.DateTimeOffsetConverter.Write(writer, value.Timestamp.Value, EcsJsonConfiguration.SerializerOptions);
var converter = GetDateTimeOffsetConverter(options);
converter.Write(writer, value.Timestamp.Value, options);
}
}

Expand Down
47 changes: 34 additions & 13 deletions src/Elastic.CommonSchema/Serialization/EcsJsonConverterBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,22 @@ namespace Elastic.CommonSchema.Serialization
public abstract class EcsJsonConverterBase<T> : JsonConverter<T>
{
/// <summary></summary>
protected static bool ReadDateTime(ref Utf8JsonReader reader, ref DateTimeOffset? dateTime)
protected static JsonConverter<DateTimeOffset> GetDateTimeOffsetConverter(JsonSerializerOptions options) =>
options == EcsJsonConfiguration.SerializerOptions
? EcsJsonConfiguration.DateTimeOffsetConverter
: (JsonConverter<DateTimeOffset>)options.GetConverter(typeof(DateTimeOffset));

/// <summary></summary>
protected static bool ReadDateTime(ref Utf8JsonReader reader, ref DateTimeOffset? dateTime, JsonSerializerOptions options)
{
if (reader.TokenType == JsonTokenType.Null)
{
dateTime = null;
return true;
}

dateTime = EcsJsonConfiguration.DateTimeOffsetConverter.Read(ref reader, typeof(DateTimeOffset), EcsJsonConfiguration.SerializerOptions);
var converter = GetDateTimeOffsetConverter(options);
dateTime = converter.Read(ref reader, typeof(DateTimeOffset), options);
return true;
}

Expand All @@ -34,11 +41,27 @@ protected static bool ReadString(ref Utf8JsonReader reader, ref string? stringPr
}

/// <summary></summary>
protected static void WriteProp<TValue>(Utf8JsonWriter writer, string key, TValue value)
protected static void WritePropLong(Utf8JsonWriter writer, string key, long? value)
{
if (value == null) return;

writer.WritePropertyName(key);
writer.WriteNumberValue(value.Value);
}

/// <summary></summary>
protected static void WritePropString(Utf8JsonWriter writer, string key, string? value)
{
if (value == null) return;

var options = EcsJsonConfiguration.SerializerOptions;
writer.WritePropertyName(key);
writer.WriteStringValue(value);
}

/// <summary></summary>
protected static void WriteProp<TValue>(Utf8JsonWriter writer, string key, TValue value, JsonSerializerOptions options)
{
if (value == null) return;

writer.WritePropertyName(key);
// Attempt to use existing converter first before re-entering through JsonSerializer.Serialize().
Expand All @@ -47,7 +70,8 @@ protected static void WriteProp<TValue>(Utf8JsonWriter writer, string key, TValu
}

/// <summary></summary>
protected static void WriteProp<TValue>(Utf8JsonWriter writer, string key, TValue value, JsonTypeInfo<TValue> typeInfo)
protected static void WriteProp<TValue>(Utf8JsonWriter writer, string key, TValue value, JsonTypeInfo<TValue> typeInfo,
JsonSerializerOptions options)
{
if (value == null) return;

Expand All @@ -56,16 +80,14 @@ protected static void WriteProp<TValue>(Utf8JsonWriter writer, string key, TValu

//To support user supplied subtypes
if (type != typeof(TValue))
JsonSerializer.Serialize(writer, value, type, EcsJsonConfiguration.SerializerOptions);
JsonSerializer.Serialize(writer, value, type, options);
else JsonSerializer.Serialize(writer, value, typeInfo);
}

internal static object? ReadPropDeserialize(ref Utf8JsonReader reader, Type type)
internal static object? ReadPropDeserialize(ref Utf8JsonReader reader, Type type, JsonSerializerOptions options)
{
if (reader.TokenType == JsonTokenType.Null) return null;

var options = EcsJsonConfiguration.SerializerOptions;

return JsonSerializer.Deserialize(ref reader, type, options);
}

Expand Down Expand Up @@ -100,19 +122,18 @@ protected static bool ReadPropString(ref Utf8JsonReader reader, string key, T b,

/// <summary></summary>
// ReSharper disable once UnusedParameter.Local (key is used for readability)
private static TValue? ReadProp<TValue>(ref Utf8JsonReader reader, string key) where TValue : class
private static TValue? ReadProp<TValue>(ref Utf8JsonReader reader, string key, JsonSerializerOptions options) where TValue : class
{
if (reader.TokenType == JsonTokenType.Null) return null;

var options = EcsJsonConfiguration.SerializerOptions;
return JsonSerializer.Deserialize<TValue>(ref reader, options);
}

/// <summary></summary>
protected static bool ReadProp<TValue>(ref Utf8JsonReader reader, string key, T b, Action<T, TValue?> set)
protected static bool ReadProp<TValue>(ref Utf8JsonReader reader, string key, T b, Action<T, TValue?> set, JsonSerializerOptions options)
where TValue : class
{
set(b, ReadProp<TValue>(ref reader, key));
set(b, ReadProp<TValue>(ref reader, key, options));
return true;
}

Expand Down
Loading

0 comments on commit 6ce0252

Please sign in to comment.