Serialization Simplification

.github/workflows/perf.yml

@@ -47,7 +47,8 @@ jobs:
       - name: Setup dotnet
         uses: actions/setup-dotnet@v3
         with:
-          dotnet-version: '6.x'
+          dotnet-version: '8.x'
+          dotnet-quality: 'preview'
 
       - name: Release Build
         run: dotnet build -c Release tests/NATS.Client.Perf/NATS.Client.Perf.csproj

.github/workflows/release.yml

@@ -39,9 +39,8 @@ jobs:
         name: Setup dotnet
         uses: actions/setup-dotnet@v3
         with:
-          dotnet-version: |
-            6.x
-            7.x
+          dotnet-version: '8.x'
+          dotnet-quality: 'preview'
 
       - if: ${{ fromJSON(steps.tag.outputs.create) }}
         name: Pack

.github/workflows/test.yml

@@ -39,7 +39,8 @@ jobs:
       - name: Setup dotnet
         uses: actions/setup-dotnet@v3
         with:
-          dotnet-version: '6.x'
+          dotnet-version: '8.x'
+          dotnet-quality: 'preview'
 
       - name: Build
         run: dotnet build -c Debug
@@ -59,6 +60,15 @@ jobs:
       - name: Test Services
         run: dotnet test -c Debug --no-build --logger:"console;verbosity=normal" tests/NATS.Client.Services.Tests/NATS.Client.Services.Tests.csproj
 
+      - name: Check Native AOT
+        run: |
+          cd tests/NATS.Client.CheckNativeAot
+          dotnet publish -r linux-x64 -c Release -o dist
+          cd dist
+          ls -lh
+          file NATS.Client.CheckNativeAot
+          ./NATS.Client.CheckNativeAot
+
   memory_test:
     name: memory test
     strategy:
@@ -82,7 +92,8 @@ jobs:
       - name: Setup dotnet
         uses: actions/setup-dotnet@v3
         with:
-          dotnet-version: '6.x'
+          dotnet-version: '8.x'
+          dotnet-quality: 'preview'
 
       - name: Get nats-server
         shell: bash

NATS.Client.sln

@@ -87,6 +87,14 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example.Services", "sandbox
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example.TlsFirst", "sandbox\Example.TlsFirst\Example.TlsFirst.csproj", "{88625045-978F-417F-9F51-A4E3A9718945}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example.NativeAot", "sandbox\Example.NativeAot\Example.NativeAot.csproj", "{51362D87-49C8-414C-AAB7-E51B946231E7}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NATS.Client.CheckNativeAot", "tests\NATS.Client.CheckNativeAot\NATS.Client.CheckNativeAot.csproj", "{CF44A42E-C075-4C5C-BE8B-3DF266FC617B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Example.ProtoBufMessages", "sandbox\Example.ProtoBufMessages\Example.ProtoBufMessages.csproj", "{9FCD9377-FE5F-4D94-BDCF-54427DB6487D}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NATS.Client.Serializers.Json", "src\NATS.Client.Serializers.Json\NATS.Client.Serializers.Json.csproj", "{5B4B30A5-941B-44F9-98C6-06F0BB2242AB}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -225,6 +233,22 @@ Global
 		{88625045-978F-417F-9F51-A4E3A9718945}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{88625045-978F-417F-9F51-A4E3A9718945}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{88625045-978F-417F-9F51-A4E3A9718945}.Release|Any CPU.Build.0 = Release|Any CPU
+		{51362D87-49C8-414C-AAB7-E51B946231E7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{51362D87-49C8-414C-AAB7-E51B946231E7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{51362D87-49C8-414C-AAB7-E51B946231E7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{51362D87-49C8-414C-AAB7-E51B946231E7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{CF44A42E-C075-4C5C-BE8B-3DF266FC617B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{9FCD9377-FE5F-4D94-BDCF-54427DB6487D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{9FCD9377-FE5F-4D94-BDCF-54427DB6487D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{9FCD9377-FE5F-4D94-BDCF-54427DB6487D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{9FCD9377-FE5F-4D94-BDCF-54427DB6487D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5B4B30A5-941B-44F9-98C6-06F0BB2242AB}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -264,6 +288,10 @@ Global
 		{749CAE39-4C1E-4627-9E31-A36B987BC453} = {C526E8AB-739A-48D7-8FC4-048978C9B650}
 		{DD0AB72A-D6CD-4054-A9C9-0DCA3EDBA00F} = {95A69671-16CA-4133-981C-CC381B7AAA30}
 		{88625045-978F-417F-9F51-A4E3A9718945} = {95A69671-16CA-4133-981C-CC381B7AAA30}
+		{51362D87-49C8-414C-AAB7-E51B946231E7} = {95A69671-16CA-4133-981C-CC381B7AAA30}
+		{CF44A42E-C075-4C5C-BE8B-3DF266FC617B} = {C526E8AB-739A-48D7-8FC4-048978C9B650}
+		{9FCD9377-FE5F-4D94-BDCF-54427DB6487D} = {95A69671-16CA-4133-981C-CC381B7AAA30}
+		{5B4B30A5-941B-44F9-98C6-06F0BB2242AB} = {4827B3EC-73D8-436D-AE2A-5E29AC95FD0C}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {8CBB7278-D093-448E-B3DE-B5991209A1AA}

docs/documentation/serialization.md

@@ -0,0 +1,277 @@
+# Serialization
+
+NATS .NET Client supports serialization of messages using a simple interface [`INatsSerializer`](xref:NATS.Client.Core.INatsSerializer).
+
+```csharp
+public interface INatsSerializer
+{
+    // Serialize the value to the buffer.
+    void Serialize<T>(IBufferWriter<byte> bufferWriter, T value);
+
+    // Deserialize the value from the buffer.
+    T? Deserialize<T>(in ReadOnlySequence<byte> buffer);
+}
+```
+
+By default, the client uses the [`NatsDefaultSerializer`](xref:NATS.Client.Core.NatsDefaultSerializer) which can handle binary data, UTF8 strings and numbers. You can provide your own
+serializer by implementing the [`INatsSerializer`](xref:NATS.Client.Core.INatsSerializer) interface or using the [`NatsJsonContextSerializer`](xref:NATS.Client.Core.NatsJsonContextSerializer) for generated
+JSON serialization. Serializers can also be chained together to provide multiple serialization formats typically
+depending on the types being used.
+
+## Default Serializer
+
+Default serializer is used when no serializer is provided to the connection options. It can handle binary data, UTF8
+strings and numbers. It uses the following rules to determine the type of the data:
+
+- If the data is a byte array, [`Memory<byte>`](https://learn.microsoft.com/dotnet/api/system.memory-1), [`IMemoryOwner<byte>`](https://learn.microsoft.com/dotnet/api/system.buffers.imemoryowner-1) or similar it is treated as binary data.
+- If the data is a string or similar it is treated as UTF8 string.
+- If the data is a primitive (for example `DateTime`, `int` or `double`. See also [`NatsUtf8PrimitivesSerializer`](xref:NATS.Client.Core.NatsUtf8PrimitivesSerializer)) it is treated as the primitive encoded as a UTF8 string.
+- For any other type, the serializer will throw an exception.
+
+```csharp
+// Same as not specifying a serializer.
+var natsOpts = NatsOpts.Default with { Serializer = NatsDefaultSerializer.Default };
+
+await using var nats = new NatsConnection(natsOpts);
+
+await using INatsSub<string> sub = await nats.SubscribeAsync<string>(subject: "foo");
+
+// Flush the the network buffers to make sure the subscription request has been processed.
+await nats.PingAsync();
+
+await nats.PublishAsync<string>(subject: "foo", data: "Hello World");
+
+NatsMsg<string?> msg = await sub.Msgs.ReadAsync();
+
+// Outputs 'Hello World'
+Console.WriteLine(msg.Data);
+```
+
+The default serializer is designed to be used by developers who want to only work with binary data, and provide an out
+of the box experience for basic use cases like sending and receiving UTF8 strings.
+
+### Using JSON Serialization with Reflection
+
+If you're not using [Native AOT deployments](https://learn.microsoft.com/dotnet/core/deploying/native-aot) you can use
+the [`NatsJsonSerializer`](xref:NATS.Client.Core.Serializers.Json.NatsJsonSerializer) to serialize and deserialize
+messages. [`NatsJsonSerializer`](xref:NATS.Client.Core.Serializers.Json.NatsJsonSerializer) uses [`System.Text.Json`](https://learn.microsoft.com/dotnet/api/system.text.json)
+APIs that can work with types that are not registered to generate serialization code.
+
+Using this serializer is most useful for use cases where you want to send and receive JSON messages and you don't want to
+worry about registering types. It's also useful for prototyping and testing. To use the serializer you need to install
+the `NATS.Client.Serializers.Json` Nuget package.
+
+```shell
+$ dotnet add package NATS.Client.Serializers.Json --prerelease
+```
+
+Then set the serializer as the default for the connection:
+
+```csharp
+using NATS.Client.Serializers.Json;
+
+var natsOpts = NatsOpts.Default with { Serializer = NatsJsonSerializer.Default };
+
+await using var nats = new NatsConnection(natsOpts);
+```
+
+## Using JSON Serializer Context
+
+The [`NatsJsonContextSerializer`](xref:NATS.Client.Core.NatsJsonContextSerializer) uses the [`System.Text.Json`](https://learn.microsoft.com/dotnet/api/system.text.json) serializer to serialize and deserialize messages. It relies
+on the [`System.Text.Json` source generator](https://devblogs.microsoft.com/dotnet/try-the-new-system-text-json-source-generator/)
+to generate the serialization code at compile time. This is the recommended JSON serializer for most use cases and it's
+required for [Native AOT deployments](https://learn.microsoft.com/dotnet/core/deploying/native-aot).
+
+First you need to define your JSON classes and a context to generate the serialization code:
+```csharp
+public record MyData
+{
+    [JsonPropertyName("id")]
+    public int Id { get; set; }
+
+    [JsonPropertyName("name")]
+    public string? Name { get; set; }
+}
+
+[JsonSerializable(typeof(MyData))]
+internal partial class MyJsonContext : JsonSerializerContext;
+```
+
+Then you can use the [`NatsJsonContextSerializer`](xref:NATS.Client.Core.NatsJsonContextSerializer) to serialize and deserialize messages:
+```csharp
+// Set the custom serializer as the default for the connection.
+var natsOpts = NatsOpts.Default with { Serializer = new NatsJsonContextSerializer(MyJsonContext.Default) };
+
+await using var nats = new NatsConnection(natsOpts);
+
+await using INatsSub<MyData> sub = await nats.SubscribeAsync<MyData>(subject: "foo");
+
+// Flush the the network buffers to make sure the subscription request has been processed.
+await nats.PingAsync();
+
+await nats.PublishAsync<MyData>(subject: "foo", data: new MyData { Id = 1, Name = "bar" });
+
+NatsMsg<MyData?> msg = await sub.Msgs.ReadAsync();
+
+// Outputs 'MyData { Id = 1, Name = bar }'
+Console.WriteLine(msg.Data);
+```
+
+You can also set the serializer for a specific subscription or publish call:
+```csharp
+await using var nats = new NatsConnection();
+
+var natsSubOpts = new NatsSubOpts { Serializer = new NatsJsonContextSerializer(MyJsonContext.Default) };
+await using INatsSub<MyData> sub = await nats.SubscribeAsync<MyData>(subject: "foo", opts: natsSubOpts);
+
+// Flush the the network buffers to make sure the subscription request has been processed.
+await nats.PingAsync();
+
+var natsPubOpts = new NatsPubOpts { Serializer = new NatsJsonContextSerializer(MyJsonContext.Default) };
+await nats.PublishAsync<MyData>(subject: "foo", data: new MyData { Id = 1, Name = "bar" }, opts: natsPubOpts);
+
+NatsMsg<MyData?> msg = await sub.Msgs.ReadAsync();
+
+// Outputs 'MyData { Id = 1, Name = bar }'
+Console.WriteLine(msg.Data);
+```
+
+## Using Custom Serializer
+
+You can also provide your own serializer by implementing the [`INatsSerializer`](xref:NATS.Client.Core.INatsSerializer) interface. This is useful if you need to
+support a custom serialization format or if you need to support multiple serialization formats.
+
+Here is an example of a custom serializer that uses the Google ProtoBuf serializer to serialize and deserialize:
+
+```csharp
+public class MyProtoBufSerializer : INatsSerializer
+{
+    public static readonly INatsSerializer Default = new MyProtoBufSerializer();
+
+    public INatsSerializer? Next => default;
+
+    public void Serialize<T>(IBufferWriter<byte> bufferWriter, T value)
+    {
+        if (value is IMessage message)
+        {
+            message.WriteTo(bufferWriter);
+        }
+        else
+        {
+            throw new NatsException($"Can't serialize {typeof(T)}");
+        }
+    }
+
+    public T? Deserialize<T>(in ReadOnlySequence<byte> buffer)
+    {
+        if (typeof(T) == typeof(Greeting))
+        {
+            return (T)(object)Greeting.Parser.ParseFrom(buffer);
+        }
+
+        throw new NatsException($"Can't deserialize {typeof(T)}");
+    }
+}
+```
+
+You can then use the custom serializer as the default for the connection:
+
+```csharp
+var natsOpts = NatsOpts.Default with { Serializer = MyProtoBufSerializer.Default };
+
+await using var nats = new NatsConnection(natsOpts);
+
+await using var sub = await nats.SubscribeAsync<Greeting>(subject: "foo");
+
+// Flush the the network buffers to make sure the subscription request has been processed.
+await nats.PingAsync();
+
+await nats.PublishAsync(subject: "foo", data: new Greeting { Id = 42, Name = "Marvin" });
+
+var msg = await sub.Msgs.ReadAsync();
+
+// Outputs '{ "id": 42, "name": "Marvin" }'
+Console.WriteLine(msg.Data);
+```
+
+## Using Multiple Serializers (chaining)
+
+You can also chain multiple serializers together to support multiple serialization formats. The first serializer in the
+chain that can handle the data will be used. This is useful if you need to support multiple serialization formats and
+reuse them.
+
+Note that chaining serializers is implemented by convention and not enforced by the [`INatsSerializer`](xref:NATS.Client.Core.INatsSerializer)
+interface since the next serializer would not be exposed to external users of the interface.
+
+Here is an example of a serializer that uses the Google ProtoBuf serializer and the [`NatsJsonContextSerializer`](xref:NATS.Client.Core.NatsJsonContextSerializer) to
+serialize and deserialize messages based on the type:
+
+```csharp
+var serializers = new NatsJsonContextSerializer(MyJsonContext.Default, next: MyProtoBufSerializer.Default);
+var natsOpts = NatsOpts.Default with { Serializer = serializers };
+
+await using var nats = new NatsConnection(natsOpts);
+
+await using var sub1 = await nats.SubscribeAsync<Greeting>(subject: "greet");
+await using var sub2 = await nats.SubscribeAsync<MyData>(subject: "data");
+
+// Flush the the network buffers to make sure the subscription request has been processed.
+await nats.PingAsync();
+
+await nats.PublishAsync(subject: "greet", data: new Greeting { Id = 42, Name = "Marvin" });
+await nats.PublishAsync(subject: "data", data: new MyData { Id = 1, Name = "Bob" });
+
+var msg1 = await sub1.Msgs.ReadAsync();
+var msg2 = await sub2.Msgs.ReadAsync();
+
+// Outputs '{ "id": 42, "name": "Marvin" }'
+Console.WriteLine(msg1.Data);
+
+// Outputs 'MyData { Id = 1, Name = bar }'
+Console.WriteLine(msg2.Data);
+```
+
+## Dealing with Binary Data and Buffers
+
+The default serializer can handle binary data and buffers. This is typically archived by using [`IMemoryOwner<byte>`](https://learn.microsoft.com/dotnet/api/system.buffers.imemoryowner-1)
+implementations. NATS .NET Client provides a [`NatsMemoryOwner<T>`](xref:NATS.Client.Core.NatsMemoryOwner`1) implementation that can be used to allocate buffers.
+The [`NatsMemoryOwner<T>`](xref:NATS.Client.Core.NatsMemoryOwner`1) and [`NatsBufferWriter<T>`](xref:NATS.Client.Core.NatsBufferWriter`1) (adapted from [.NET Community Toolkit](https://learn.microsoft.com/en-us/dotnet/communitytoolkit/high-performance/memoryowner))
+are [`IMemoryOwner<byte>`](https://learn.microsoft.com/dotnet/api/system.buffers.imemoryowner-1) and [`IBufferWriter<T>`](https://learn.microsoft.com/dotnet/api/system.buffers.ibufferwriter-1) implementations that use the [`ArrayPool`](https://learn.microsoft.com/dotnet/api/system.buffers.arraypool-1)
+to allocate buffers. They can be used with the default serializer.
+
+```csharp
+// Same as not specifying a serializer.
+var natsOpts = NatsOpts.Default with { Serializer = NatsDefaultSerializer.Default };
+
+await using var nats = new NatsConnection(natsOpts);
+
+await using var sub = await nats.SubscribeAsync<NatsMemoryOwner<byte>>(subject: "foo");
+
+// Flush the the network buffers to make sure the subscription request has been processed.
+await nats.PingAsync();
+
+// Don't reuse NatsBufferWriter, it's disposed and returned to the pool
+// by the publisher after being written to network.
+var bw = new NatsBufferWriter<byte>();
+var memory = bw.GetMemory(2);
+memory.Span[0] = (byte)'H';
+memory.Span[1] = (byte)'i';
+bw.Advance(2);
+
+await nats.PublishAsync(subject: "foo", data: bw);
+
+var msg = await sub.Msgs.ReadAsync();
+
+// Dispose the memory owner after using it so it can be retunrned to the pool.
+using (var memoryOwner = msg.Data)
+{
+    // Outputs 'Hi'
+    Console.WriteLine(Encoding.ASCII.GetString(memoryOwner.Memory.Span));
+}
+```
+
+Advantage of using [`NatsMemoryOwner<T>`](xref:NATS.Client.Core.NatsMemoryOwner`1) and [`NatsBufferWriter<T>`](xref:NATS.Client.Core.NatsBufferWriter`1) is that they can be used with the default serializer and
+they can be used to allocate buffers from the [`ArrayPool<T>`](https://learn.microsoft.com/dotnet/api/system.buffers.arraypool-1) which can be reused. This is useful if you need to allocate
+buffers for binary data and you want to avoid allocating buffers on for every operation (e.g. `new byte[]`) reducing
+garbage collection pressure. They may also be useful for example, if your subscription may receive messages with
+different formats and the only way to determine the format is by reading the message.

docs/documentation/toc.yml

@@ -27,5 +27,11 @@
 - name: Object Store
   href: object-store/intro.md
 
+- name: Services
+  href: services/intro.md
+
+- name: Serialization
+  href: serialization.md
+
 - name: Updating Documentation
   href: update-docs.md