diff --git a/README.md b/README.md index b5169a5..451547f 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,9 @@ var rocksDbBuilder = builder.Services.AddRocksDb(options => options.SerializerFactories.Add(new SystemTextJsonSerializerFactory()); }); ``` + ### Register your store + Before your store can be used, you need to register it with RocksDb. You can do this as follows: ```csharp @@ -72,6 +74,7 @@ rocksDbBuilder.AddStore("users-store"); This registers an instance of `UsersStore` with RocksDb under the name "users-store". ### Use your store + Once you have registered your store, you can use it to add, get, and remove data from RocksDb. For example: ```csharp @@ -126,4 +129,40 @@ var rocksDbBuilder = builder.Services.AddRocksDb(options => When this option is set to true, the existing database will be deleted on startup and a new one will be created. Note that all data in the existing database will be lost when this option is used. -By default, the `DeleteExistingDatabaseOnStartup` option is set to false to preserve the current behavior of not automatically deleting the database. If you need to ensure a clean start for your application, set this option to true in your configuration. \ No newline at end of file +By default, the `DeleteExistingDatabaseOnStartup` option is set to false to preserve the current behavior of not automatically deleting the database. If you need to ensure a clean start for your application, set this option to true in your configuration. + +## Collections Support + +RocksDb.Extensions provides built-in support for collections across different serialization packages: + +### System.Text.Json and ProtoBufNet + +The `RocksDb.Extensions.System.Text.Json` and `RocksDb.Extensions.ProtoBufNet` packages support collections out of the box. You can use any collection type like `List` or arrays without additional configuration. + +### Protocol Buffers and Primitive Types Support + +The library includes specialized support for collections when working with: + +1. Protocol Buffer message types +2. Primitive types (int, long, string, etc.) + +When using `IList` with these types, the library automatically handles serialization/deserialization without requiring wrapper message types. This is particularly useful for Protocol Buffers, where `RepeatedField` typically cannot be serialized as a standalone entity. + +The serialization format varies depending on the element type: + +#### Fixed-Size Types (int, long, etc.) + +``` +[4 bytes: List length][Contiguous array of serialized elements] +``` + +#### Variable-Size Types (string, protobuf messages) + +``` +[4 bytes: List length][For each element: [4 bytes: Element size][N bytes: Element data]] +``` + +Example types that work automatically with this support: + +- Protocol Buffer message types: `IList` +- Primitive types: `IList`, `IList`, `IList`, etc. \ No newline at end of file diff --git a/src/RocksDb.Extensions/FixedSizeListSerializer.cs b/src/RocksDb.Extensions/FixedSizeListSerializer.cs new file mode 100644 index 0000000..48c9a45 --- /dev/null +++ b/src/RocksDb.Extensions/FixedSizeListSerializer.cs @@ -0,0 +1,87 @@ +using System.Buffers; + +namespace RocksDb.Extensions; + +/// +/// Serializes lists of fixed-size elements like primitive types (int, long, etc.) where each element +/// occupies the same number of bytes in memory. This implementation optimizes for performance by +/// pre-calculating buffer sizes based on element count. +/// +/// +/// Use this serializer when working with lists of primitive types or structs where all elements +/// have identical size. The serialized format consists of: +/// - 4 bytes: List length (number of elements) +/// - Remaining bytes: Contiguous array of serialized elements +/// +internal class FixedSizeListSerializer : ISerializer> +{ + private readonly ISerializer _scalarSerializer; + + public FixedSizeListSerializer(ISerializer scalarSerializer) + { + _scalarSerializer = scalarSerializer; + } + + public bool TryCalculateSize(ref IList value, out int size) + { + size = sizeof(int); // size of the list + if (value.Count == 0) + { + return true; + } + + var referentialElement = value[0]; + if (_scalarSerializer.TryCalculateSize(ref referentialElement, out var elementSize)) + { + size += value.Count * elementSize; + return true; + } + + return false; + } + + public void WriteTo(ref IList value, ref Span span) + { + // Write the size of the list + var slice = span.Slice(0, sizeof(int)); + BitConverter.TryWriteBytes(slice, value.Count); + + // Write the elements of the list + int offset = sizeof(int); + var elementSize = (span.Length - offset) / value.Count; + for (int i = 0; i < value.Count; i++) + { + var element = value[i]; + slice = span.Slice(offset, elementSize); + _scalarSerializer.WriteTo(ref element, ref slice); + offset += elementSize; + } + } + + public void WriteTo(ref IList value, IBufferWriter buffer) + { + throw new NotImplementedException(); + } + + public IList Deserialize(ReadOnlySpan buffer) + { + // Read the size of the list + var slice = buffer.Slice(0, sizeof(int)); + var size = BitConverter.ToInt32(slice); + + var list = new List(size); + + // Read the elements of the list + int offset = sizeof(int); + var elementSize = (buffer.Length - offset) / size; + for (int i = 0; i < size; i++) + { + slice = buffer.Slice(offset, elementSize); + var element = _scalarSerializer.Deserialize(slice); + list.Add(element); + offset += elementSize; + } + + return list; + } +} \ No newline at end of file diff --git a/src/RocksDb.Extensions/RocksDbBuilder.cs b/src/RocksDb.Extensions/RocksDbBuilder.cs index fbb40d2..0d38ef9 100644 --- a/src/RocksDb.Extensions/RocksDbBuilder.cs +++ b/src/RocksDb.Extensions/RocksDbBuilder.cs @@ -1,3 +1,4 @@ +using System.Reflection; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; @@ -51,6 +52,26 @@ private static ISerializer CreateSerializer(IReadOnlyList(); } } + + if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(IList<>)) + { + var elementType = type.GetGenericArguments()[0]; + + // Use reflection to call CreateSerializer method with generic type argument + // This is equivalent to calling CreateSerializer(serializerFactories) + var scalarSerializer = typeof(RocksDbBuilder).GetMethod(nameof(CreateSerializer), BindingFlags.NonPublic | BindingFlags.Static) + ?.MakeGenericMethod(elementType) + .Invoke(null, new object[] { serializerFactories }); + + if (elementType.IsPrimitive) + { + // Use fixed size list serializer for primitive types + return (ISerializer) Activator.CreateInstance(typeof(FixedSizeListSerializer<>).MakeGenericType(elementType), scalarSerializer); + } + + // Use variable size list serializer for non-primitive types + return (ISerializer) Activator.CreateInstance(typeof(VariableSizeListSerializer<>).MakeGenericType(elementType), scalarSerializer); + } throw new InvalidOperationException($"Type {type.FullName} cannot be used as RocksDbStore key/value. " + $"Consider registering {nameof(ISerializerFactory)} that support this type."); diff --git a/src/RocksDb.Extensions/VariableSizeListSerializer.cs b/src/RocksDb.Extensions/VariableSizeListSerializer.cs new file mode 100644 index 0000000..6e51e5c --- /dev/null +++ b/src/RocksDb.Extensions/VariableSizeListSerializer.cs @@ -0,0 +1,100 @@ +using System.Buffers; + +namespace RocksDb.Extensions; + +/// +/// Serializes lists containing variable-size elements like strings or complex objects where each element +/// may occupy a different number of bytes when serialized. +/// +/// +/// Use this serializer for lists containing elements that may have different sizes (strings, nested objects, etc.). +/// The serialized format consists of: +/// - 4 bytes: List length (number of elements) +/// - For each element: +/// - 4 bytes: Size of the serialized element +/// - N bytes: Serialized element data +/// +internal class VariableSizeListSerializer : ISerializer> +{ + private readonly ISerializer _scalarSerializer; + + public VariableSizeListSerializer(ISerializer scalarSerializer) + { + _scalarSerializer = scalarSerializer; + } + + public bool TryCalculateSize(ref IList value, out int size) + { + size = sizeof(int); // size of the list + if (value.Count == 0) + { + return true; + } + + for (int i = 0; i < value.Count; i++) + { + var element = value[i]; + if (_scalarSerializer.TryCalculateSize(ref element, out var elementSize)) + { + size += sizeof(int); + size += elementSize; + } + } + + return true; + } + + public void WriteTo(ref IList value, ref Span span) + { + // Write the size of the list + var slice = span.Slice(0, sizeof(int)); + BitConverter.TryWriteBytes(slice, value.Count); + + // Write the elements of the list + int offset = sizeof(int); + for (int i = 0; i < value.Count; i++) + { + var element = value[i]; + if (_scalarSerializer.TryCalculateSize(ref element, out var elementSize)) + { + slice = span.Slice(offset, sizeof(int)); + BitConverter.TryWriteBytes(slice, elementSize); + offset += sizeof(int); + + slice = span.Slice(offset, elementSize); + _scalarSerializer.WriteTo(ref element, ref slice); + offset += elementSize; + } + } + } + + public void WriteTo(ref IList value, IBufferWriter buffer) + { + throw new NotImplementedException(); + } + + public IList Deserialize(ReadOnlySpan buffer) + { + // Read the size of the list + var slice = buffer.Slice(0, sizeof(int)); + var size = BitConverter.ToInt32(slice); + + var list = new List(size); + + // Read the elements of the list + int offset = sizeof(int); + for (int i = 0; i < size; i++) + { + slice = buffer.Slice(offset, sizeof(int)); + var elementSize = BitConverter.ToInt32(slice); + offset += sizeof(int); + + slice = buffer.Slice(offset, elementSize); + var element = _scalarSerializer.Deserialize(slice); + list.Add(element); + offset += elementSize; + } + + return list; + } +} \ No newline at end of file diff --git a/test/RocksDb.Extensions.Tests/RocksDbStoreWithJsonSerializerTests.cs b/test/RocksDb.Extensions.Tests/RocksDbStoreWithJsonSerializerTests.cs index e75cd5f..6f593e3 100644 --- a/test/RocksDb.Extensions.Tests/RocksDbStoreWithJsonSerializerTests.cs +++ b/test/RocksDb.Extensions.Tests/RocksDbStoreWithJsonSerializerTests.cs @@ -12,7 +12,7 @@ public class RocksDbStoreWithJsonSerializerTests public void should_put_and_retrieve_data_from_store() { // Arrange - using var testFixture = CreateTestFixture(); + using var testFixture = CreateTestFixture(); var store = testFixture.GetStore>(); var cacheKey = new ProtoNetCacheKey @@ -38,7 +38,7 @@ public void should_put_and_retrieve_data_from_store() public void should_put_and_remove_data_from_store() { // Arrange - using var testFixture = CreateTestFixture(); + using var testFixture = CreateTestFixture(); var store = testFixture.GetStore>(); var cacheKey = new ProtoNetCacheKey @@ -64,7 +64,7 @@ public void should_put_and_remove_data_from_store() public void should_put_range_of_data_to_store() { // Arrange - using var testFixture = CreateTestFixture(); + using var testFixture = CreateTestFixture(); var store = testFixture.GetStore>(); // Act @@ -90,7 +90,7 @@ public void should_put_range_of_data_to_store() public void should_put_range_of_data_to_store_when_key_is_derived_from_value() { // Arrange - using var testFixture = CreateTestFixture(); + using var testFixture = CreateTestFixture(); var store = testFixture.GetStore>(); // Act @@ -108,17 +108,47 @@ public void should_put_range_of_data_to_store_when_key_is_derived_from_value() cacheValue.ShouldBeEquivalentTo(expectedCacheValue); } } + + [Test] + public void should_put_and_retrieve_data_with_lists_from_store() + { + // Arrange + using var testFixture = CreateTestFixture, IList>(); + var store = testFixture.GetStore, IList>>(); + + // Act + var cacheKey = Enumerable.Range(0, 100) + .Select(x => new ProtoNetCacheKey + { + Id = x, + }) + .ToList(); + + var cacheValue = Enumerable.Range(0, 100) + .Select(x => new ProtoNetCacheValue + { + Id = x, + Value = $"value-{x}", + }) + .ToList(); + + store.Put(cacheKey, cacheValue); - private static TestFixture CreateTestFixture() + store.HasKey(cacheKey).ShouldBeTrue(); + store.TryGet(cacheKey, out var value).ShouldBeTrue(); + value.ShouldBeEquivalentTo(cacheValue); + } + + private static TestFixture CreateTestFixture() { var testFixture = TestFixture.Create(rockDb => { - _ = rockDb.AddStore>("my-store"); + _ = rockDb.AddStore>("my-store"); }, options => { options.SerializerFactories.Clear(); options.SerializerFactories.Add(new SystemTextJsonSerializerFactory()); }); return testFixture; - } + } } diff --git a/test/RocksDb.Extensions.Tests/RocksDbStoreWithPrimitiveSerializerTests.cs b/test/RocksDb.Extensions.Tests/RocksDbStoreWithPrimitiveSerializerTests.cs index 8718300..daafc59 100644 --- a/test/RocksDb.Extensions.Tests/RocksDbStoreWithPrimitiveSerializerTests.cs +++ b/test/RocksDb.Extensions.Tests/RocksDbStoreWithPrimitiveSerializerTests.cs @@ -262,6 +262,42 @@ public void should_put_range_of_data_to_store_using_bool_types() cacheValue.ShouldBe(cacheValues[index]); } } + + [Test] + public void should_put_and_retrieve_data_from_store_using_list_of_primitive_types() + { + // Arrange + using var testFixture = CreateTestFixture, IList>(); + var store = testFixture.GetStore, IList>>(); + + // Act + var key = new List { 1, 2 }; + var value = new List { 3, 4 }; + store.Put(key, value); + + // Assert + store.HasKey(key).ShouldBeTrue(); + store.TryGet(key, out var cacheValue).ShouldBeTrue(); + cacheValue.ShouldBe(value); + } + + [Test] + public void should_put_and_retrieve_data_from_store_using_list_of_strings() + { + // Arrange + using var testFixture = CreateTestFixture, IList>(); + var store = testFixture.GetStore, IList>>(); + var key = new List { "key1", "key2", string.Empty, "key3" }; + var value = new List { "value1", string.Empty, "value2", "value3" }; + + // Act + store.Put(key, value); + + // Assert + store.HasKey(key).ShouldBeTrue(); + store.TryGet(key, out var retrievedValue).ShouldBeTrue(); + retrievedValue.ShouldBe(value); + } private static TestFixture CreateTestFixture() { diff --git a/test/RocksDb.Extensions.Tests/RocksDbStoreWithProtoBufNetSerializerTests.cs b/test/RocksDb.Extensions.Tests/RocksDbStoreWithProtoBufNetSerializerTests.cs index 53a1e47..3196a8d 100644 --- a/test/RocksDb.Extensions.Tests/RocksDbStoreWithProtoBufNetSerializerTests.cs +++ b/test/RocksDb.Extensions.Tests/RocksDbStoreWithProtoBufNetSerializerTests.cs @@ -13,7 +13,7 @@ public class RocksDbStoreWithProtoBufNetSerializerTests public void should_put_and_retrieve_data_from_store() { // Arrange - using var testFixture = CreateTestFixture(); + using var testFixture = CreateTestFixture(); var store = testFixture.GetStore>(); var cacheKey = new ProtoNetCacheKey @@ -39,7 +39,7 @@ public void should_put_and_retrieve_data_from_store() public void should_put_and_remove_data_from_store() { // Arrange - using var testFixture = CreateTestFixture(); + using var testFixture = CreateTestFixture(); var store = testFixture.GetStore>(); var cacheKey = new ProtoNetCacheKey @@ -65,7 +65,7 @@ public void should_put_and_remove_data_from_store() public void should_put_range_of_data_to_store() { // Arrange - using var testFixture = CreateTestFixture(); + using var testFixture = CreateTestFixture(); var store = testFixture.GetStore>(); // Act @@ -91,7 +91,7 @@ public void should_put_range_of_data_to_store() public void should_put_range_of_data_when_key_is_derived_from_value() { // Arrange - using var testFixture = CreateTestFixture(); + using var testFixture = CreateTestFixture(); var store = testFixture.GetStore>(); // Act @@ -109,12 +109,42 @@ public void should_put_range_of_data_when_key_is_derived_from_value() cacheValue.ShouldBeEquivalentTo(expectedCacheValue); } } + + [Test] + public void should_put_and_retrieve_data_with_lists_from_store() + { + // Arrange + using var testFixture = CreateTestFixture, IList>(); + var store = testFixture.GetStore, IList>>(); + + // Act + var cacheKey = Enumerable.Range(0, 100) + .Select(x => new ProtoNetCacheKey + { + Id = x, + }) + .ToList(); + + var cacheValue = Enumerable.Range(0, 100) + .Select(x => new ProtoNetCacheValue + { + Id = x, + Value = $"value-{x}", + }) + .ToList(); + + store.Put(cacheKey, cacheValue); - private static TestFixture CreateTestFixture() + store.HasKey(cacheKey).ShouldBeTrue(); + store.TryGet(cacheKey, out var value).ShouldBeTrue(); + value.ShouldBeEquivalentTo(cacheValue); + } + + private static TestFixture CreateTestFixture() { var testFixture = TestFixture.Create(rockDb => { - _ = rockDb.AddStore>("my-store"); + _ = rockDb.AddStore>("my-store"); }, options => { options.SerializerFactories.Clear(); diff --git a/test/RocksDb.Extensions.Tests/RocksDbStoreWithProtobufSerializerTests.cs b/test/RocksDb.Extensions.Tests/RocksDbStoreWithProtobufSerializerTests.cs index 60b9c42..d8afcc6 100644 --- a/test/RocksDb.Extensions.Tests/RocksDbStoreWithProtobufSerializerTests.cs +++ b/test/RocksDb.Extensions.Tests/RocksDbStoreWithProtobufSerializerTests.cs @@ -12,7 +12,7 @@ public class RocksDbStoreWithProtobufSerializerTests public void should_put_and_retrieve_data_from_store(int payloadSize) { // Arrange - using var testFixture = CreateTestFixture(); + using var testFixture = CreateTestFixture(); var store = testFixture.GetStore>(); var cacheKeys = Enumerable.Range(0, 100) @@ -42,7 +42,7 @@ public void should_put_and_retrieve_data_from_store(int payloadSize) public void should_put_and_remove_data_from_store(int payloadSize) { // Arrange - using var testFixture = CreateTestFixture(); + using var testFixture = CreateTestFixture(); var store = testFixture.GetStore>(); var cacheKeys = Enumerable.Range(0, 100) @@ -71,7 +71,7 @@ public void should_put_and_remove_data_from_store(int payloadSize) public void should_put_range_of_data_to_store() { // Arrange - using var testFixture = CreateTestFixture(); + using var testFixture = CreateTestFixture(); var store = testFixture.GetStore>(); // Act @@ -97,7 +97,7 @@ public void should_put_range_of_data_to_store() public void should_retrieve_all_elements_from_store() { // Arrange - using var testFixture = CreateTestFixture(); + using var testFixture = CreateTestFixture(); var store = testFixture.GetStore>(); var cacheKeys = Enumerable.Range(0, 100) @@ -121,7 +121,7 @@ public void should_retrieve_all_elements_from_store() public void should_put_range_of_data_when_key_is_derived_from_value() { // Arrange - using var testFixture = CreateTestFixture(); + using var testFixture = CreateTestFixture(); var store = testFixture.GetStore>(); // Act @@ -139,17 +139,47 @@ public void should_put_range_of_data_when_key_is_derived_from_value() cacheValue.ShouldBeEquivalentTo(expectedCacheValue); } } + + [Test] + public void should_put_and_retrieve_data_with_lists_from_store() + { + // Arrange + using var testFixture = CreateTestFixture, IList>(); + var store = testFixture.GetStore, IList>>(); - private static TestFixture CreateTestFixture() + // Act + var cacheKey = Enumerable.Range(0, 100) + .Select(x => new CacheKey + { + Id = x, + }) + .ToList(); + + var cacheValue = Enumerable.Range(0, 100) + .Select(x => new CacheValue + { + Id = x, + Value = $"value-{x}", + }) + .ToList(); + + store.Put(cacheKey, cacheValue); + + store.HasKey(cacheKey).ShouldBeTrue(); + store.TryGet(cacheKey, out var value).ShouldBeTrue(); + value.ShouldBeEquivalentTo(cacheValue); + } + + private static TestFixture CreateTestFixture() { var testFixture = TestFixture.Create(rockDb => { - _ = rockDb.AddStore>("my-store"); + _ = rockDb.AddStore>("my-store"); }, options => { options.SerializerFactories.Clear(); options.SerializerFactories.Add(new ProtobufSerializerFactory()); }); return testFixture; - } + } }