Skip to content

Commit 2e735cf

Browse files
HavretHavret
committed
Add serialization for primitive lists and protobuf lists
1 parent 815e7e2 commit 2e735cf

File tree

8 files changed

+395
-22
lines changed

8 files changed

+395
-22
lines changed

‎README.md‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,9 @@ var rocksDbBuilder = builder.Services.AddRocksDb(options =>
6262
options.SerializerFactories.Add(new SystemTextJsonSerializerFactory());
6363
});
6464
```
65+
6566
### Register your store
67+
6668
Before your store can be used, you need to register it with RocksDb. You can do this as follows:
6769

6870
```csharp
@@ -72,6 +74,7 @@ rocksDbBuilder.AddStore<string, User, UsersStore>("users-store");
7274
This registers an instance of `UsersStore` with RocksDb under the name "users-store".
7375

7476
### Use your store
77+
7578
Once you have registered your store, you can use it to add, get, and remove data from RocksDb. For example:
7679

7780
```csharp
@@ -126,4 +129,40 @@ var rocksDbBuilder = builder.Services.AddRocksDb(options =>
126129

127130
When this option is set to true, the existing database will be deleted on startup and a new one will be created. Note that all data in the existing database will be lost when this option is used.
128131

129-
By default, the `DeleteExistingDatabaseOnStartup` option is set to false to preserve the current behavior of not automatically deleting the database. If you need to ensure a clean start for your application, set this option to true in your configuration.
132+
By default, the `DeleteExistingDatabaseOnStartup` option is set to false to preserve the current behavior of not automatically deleting the database. If you need to ensure a clean start for your application, set this option to true in your configuration.
133+
134+
## Collections Support
135+
136+
RocksDb.Extensions provides built-in support for collections across different serialization packages:
137+
138+
### System.Text.Json and ProtoBufNet
139+
140+
The `RocksDb.Extensions.System.Text.Json` and `RocksDb.Extensions.ProtoBufNet` packages support collections out of the box. You can use any collection type like `List<T>` or arrays without additional configuration.
141+
142+
### Protocol Buffers and Primitive Types Support
143+
144+
The library includes specialized support for collections when working with:
145+
146+
1. Protocol Buffer message types
147+
2. Primitive types (int, long, string, etc.)
148+
149+
When using `IList<T>` with these types, the library automatically handles serialization/deserialization without requiring wrapper message types. This is particularly useful for Protocol Buffers, where `RepeatedField<T>` typically cannot be serialized as a standalone entity.
150+
151+
The serialization format varies depending on the element type:
152+
153+
#### Fixed-Size Types (int, long, etc.)
154+
155+
```
156+
[4 bytes: List length][Contiguous array of serialized elements]
157+
```
158+
159+
#### Variable-Size Types (string, protobuf messages)
160+
161+
```
162+
[4 bytes: List length][For each element: [4 bytes: Element size][N bytes: Element data]]
163+
```
164+
165+
Example types that work automatically with this support:
166+
167+
- Protocol Buffer message types: `IList<YourProtobufMessage>`
168+
- Primitive types: `IList<int>`, `IList<long>`, `IList<string>`, etc.

‎src/RocksDb.Extensions/FixedSizeListSerializer.cs‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
using System.Buffers;
2+
3+
namespace RocksDb.Extensions;
4+
5+
/// <summary>
6+
/// Serializes lists of fixed-size elements like primitive types (int, long, etc.) where each element
7+
/// occupies the same number of bytes in memory. This implementation optimizes for performance by
8+
/// pre-calculating buffer sizes based on element count.
9+
/// </summary>
10+
/// <remarks>
11+
/// Use this serializer when working with lists of primitive types or structs where all elements
12+
/// have identical size. The serialized format consists of:
13+
/// - 4 bytes: List length (number of elements)
14+
/// - Remaining bytes: Contiguous array of serialized elements
15+
/// </remarks>
16+
internal class FixedSizeListSerializer<T> : ISerializer<IList<T>>
17+
{
18+
private readonly ISerializer<T> _scalarSerializer;
19+
20+
public FixedSizeListSerializer(ISerializer<T> scalarSerializer)
21+
{
22+
_scalarSerializer = scalarSerializer;
23+
}
24+
25+
public bool TryCalculateSize(ref IList<T> value, out int size)
26+
{
27+
size = sizeof(int); // size of the list
28+
if (value.Count == 0)
29+
{
30+
return true;
31+
}
32+
33+
var referentialElement = value[0];
34+
if (_scalarSerializer.TryCalculateSize(ref referentialElement, out var elementSize))
35+
{
36+
size += value.Count * elementSize;
37+
return true;
38+
}
39+
40+
return false;
41+
}
42+
43+
public void WriteTo(ref IList<T> value, ref Span<byte> span)
44+
{
45+
// Write the size of the list
46+
var slice = span.Slice(0, sizeof(int));
47+
BitConverter.TryWriteBytes(slice, value.Count);
48+
49+
// Write the elements of the list
50+
int offset = sizeof(int);
51+
var elementSize = (span.Length - offset) / value.Count;
52+
for (int i = 0; i < value.Count; i++)
53+
{
54+
var element = value[i];
55+
slice = span.Slice(offset, elementSize);
56+
_scalarSerializer.WriteTo(ref element, ref slice);
57+
offset += elementSize;
58+
}
59+
}
60+
61+
public void WriteTo(ref IList<T> value, IBufferWriter<byte> buffer)
62+
{
63+
throw new NotImplementedException();
64+
}
65+
66+
public IList<T> Deserialize(ReadOnlySpan<byte> buffer)
67+
{
68+
// Read the size of the list
69+
var slice = buffer.Slice(0, sizeof(int));
70+
var size = BitConverter.ToInt32(slice);
71+
72+
var list = new List<T>(size);
73+
74+
// Read the elements of the list
75+
int offset = sizeof(int);
76+
var elementSize = (buffer.Length - offset) / size;
77+
for (int i = 0; i < size; i++)
78+
{
79+
slice = buffer.Slice(offset, elementSize);
80+
var element = _scalarSerializer.Deserialize(slice);
81+
list.Add(element);
82+
offset += elementSize;
83+
}
84+
85+
return list;
86+
}
87+
}

‎src/RocksDb.Extensions/RocksDbBuilder.cs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
using System.Reflection;
12
using Microsoft.Extensions.DependencyInjection;
23
using Microsoft.Extensions.Options;
34

@@ -51,6 +52,26 @@ private static ISerializer<T> CreateSerializer<T>(IReadOnlyList<ISerializerFacto
5152
return serializerFactory.CreateSerializer<T>();
5253
}
5354
}
55+
56+
if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(IList<>))
57+
{
58+
var elementType = type.GetGenericArguments()[0];
59+
60+
// Use reflection to call CreateSerializer method with generic type argument
61+
// This is equivalent to calling CreateSerializer<elementType>(serializerFactories)
62+
var scalarSerializer = typeof(RocksDbBuilder).GetMethod(nameof(CreateSerializer), BindingFlags.NonPublic | BindingFlags.Static)
63+
?.MakeGenericMethod(elementType)
64+
.Invoke(null, new object[] { serializerFactories });
65+
66+
if (elementType.IsPrimitive)
67+
{
68+
// Use fixed size list serializer for primitive types
69+
return (ISerializer<T>) Activator.CreateInstance(typeof(FixedSizeListSerializer<>).MakeGenericType(elementType), scalarSerializer);
70+
}
71+
72+
// Use variable size list serializer for non-primitive types
73+
return (ISerializer<T>) Activator.CreateInstance(typeof(VariableSizeListSerializer<>).MakeGenericType(elementType), scalarSerializer);
74+
}
5475

5576
throw new InvalidOperationException($"Type {type.FullName} cannot be used as RocksDbStore key/value. " +
5677
$"Consider registering {nameof(ISerializerFactory)} that support this type.");

‎src/RocksDb.Extensions/VariableSizeListSerializer.cs‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
using System.Buffers;
2+
3+
namespace RocksDb.Extensions;
4+
5+
/// <summary>
6+
/// Serializes lists containing variable-size elements like strings or complex objects where each element
7+
/// may occupy a different number of bytes when serialized.
8+
/// </summary>
9+
/// <remarks>
10+
/// Use this serializer for lists containing elements that may have different sizes (strings, nested objects, etc.).
11+
/// The serialized format consists of:
12+
/// - 4 bytes: List length (number of elements)
13+
/// - For each element:
14+
/// - 4 bytes: Size of the serialized element
15+
/// - N bytes: Serialized element data
16+
/// </remarks>
17+
internal class VariableSizeListSerializer<T> : ISerializer<IList<T>>
18+
{
19+
private readonly ISerializer<T> _scalarSerializer;
20+
21+
public VariableSizeListSerializer(ISerializer<T> scalarSerializer)
22+
{
23+
_scalarSerializer = scalarSerializer;
24+
}
25+
26+
public bool TryCalculateSize(ref IList<T> value, out int size)
27+
{
28+
size = sizeof(int); // size of the list
29+
if (value.Count == 0)
30+
{
31+
return true;
32+
}
33+
34+
for (int i = 0; i < value.Count; i++)
35+
{
36+
var element = value[i];
37+
if (_scalarSerializer.TryCalculateSize(ref element, out var elementSize))
38+
{
39+
size += sizeof(int);
40+
size += elementSize;
41+
}
42+
}
43+
44+
return true;
45+
}
46+
47+
public void WriteTo(ref IList<T> value, ref Span<byte> span)
48+
{
49+
// Write the size of the list
50+
var slice = span.Slice(0, sizeof(int));
51+
BitConverter.TryWriteBytes(slice, value.Count);
52+
53+
// Write the elements of the list
54+
int offset = sizeof(int);
55+
for (int i = 0; i < value.Count; i++)
56+
{
57+
var element = value[i];
58+
if (_scalarSerializer.TryCalculateSize(ref element, out var elementSize))
59+
{
60+
slice = span.Slice(offset, sizeof(int));
61+
BitConverter.TryWriteBytes(slice, elementSize);
62+
offset += sizeof(int);
63+
64+
slice = span.Slice(offset, elementSize);
65+
_scalarSerializer.WriteTo(ref element, ref slice);
66+
offset += elementSize;
67+
}
68+
}
69+
}
70+
71+
public void WriteTo(ref IList<T> value, IBufferWriter<byte> buffer)
72+
{
73+
throw new NotImplementedException();
74+
}
75+
76+
public IList<T> Deserialize(ReadOnlySpan<byte> buffer)
77+
{
78+
// Read the size of the list
79+
var slice = buffer.Slice(0, sizeof(int));
80+
var size = BitConverter.ToInt32(slice);
81+
82+
var list = new List<T>(size);
83+
84+
// Read the elements of the list
85+
int offset = sizeof(int);
86+
for (int i = 0; i < size; i++)
87+
{
88+
slice = buffer.Slice(offset, sizeof(int));
89+
var elementSize = BitConverter.ToInt32(slice);
90+
offset += sizeof(int);
91+
92+
slice = buffer.Slice(offset, elementSize);
93+
var element = _scalarSerializer.Deserialize(slice);
94+
list.Add(element);
95+
offset += elementSize;
96+
}
97+
98+
return list;
99+
}
100+
}

‎test/RocksDb.Extensions.Tests/RocksDbStoreWithJsonSerializerTests.cs‎

Lines changed: 37 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ public class RocksDbStoreWithJsonSerializerTests
1212
public void should_put_and_retrieve_data_from_store()
1313
{
1414
// Arrange
15-
using var testFixture = CreateTestFixture();
15+
using var testFixture = CreateTestFixture<ProtoNetCacheKey, ProtoNetCacheValue>();
1616

1717
var store = testFixture.GetStore<RocksDbGenericStore<ProtoNetCacheKey, ProtoNetCacheValue>>();
1818
var cacheKey = new ProtoNetCacheKey
@@ -38,7 +38,7 @@ public void should_put_and_retrieve_data_from_store()
3838
public void should_put_and_remove_data_from_store()
3939
{
4040
// Arrange
41-
using var testFixture = CreateTestFixture();
41+
using var testFixture = CreateTestFixture<ProtoNetCacheKey, ProtoNetCacheValue>();
4242

4343
var store = testFixture.GetStore<RocksDbGenericStore<ProtoNetCacheKey, ProtoNetCacheValue>>();
4444
var cacheKey = new ProtoNetCacheKey
@@ -64,7 +64,7 @@ public void should_put_and_remove_data_from_store()
6464
public void should_put_range_of_data_to_store()
6565
{
6666
// Arrange
67-
using var testFixture = CreateTestFixture();
67+
using var testFixture = CreateTestFixture<ProtoNetCacheKey, ProtoNetCacheValue>();
6868
var store = testFixture.GetStore<RocksDbGenericStore<ProtoNetCacheKey, ProtoNetCacheValue>>();
6969

7070
// Act
@@ -90,7 +90,7 @@ public void should_put_range_of_data_to_store()
9090
public void should_put_range_of_data_to_store_when_key_is_derived_from_value()
9191
{
9292
// Arrange
93-
using var testFixture = CreateTestFixture();
93+
using var testFixture = CreateTestFixture<ProtoNetCacheKey, ProtoNetCacheValue>();
9494
var store = testFixture.GetStore<RocksDbGenericStore<ProtoNetCacheKey, ProtoNetCacheValue>>();
9595

9696
// Act
@@ -108,17 +108,47 @@ public void should_put_range_of_data_to_store_when_key_is_derived_from_value()
108108
cacheValue.ShouldBeEquivalentTo(expectedCacheValue);
109109
}
110110
}
111+
112+
[Test]
113+
public void should_put_and_retrieve_data_with_lists_from_store()
114+
{
115+
// Arrange
116+
using var testFixture = CreateTestFixture<IList<ProtoNetCacheKey>, IList<ProtoNetCacheValue>>();
117+
var store = testFixture.GetStore<RocksDbGenericStore<IList<ProtoNetCacheKey>, IList<ProtoNetCacheValue>>>();
118+
119+
// Act
120+
var cacheKey = Enumerable.Range(0, 100)
121+
.Select(x => new ProtoNetCacheKey
122+
{
123+
Id = x,
124+
})
125+
.ToList();
126+
127+
var cacheValue = Enumerable.Range(0, 100)
128+
.Select(x => new ProtoNetCacheValue
129+
{
130+
Id = x,
131+
Value = $"value-{x}",
132+
})
133+
.ToList();
134+
135+
store.Put(cacheKey, cacheValue);
111136

112-
private static TestFixture CreateTestFixture()
137+
store.HasKey(cacheKey).ShouldBeTrue();
138+
store.TryGet(cacheKey, out var value).ShouldBeTrue();
139+
value.ShouldBeEquivalentTo(cacheValue);
140+
}
141+
142+
private static TestFixture CreateTestFixture<TKey, TValue>()
113143
{
114144
var testFixture = TestFixture.Create(rockDb =>
115145
{
116-
_ = rockDb.AddStore<ProtoNetCacheKey, ProtoNetCacheValue, RocksDbGenericStore<ProtoNetCacheKey, ProtoNetCacheValue>>("my-store");
146+
_ = rockDb.AddStore<TKey, TValue, RocksDbGenericStore<TKey, TValue>>("my-store");
117147
}, options =>
118148
{
119149
options.SerializerFactories.Clear();
120150
options.SerializerFactories.Add(new SystemTextJsonSerializerFactory());
121151
});
122152
return testFixture;
123-
}
153+
}
124154
}

0 commit comments

Comments
 (0)