Skip to content

Commit 153a33d

Browse files
HavretHavret
committed
Add Set support
1 parent a6f83d9 commit 153a33d

12 files changed

+819
-182
lines changed

‎README.md‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -155,26 +155,26 @@ The library includes specialized support for collections when working with:
155155
1. Protocol Buffer message types
156156
2. Primitive types (int, long, string, etc.)
157157

158-
When using `IList<T>` with these types, the library automatically handles serialization/deserialization without requiring wrapper message types. This is particularly useful for Protocol Buffers, where `RepeatedField<T>` typically cannot be serialized as a standalone entity.
158+
When using `IList<T>` or `ISet<T>` with these types, the library automatically handles serialization/deserialization without requiring wrapper message types. This is particularly useful for Protocol Buffers, where `RepeatedField<T>` typically cannot be serialized as a standalone entity.
159159

160160
The serialization format varies depending on the element type:
161161

162162
#### Fixed-Size Types (int, long, etc.)
163163

164164
```
165-
[4 bytes: List length][Contiguous array of serialized elements]
165+
[4 bytes: Collection length][Contiguous array of serialized elements]
166166
```
167167

168168
#### Variable-Size Types (string, protobuf messages)
169169

170170
```
171-
[4 bytes: List length][For each element: [4 bytes: Element size][N bytes: Element data]]
171+
[4 bytes: Collection length][For each element: [4 bytes: Element size][N bytes: Element data]]
172172
```
173173

174174
Example types that work automatically with this support:
175175

176-
- Protocol Buffer message types: `IList<YourProtobufMessage>`
177-
- Primitive types: `IList<int>`, `IList<long>`, `IList<string>`, etc.
176+
- Protocol Buffer message types: `IList<YourProtobufMessage>`, `ISet<YourProtobufMessage>`
177+
- Primitive types: `IList<int>`, `IList<long>`, `IList<string>`, `ISet<int>`, `ISet<string>`, etc.
178178

179179
## Merge Operators
180180

‎src/RocksDb.Extensions/FixedSizeCollectionSerializer.cs‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
using System.Buffers;
2+
3+
namespace RocksDb.Extensions;
4+
5+
/// <summary>
6+
/// Base class for serializing collections of fixed-size elements like primitive types (int, long, etc.)
7+
/// where each element occupies the same number of bytes in memory.
8+
/// </summary>
9+
/// <remarks>
10+
/// The serialized format consists of:
11+
/// - 4 bytes: Collection length (number of elements)
12+
/// - Remaining bytes: Contiguous array of serialized elements
13+
/// </remarks>
14+
/// <typeparam name="TCollection">The collection type (e.g., IList{T}, ISet{T})</typeparam>
15+
/// <typeparam name="TElement">The element type</typeparam>
16+
internal abstract class FixedSizeCollectionSerializer<TCollection, TElement> : ISerializer<TCollection>
17+
where TCollection : ICollection<TElement>
18+
{
19+
private readonly ISerializer<TElement> _scalarSerializer;
20+
21+
protected FixedSizeCollectionSerializer(ISerializer<TElement> scalarSerializer)
22+
{
23+
_scalarSerializer = scalarSerializer;
24+
}
25+
26+
/// <summary>
27+
/// Creates a new collection instance with the specified capacity.
28+
/// </summary>
29+
protected abstract TCollection CreateCollection(int capacity);
30+
31+
/// <summary>
32+
/// Adds an element to the collection.
33+
/// </summary>
34+
protected abstract void AddElement(TCollection collection, TElement element);
35+
36+
public bool TryCalculateSize(ref TCollection value, out int size)
37+
{
38+
size = sizeof(int); // size of the collection
39+
if (value.Count == 0)
40+
{
41+
return true;
42+
}
43+
44+
var referentialElement = value.First();
45+
if (_scalarSerializer.TryCalculateSize(ref referentialElement, out var elementSize))
46+
{
47+
size += value.Count * elementSize;
48+
return true;
49+
}
50+
51+
return false;
52+
}
53+
54+
public void WriteTo(ref TCollection value, ref Span<byte> span)
55+
{
56+
// Write the size of the collection
57+
var slice = span.Slice(0, sizeof(int));
58+
BitConverter.TryWriteBytes(slice, value.Count);
59+
60+
if (value.Count == 0)
61+
{
62+
return;
63+
}
64+
65+
// Write the elements of the collection
66+
int offset = sizeof(int);
67+
var elementSize = (span.Length - offset) / value.Count;
68+
foreach (var item in value)
69+
{
70+
var element = item;
71+
slice = span.Slice(offset, elementSize);
72+
_scalarSerializer.WriteTo(ref element, ref slice);
73+
offset += elementSize;
74+
}
75+
}
76+
77+
public void WriteTo(ref TCollection value, IBufferWriter<byte> buffer)
78+
{
79+
throw new NotImplementedException();
80+
}
81+
82+
public TCollection Deserialize(ReadOnlySpan<byte> buffer)
83+
{
84+
// Read the size of the collection
85+
var slice = buffer.Slice(0, sizeof(int));
86+
var size = BitConverter.ToInt32(slice);
87+
88+
var collection = CreateCollection(size);
89+
90+
if (size == 0)
91+
{
92+
return collection;
93+
}
94+
95+
// Read the elements of the collection
96+
int offset = sizeof(int);
97+
var elementSize = (buffer.Length - offset) / size;
98+
for (int i = 0; i < size; i++)
99+
{
100+
slice = buffer.Slice(offset, elementSize);
101+
var element = _scalarSerializer.Deserialize(slice);
102+
AddElement(collection, element);
103+
offset += elementSize;
104+
}
105+
106+
return collection;
107+
}
108+
}
109+

‎src/RocksDb.Extensions/FixedSizeListSerializer.cs‎

Lines changed: 4 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -1,87 +1,12 @@
1-
using System.Buffers;
2-
31
namespace RocksDb.Extensions;
42

5-
/// <summary>
6-
/// Serializes lists of fixed-size elements like primitive types (int, long, etc.) where each element
7-
/// occupies the same number of bytes in memory. This implementation optimizes for performance by
8-
/// pre-calculating buffer sizes based on element count.
9-
/// </summary>
10-
/// <remarks>
11-
/// Use this serializer when working with lists of primitive types or structs where all elements
12-
/// have identical size. The serialized format consists of:
13-
/// - 4 bytes: List length (number of elements)
14-
/// - Remaining bytes: Contiguous array of serialized elements
15-
/// </remarks>
16-
internal class FixedSizeListSerializer<T> : ISerializer<IList<T>>
3+
internal class FixedSizeListSerializer<T> : FixedSizeCollectionSerializer<IList<T>, T>
174
{
18-
private readonly ISerializer<T> _scalarSerializer;
19-
20-
public FixedSizeListSerializer(ISerializer<T> scalarSerializer)
21-
{
22-
_scalarSerializer = scalarSerializer;
23-
}
24-
25-
public bool TryCalculateSize(ref IList<T> value, out int size)
5+
public FixedSizeListSerializer(ISerializer<T> scalarSerializer) : base(scalarSerializer)
266
{
27-
size = sizeof(int); // size of the list
28-
if (value.Count == 0)
29-
{
30-
return true;
31-
}
32-
33-
var referentialElement = value[0];
34-
if (_scalarSerializer.TryCalculateSize(ref referentialElement, out var elementSize))
35-
{
36-
size += value.Count * elementSize;
37-
return true;
38-
}
39-
40-
return false;
417
}
428

43-
public void WriteTo(ref IList<T> value, ref Span<byte> span)
44-
{
45-
// Write the size of the list
46-
var slice = span.Slice(0, sizeof(int));
47-
BitConverter.TryWriteBytes(slice, value.Count);
48-
49-
// Write the elements of the list
50-
int offset = sizeof(int);
51-
var elementSize = (span.Length - offset) / value.Count;
52-
for (int i = 0; i < value.Count; i++)
53-
{
54-
var element = value[i];
55-
slice = span.Slice(offset, elementSize);
56-
_scalarSerializer.WriteTo(ref element, ref slice);
57-
offset += elementSize;
58-
}
59-
}
9+
protected override IList<T> CreateCollection(int capacity) => new List<T>(capacity);
6010

61-
public void WriteTo(ref IList<T> value, IBufferWriter<byte> buffer)
62-
{
63-
throw new NotImplementedException();
64-
}
65-
66-
public IList<T> Deserialize(ReadOnlySpan<byte> buffer)
67-
{
68-
// Read the size of the list
69-
var slice = buffer.Slice(0, sizeof(int));
70-
var size = BitConverter.ToInt32(slice);
71-
72-
var list = new List<T>(size);
73-
74-
// Read the elements of the list
75-
int offset = sizeof(int);
76-
var elementSize = (buffer.Length - offset) / size;
77-
for (int i = 0; i < size; i++)
78-
{
79-
slice = buffer.Slice(offset, elementSize);
80-
var element = _scalarSerializer.Deserialize(slice);
81-
list.Add(element);
82-
offset += elementSize;
83-
}
84-
85-
return list;
86-
}
11+
protected override void AddElement(IList<T> collection, T element) => collection.Add(element);
8712
}

‎src/RocksDb.Extensions/FixedSizeSetSerializer.cs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
namespace RocksDb.Extensions;
2+
3+
internal class FixedSizeSetSerializer<T> : FixedSizeCollectionSerializer<ISet<T>, T>
4+
{
5+
public FixedSizeSetSerializer(ISerializer<T> scalarSerializer) : base(scalarSerializer)
6+
{
7+
}
8+
9+
protected override ISet<T> CreateCollection(int capacity) => new HashSet<T>(capacity);
10+
11+
protected override void AddElement(ISet<T> collection, T element) => collection.Add(element);
12+
}

‎src/RocksDb.Extensions/RocksDbBuilder.cs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,26 @@ private static ISerializer<T> CreateSerializer<T>(IReadOnlyList<ISerializerFacto
124124
return (ISerializer<T>) Activator.CreateInstance(typeof(VariableSizeListSerializer<>).MakeGenericType(elementType), scalarSerializer)!;
125125
}
126126

127+
if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(ISet<>))
128+
{
129+
var elementType = type.GetGenericArguments()[0];
130+
131+
// Use reflection to call CreateSerializer method with generic type argument
132+
// This is equivalent to calling CreateSerializer<elementType>(serializerFactories)
133+
var scalarSerializer = typeof(RocksDbBuilder).GetMethod(nameof(CreateSerializer), BindingFlags.NonPublic | BindingFlags.Static)
134+
?.MakeGenericMethod(elementType)
135+
.Invoke(null, new object[] { serializerFactories });
136+
137+
if (elementType.IsPrimitive)
138+
{
139+
// Use fixed size set serializer for primitive types
140+
return (ISerializer<T>) Activator.CreateInstance(typeof(FixedSizeSetSerializer<>).MakeGenericType(elementType), scalarSerializer)!;
141+
}
142+
143+
// Use variable size set serializer for non-primitive types
144+
return (ISerializer<T>) Activator.CreateInstance(typeof(VariableSizeSetSerializer<>).MakeGenericType(elementType), scalarSerializer)!;
145+
}
146+
127147
// Handle CollectionOperation<T> for the ListMergeOperator
128148
if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(MergeOperators.CollectionOperation<>))
129149
{

‎src/RocksDb.Extensions/VariableSizeCollectionSerializer.cs‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
using System.Buffers;
2+
3+
namespace RocksDb.Extensions;
4+
5+
/// <summary>
6+
/// Base class for serializing collections containing variable-size elements like strings or complex objects
7+
/// where each element may occupy a different number of bytes when serialized.
8+
/// </summary>
9+
/// <remarks>
10+
/// The serialized format consists of:
11+
/// - 4 bytes: Collection length (number of elements)
12+
/// - For each element:
13+
/// - 4 bytes: Size of the serialized element
14+
/// - N bytes: Serialized element data
15+
/// </remarks>
16+
/// <typeparam name="TCollection">The collection type (e.g., IList{T}, ISet{T})</typeparam>
17+
/// <typeparam name="TElement">The element type</typeparam>
18+
internal abstract class VariableSizeCollectionSerializer<TCollection, TElement> : ISerializer<TCollection>
19+
where TCollection : ICollection<TElement>
20+
{
21+
private readonly ISerializer<TElement> _scalarSerializer;
22+
23+
protected VariableSizeCollectionSerializer(ISerializer<TElement> scalarSerializer)
24+
{
25+
_scalarSerializer = scalarSerializer;
26+
}
27+
28+
/// <summary>
29+
/// Creates a new collection instance with the specified capacity.
30+
/// </summary>
31+
protected abstract TCollection CreateCollection(int capacity);
32+
33+
/// <summary>
34+
/// Adds an element to the collection.
35+
/// </summary>
36+
protected abstract void AddElement(TCollection collection, TElement element);
37+
38+
public bool TryCalculateSize(ref TCollection value, out int size)
39+
{
40+
size = sizeof(int); // size of the collection
41+
if (value.Count == 0)
42+
{
43+
return true;
44+
}
45+
46+
foreach (var item in value)
47+
{
48+
var element = item;
49+
if (_scalarSerializer.TryCalculateSize(ref element, out var elementSize))
50+
{
51+
size += sizeof(int);
52+
size += elementSize;
53+
}
54+
else
55+
{
56+
// Element serializer can't calculate size, so we can't either
57+
size = 0;
58+
return false;
59+
}
60+
}
61+
62+
return true;
63+
}
64+
65+
public void WriteTo(ref TCollection value, ref Span<byte> span)
66+
{
67+
// Write the size of the collection
68+
var slice = span.Slice(0, sizeof(int));
69+
BitConverter.TryWriteBytes(slice, value.Count);
70+
71+
// Write the elements of the collection
72+
int offset = sizeof(int);
73+
foreach (var item in value)
74+
{
75+
var element = item;
76+
if (_scalarSerializer.TryCalculateSize(ref element, out var elementSize))
77+
{
78+
slice = span.Slice(offset, sizeof(int));
79+
BitConverter.TryWriteBytes(slice, elementSize);
80+
offset += sizeof(int);
81+
82+
slice = span.Slice(offset, elementSize);
83+
_scalarSerializer.WriteTo(ref element, ref slice);
84+
offset += elementSize;
85+
}
86+
}
87+
}
88+
89+
public void WriteTo(ref TCollection value, IBufferWriter<byte> buffer)
90+
{
91+
throw new NotImplementedException();
92+
}
93+
94+
public TCollection Deserialize(ReadOnlySpan<byte> buffer)
95+
{
96+
// Read the size of the collection
97+
var slice = buffer.Slice(0, sizeof(int));
98+
var size = BitConverter.ToInt32(slice);
99+
100+
var collection = CreateCollection(size);
101+
102+
// Read the elements of the collection
103+
int offset = sizeof(int);
104+
for (int i = 0; i < size; i++)
105+
{
106+
slice = buffer.Slice(offset, sizeof(int));
107+
var elementSize = BitConverter.ToInt32(slice);
108+
offset += sizeof(int);
109+
110+
slice = buffer.Slice(offset, elementSize);
111+
var element = _scalarSerializer.Deserialize(slice);
112+
AddElement(collection, element);
113+
offset += elementSize;
114+
}
115+
116+
return collection;
117+
}
118+
}
119+

0 commit comments

Comments
 (0)