From 6f77e2b1c93c0acdc55a9422457956a2e92dbf4c Mon Sep 17 00:00:00 2001 From: Havret Date: Thu, 4 Dec 2025 09:54:31 +0100 Subject: [PATCH 01/19] WIP --- src/RocksDb.Extensions/IMergeOperator.cs | 39 +++++++ src/RocksDb.Extensions/IRocksDbAccessor.cs | 1 + src/RocksDb.Extensions/IRocksDbBuilder.cs | 22 ++++ src/RocksDb.Extensions/IRocksDbStore.cs | 19 ++++ src/RocksDb.Extensions/MergeOperatorConfig.cs | 29 +++++ .../MergeOperators/Int64AddMergeOperator.cs | 46 ++++++++ .../MergeOperators/ListAppendMergeOperator.cs | 60 ++++++++++ .../RocksDb.Extensions.csproj | 2 +- src/RocksDb.Extensions/RocksDbAccessor.cs | 70 ++++++++++++ src/RocksDb.Extensions/RocksDbBuilder.cs | 103 +++++++++++++++++- src/RocksDb.Extensions/RocksDbContext.cs | 50 ++++++++- src/RocksDb.Extensions/RocksDbOptions.cs | 5 + 12 files changed, 439 insertions(+), 7 deletions(-) create mode 100644 src/RocksDb.Extensions/IMergeOperator.cs create mode 100644 src/RocksDb.Extensions/MergeOperatorConfig.cs create mode 100644 src/RocksDb.Extensions/MergeOperators/Int64AddMergeOperator.cs create mode 100644 src/RocksDb.Extensions/MergeOperators/ListAppendMergeOperator.cs diff --git a/src/RocksDb.Extensions/IMergeOperator.cs b/src/RocksDb.Extensions/IMergeOperator.cs new file mode 100644 index 0000000..5453255 --- /dev/null +++ b/src/RocksDb.Extensions/IMergeOperator.cs @@ -0,0 +1,39 @@ +namespace RocksDb.Extensions; + +/// +/// Defines a merge operator for RocksDB that enables atomic read-modify-write operations. +/// Merge operators allow efficient updates without requiring a separate read before write, +/// which is particularly useful for counters, list appends, set unions, and other accumulative operations. +/// +/// The type of the value being merged. +public interface IMergeOperator +{ + /// + /// Gets the name of the merge operator. This name is stored in the database + /// and must remain consistent across database opens. + /// + string Name { get; } + + /// + /// Performs a full merge of the existing value with one or more operands. + /// Called when a Get operation encounters merge operands and needs to produce the final value. + /// + /// The key being merged. + /// The existing value in the database, or null if no value exists. + /// The list of merge operands to apply, in order. + /// The merged value. + TValue FullMerge(ReadOnlySpan key, TValue? existingValue, IReadOnlyList operands); + + /// + /// Performs a partial merge of multiple operands without the existing value. + /// Called during compaction to combine multiple merge operands into a single operand. + /// This is an optimization that reduces the number of operands that need to be stored. + /// + /// The key being merged. + /// The list of merge operands to combine, in order. + /// + /// The combined operand if partial merge is possible; otherwise, null to indicate + /// that partial merge is not supported and operands should be kept separate. + /// + TValue? PartialMerge(ReadOnlySpan key, IReadOnlyList operands); +} diff --git a/src/RocksDb.Extensions/IRocksDbAccessor.cs b/src/RocksDb.Extensions/IRocksDbAccessor.cs index e3c0b3f..90d3710 100644 --- a/src/RocksDb.Extensions/IRocksDbAccessor.cs +++ b/src/RocksDb.Extensions/IRocksDbAccessor.cs @@ -23,6 +23,7 @@ public interface IRocksDbAccessor bool HasKey(TKey key); void Clear(); int Count(); + void Merge(TKey key, TValue operand); } #pragma warning restore CS1591 \ No newline at end of file diff --git a/src/RocksDb.Extensions/IRocksDbBuilder.cs b/src/RocksDb.Extensions/IRocksDbBuilder.cs index 7d1124c..861bac3 100644 --- a/src/RocksDb.Extensions/IRocksDbBuilder.cs +++ b/src/RocksDb.Extensions/IRocksDbBuilder.cs @@ -22,4 +22,26 @@ public interface IRocksDbBuilder /// Use GetRequiredKeyedService<TStore>(columnFamily) to retrieve a specific store instance. /// IRocksDbBuilder AddStore(string columnFamily) where TStore : RocksDbStore; + + /// + /// Adds a RocksDB store to the builder for the specified column family with a merge operator. + /// + /// The name of the column family to associate with the store. + /// The merge operator to use for atomic read-modify-write operations. + /// The type of the store's key. + /// The type of the store's value. + /// The type of the store to add. + /// The builder instance for method chaining. + /// Thrown if the specified column family is already registered. + /// + /// The type must be a concrete implementation of the abstract class + /// . Each store is registered uniquely based on its column family name. + /// + /// The merge operator enables efficient atomic updates without requiring a separate read operation. + /// This is useful for counters, list appends, set unions, and other accumulative operations. + /// + /// Stores can also be resolved as keyed services using their associated column family name. + /// Use GetRequiredKeyedService<TStore>(columnFamily) to retrieve a specific store instance. + /// + IRocksDbBuilder AddStore(string columnFamily, IMergeOperator mergeOperator) where TStore : RocksDbStore; } \ No newline at end of file diff --git a/src/RocksDb.Extensions/IRocksDbStore.cs b/src/RocksDb.Extensions/IRocksDbStore.cs index c39f6b9..ea7af2d 100644 --- a/src/RocksDb.Extensions/IRocksDbStore.cs +++ b/src/RocksDb.Extensions/IRocksDbStore.cs @@ -107,4 +107,23 @@ public abstract class RocksDbStore /// /// An enumerable collection of all the keys in the store. public IEnumerable GetAllKeys() => _rocksDbAccessor.GetAllKeys(); + + /// + /// Performs an atomic merge operation on the value associated with the specified key. + /// This operation uses RocksDB's merge operator to combine the operand with the existing value + /// without requiring a separate read operation, which is more efficient than Get+Put for + /// accumulative operations like counters, list appends, or set unions. + /// + /// The key to merge the operand with. + /// The operand to merge with the existing value. + /// + /// This method requires a merge operator to be configured for the store's column family. + /// If no merge operator is configured, RocksDB will throw an error when reading the value. + /// + /// Common use cases include: + /// - Counters: Merge operand represents the increment value + /// - Lists: Merge operand represents items to append + /// - Sets: Merge operand represents items to add to the set + /// + protected void Merge(TKey key, TValue operand) => _rocksDbAccessor.Merge(key, operand); } diff --git a/src/RocksDb.Extensions/MergeOperatorConfig.cs b/src/RocksDb.Extensions/MergeOperatorConfig.cs new file mode 100644 index 0000000..dc5f2c0 --- /dev/null +++ b/src/RocksDb.Extensions/MergeOperatorConfig.cs @@ -0,0 +1,29 @@ +using RocksDbSharp; + +namespace RocksDb.Extensions; + +/// +/// Internal configuration for a merge operator associated with a column family. +/// +internal class MergeOperatorConfig +{ + /// + /// Gets the name of the merge operator. + /// + public required string Name { get; init; } + + /// + /// Gets the full merge callback delegate. + /// + public required MergeOperator.FullMergeImpl FullMerge { get; init; } + + /// + /// Gets the partial merge callback delegate. + /// + public required MergeOperator.PartialMergeImpl PartialMerge { get; init; } + + /// + /// Gets the value serializer for deserializing and serializing values. + /// + public required object ValueSerializer { get; init; } +} diff --git a/src/RocksDb.Extensions/MergeOperators/Int64AddMergeOperator.cs b/src/RocksDb.Extensions/MergeOperators/Int64AddMergeOperator.cs new file mode 100644 index 0000000..38ec7d2 --- /dev/null +++ b/src/RocksDb.Extensions/MergeOperators/Int64AddMergeOperator.cs @@ -0,0 +1,46 @@ +namespace RocksDb.Extensions.MergeOperators; + +/// +/// A merge operator that adds Int64 values together. +/// Useful for implementing atomic counters. +/// +/// +/// +/// public class CounterStore : RocksDbStore<string, long> +/// { +/// public CounterStore(IRocksDbAccessor<string, long> accessor) : base(accessor) { } +/// +/// public void Increment(string key, long delta = 1) => Merge(key, delta); +/// } +/// +/// // Registration: +/// builder.AddStore<string, long, CounterStore>("counters", new Int64AddMergeOperator()); +/// +/// +public class Int64AddMergeOperator : IMergeOperator +{ + /// + public string Name => "Int64AddMergeOperator"; + + /// + public long FullMerge(ReadOnlySpan key, long? existingValue, IReadOnlyList operands) + { + var result = existingValue ?? 0; + foreach (var operand in operands) + { + result += operand; + } + return result; + } + + /// + public long? PartialMerge(ReadOnlySpan key, IReadOnlyList operands) + { + long result = 0; + foreach (var operand in operands) + { + result += operand; + } + return result; + } +} diff --git a/src/RocksDb.Extensions/MergeOperators/ListAppendMergeOperator.cs b/src/RocksDb.Extensions/MergeOperators/ListAppendMergeOperator.cs new file mode 100644 index 0000000..3d6d6fc --- /dev/null +++ b/src/RocksDb.Extensions/MergeOperators/ListAppendMergeOperator.cs @@ -0,0 +1,60 @@ +namespace RocksDb.Extensions.MergeOperators; + +/// +/// A merge operator that appends items to a list. +/// Useful for implementing atomic list append operations without requiring a read before write. +/// +/// The type of elements in the list. +/// +/// +/// public class EventLogStore : RocksDbStore<string, IList<string>> +/// { +/// public EventLogStore(IRocksDbAccessor<string, IList<string>> accessor) : base(accessor) { } +/// +/// public void AppendEvent(string key, string eventData) +/// { +/// Merge(key, new List<string> { eventData }); +/// } +/// } +/// +/// // Registration: +/// builder.AddStore<string, IList<string>, EventLogStore>("events", new ListAppendMergeOperator<string>()); +/// +/// +public class ListAppendMergeOperator : IMergeOperator> +{ + /// + public string Name => $"ListAppendMergeOperator<{typeof(T).Name}>"; + + /// + public IList FullMerge(ReadOnlySpan key, IList? existingValue, IReadOnlyList> operands) + { + var result = existingValue != null ? new List(existingValue) : new List(); + + foreach (var operand in operands) + { + foreach (var item in operand) + { + result.Add(item); + } + } + + return result; + } + + /// + public IList? PartialMerge(ReadOnlySpan key, IReadOnlyList> operands) + { + var result = new List(); + + foreach (var operand in operands) + { + foreach (var item in operand) + { + result.Add(item); + } + } + + return result; + } +} diff --git a/src/RocksDb.Extensions/RocksDb.Extensions.csproj b/src/RocksDb.Extensions/RocksDb.Extensions.csproj index a7e0c85..f94f8d8 100644 --- a/src/RocksDb.Extensions/RocksDb.Extensions.csproj +++ b/src/RocksDb.Extensions/RocksDb.Extensions.csproj @@ -27,6 +27,6 @@ - + diff --git a/src/RocksDb.Extensions/RocksDbAccessor.cs b/src/RocksDb.Extensions/RocksDbAccessor.cs index 8ad8fe5..5915a55 100644 --- a/src/RocksDb.Extensions/RocksDbAccessor.cs +++ b/src/RocksDb.Extensions/RocksDbAccessor.cs @@ -383,5 +383,75 @@ public void Clear() Native.Instance.rocksdb_column_family_handle_destroy(prevColumnFamilyHandle.Handle); } + + public void Merge(TKey key, TValue operand) + { + byte[]? rentedKeyBuffer = null; + bool useSpanAsKey; + // ReSharper disable once AssignmentInConditionalExpression + Span keyBuffer = (useSpanAsKey = _keySerializer.TryCalculateSize(ref key, out var keySize)) + ? keySize < MaxStackSize + ? stackalloc byte[keySize] + : (rentedKeyBuffer = ArrayPool.Shared.Rent(keySize)).AsSpan(0, keySize) + : Span.Empty; + + ReadOnlySpan keySpan = keyBuffer; + ArrayPoolBufferWriter? keyBufferWriter = null; + + var value = operand; + byte[]? rentedValueBuffer = null; + bool useSpanAsValue; + // ReSharper disable once AssignmentInConditionalExpression + Span valueBuffer = (useSpanAsValue = _valueSerializer.TryCalculateSize(ref value, out var valueSize)) + ? valueSize < MaxStackSize + ? stackalloc byte[valueSize] + : (rentedValueBuffer = ArrayPool.Shared.Rent(valueSize)).AsSpan(0, valueSize) + : Span.Empty; + + + ReadOnlySpan valueSpan = valueBuffer; + ArrayPoolBufferWriter? valueBufferWriter = null; + + try + { + if (useSpanAsKey) + { + _keySerializer.WriteTo(ref key, ref keyBuffer); + } + else + { + keyBufferWriter = new ArrayPoolBufferWriter(); + _keySerializer.WriteTo(ref key, keyBufferWriter); + keySpan = keyBufferWriter.WrittenSpan; + } + + if (useSpanAsValue) + { + _valueSerializer.WriteTo(ref value, ref valueBuffer); + } + else + { + valueBufferWriter = new ArrayPoolBufferWriter(); + _valueSerializer.WriteTo(ref value, valueBufferWriter); + valueSpan = valueBufferWriter.WrittenSpan; + } + + _rocksDbContext.Db.Merge(keySpan, valueSpan, _columnFamily.Handle); + } + finally + { + keyBufferWriter?.Dispose(); + valueBufferWriter?.Dispose(); + if (rentedKeyBuffer is not null) + { + ArrayPool.Shared.Return(rentedKeyBuffer); + } + + if (rentedValueBuffer is not null) + { + ArrayPool.Shared.Return(rentedValueBuffer); + } + } + } } diff --git a/src/RocksDb.Extensions/RocksDbBuilder.cs b/src/RocksDb.Extensions/RocksDbBuilder.cs index 4123075..0d32023 100644 --- a/src/RocksDb.Extensions/RocksDbBuilder.cs +++ b/src/RocksDb.Extensions/RocksDbBuilder.cs @@ -1,7 +1,10 @@ +using System.Buffers; using System.Reflection; +using CommunityToolkit.HighPerformance.Buffers; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; +using RocksDbSharp; namespace RocksDb.Extensions; @@ -16,13 +19,33 @@ public RocksDbBuilder(IServiceCollection serviceCollection) } public IRocksDbBuilder AddStore(string columnFamily) where TStore : RocksDbStore + { + return AddStoreInternal(columnFamily, null); + } + + public IRocksDbBuilder AddStore(string columnFamily, IMergeOperator mergeOperator) where TStore : RocksDbStore + { + return AddStoreInternal(columnFamily, mergeOperator); + } + + private IRocksDbBuilder AddStoreInternal(string columnFamily, IMergeOperator? mergeOperator) where TStore : RocksDbStore { if (!_columnFamilyLookup.Add(columnFamily)) { throw new InvalidOperationException($"{columnFamily} is already registered."); } - _ = _serviceCollection.Configure(options => { options.ColumnFamilies.Add(columnFamily); }); + _ = _serviceCollection.Configure(options => + { + options.ColumnFamilies.Add(columnFamily); + + if (mergeOperator != null) + { + var valueSerializer = CreateSerializer(options.SerializerFactories); + var config = CreateMergeOperatorConfig(mergeOperator, valueSerializer); + options.MergeOperators[columnFamily] = config; + } + }); _serviceCollection.AddKeyedSingleton(columnFamily, (provider, _) => { @@ -45,6 +68,84 @@ public IRocksDbBuilder AddStore(string columnFamily) where return this; } + private static MergeOperatorConfig CreateMergeOperatorConfig(IMergeOperator mergeOperator, ISerializer valueSerializer) + { + return new MergeOperatorConfig + { + Name = mergeOperator.Name, + ValueSerializer = valueSerializer, + FullMerge = (key, existingValue, operands) => FullMergeCallback(key, existingValue, operands, mergeOperator, valueSerializer), + PartialMerge = (key, operands) => PartialMergeCallback(key, operands, mergeOperator, valueSerializer) + }; + } + + private static byte[] FullMergeCallback( + ReadOnlySpan key, + ReadOnlySpan existingValue, + ReadOnlySpan operands, + IMergeOperator mergeOperator, + ISerializer valueSerializer) + { + // Deserialize existing value if present + TValue? existing = existingValue.IsEmpty ? default : valueSerializer.Deserialize(existingValue); + + // Deserialize all operands + var operandList = new List(operands.Length); + foreach (var operand in operands) + { + operandList.Add(valueSerializer.Deserialize(operand.AsSpan())); + } + + // Call the user's merge operator + var result = mergeOperator.FullMerge(key, existing, operandList); + + // Serialize the result + return SerializeValue(result, valueSerializer); + } + + private static byte[]? PartialMergeCallback( + ReadOnlySpan key, + ReadOnlySpan operands, + IMergeOperator mergeOperator, + ISerializer valueSerializer) + { + // Deserialize all operands + var operandList = new List(operands.Length); + foreach (var operand in operands) + { + operandList.Add(valueSerializer.Deserialize(operand.AsSpan())); + } + + // Call the user's partial merge operator + var result = mergeOperator.PartialMerge(key, operandList); + + // If partial merge is not supported, return null + if (result == null) + { + return null; + } + + // Serialize the result + return SerializeValue(result, valueSerializer); + } + + private static byte[] SerializeValue(TValue value, ISerializer valueSerializer) + { + if (valueSerializer.TryCalculateSize(ref value, out var size)) + { + var buffer = new byte[size]; + var span = buffer.AsSpan(); + valueSerializer.WriteTo(ref value, ref span); + return buffer; + } + else + { + using var bufferWriter = new ArrayPoolBufferWriter(); + valueSerializer.WriteTo(ref value, bufferWriter); + return bufferWriter.WrittenSpan.ToArray(); + } + } + private static ISerializer CreateSerializer(IReadOnlyList serializerFactories) { var type = typeof(T); diff --git a/src/RocksDb.Extensions/RocksDbContext.cs b/src/RocksDb.Extensions/RocksDbContext.cs index 4cc489e..7c89faf 100644 --- a/src/RocksDb.Extensions/RocksDbContext.cs +++ b/src/RocksDb.Extensions/RocksDbContext.cs @@ -8,6 +8,7 @@ internal class RocksDbContext : IDisposable private readonly RocksDbSharp.RocksDb _rocksDb; private readonly Cache _cache; private readonly ColumnFamilyOptions _userSpecifiedOptions; + private readonly List _mergeOperators = new(); private const long BlockCacheSize = 50 * 1024 * 1024L; private const long BlockSize = 4096L; @@ -56,7 +57,7 @@ public RocksDbContext(IOptions options) _userSpecifiedOptions.EnableStatistics(); - var columnFamilies = CreateColumnFamilies(options.Value.ColumnFamilies, _userSpecifiedOptions); + var columnFamilies = CreateColumnFamilies(options.Value.ColumnFamilies, options.Value.MergeOperators, _userSpecifiedOptions); if (options.Value.DeleteExistingDatabaseOnStartup) { @@ -76,13 +77,52 @@ private static void DestroyDatabase(string path) public ColumnFamilyOptions ColumnFamilyOptions => _userSpecifiedOptions; - private static ColumnFamilies CreateColumnFamilies(IReadOnlyList columnFamilyNames, - ColumnFamilyOptions columnFamilyOptions) + private ColumnFamilies CreateColumnFamilies( + IReadOnlyList columnFamilyNames, + IReadOnlyDictionary mergeOperators, + ColumnFamilyOptions defaultColumnFamilyOptions) { - var columnFamilies = new ColumnFamilies(columnFamilyOptions); + var columnFamilies = new ColumnFamilies(defaultColumnFamilyOptions); foreach (var columnFamilyName in columnFamilyNames) { - columnFamilies.Add(columnFamilyName, columnFamilyOptions); + if (mergeOperators.TryGetValue(columnFamilyName, out var mergeOperatorConfig)) + { + // Create a copy of the default options for this column family + var cfOptions = new ColumnFamilyOptions(); + + // Apply the same settings as the default options + var tableConfig = new BlockBasedTableOptions(); + tableConfig.SetBlockCache(_cache); + tableConfig.SetBlockSize(BlockSize); + var filter = BloomFilterPolicy.Create(); + tableConfig.SetFilterPolicy(filter); + cfOptions.SetBlockBasedTableFactory(tableConfig); + cfOptions.SetWriteBufferSize(WriteBufferSize); + cfOptions.SetCompression(Compression.No); + cfOptions.SetCompactionStyle(Compaction.Universal); + cfOptions.SetMaxWriteBufferNumberToMaintain(MaxWriteBuffers); + cfOptions.SetCreateIfMissing(); + cfOptions.SetCreateMissingColumnFamilies(); + cfOptions.SetErrorIfExists(false); + cfOptions.SetInfoLogLevel(InfoLogLevel.Error); + cfOptions.EnableStatistics(); + + // Create and set the merge operator + var mergeOp = MergeOperators.CreateAssociative( + mergeOperatorConfig.Name, + mergeOperatorConfig.FullMerge, + mergeOperatorConfig.PartialMerge); + + // Keep reference to prevent GC + _mergeOperators.Add(mergeOp); + + cfOptions.SetMergeOperator(mergeOp); + columnFamilies.Add(columnFamilyName, cfOptions); + } + else + { + columnFamilies.Add(columnFamilyName, defaultColumnFamilyOptions); + } } return columnFamilies; diff --git a/src/RocksDb.Extensions/RocksDbOptions.cs b/src/RocksDb.Extensions/RocksDbOptions.cs index f8c9a60..0fc5305 100644 --- a/src/RocksDb.Extensions/RocksDbOptions.cs +++ b/src/RocksDb.Extensions/RocksDbOptions.cs @@ -33,6 +33,11 @@ public class RocksDbOptions internal List ColumnFamilies { get; } = new(); + /// + /// Internal dictionary of merge operators per column family. + /// + internal Dictionary MergeOperators { get; } = new(StringComparer.InvariantCultureIgnoreCase); + /// /// Enables direct I/O mode for reads, which bypasses the OS page cache. /// From 81f3746314b2f063b27e5dca7ba3a73db2e9d8d3 Mon Sep 17 00:00:00 2001 From: Havret Date: Thu, 4 Dec 2025 10:35:38 +0100 Subject: [PATCH 02/19] wip --- src/RocksDb.Extensions/MergeOperatorConfig.cs | 8 +- src/RocksDb.Extensions/RocksDbBuilder.cs | 45 ++++-- src/RocksDb.Extensions/RocksDbContext.cs | 6 +- .../MergeOperatorTests.cs | 153 ++++++++++++++++++ 4 files changed, 189 insertions(+), 23 deletions(-) create mode 100644 test/RocksDb.Extensions.Tests/MergeOperatorTests.cs diff --git a/src/RocksDb.Extensions/MergeOperatorConfig.cs b/src/RocksDb.Extensions/MergeOperatorConfig.cs index dc5f2c0..86e54ad 100644 --- a/src/RocksDb.Extensions/MergeOperatorConfig.cs +++ b/src/RocksDb.Extensions/MergeOperatorConfig.cs @@ -10,20 +10,20 @@ internal class MergeOperatorConfig /// /// Gets the name of the merge operator. /// - public required string Name { get; init; } + public string Name { get; set; } = null!; /// /// Gets the full merge callback delegate. /// - public required MergeOperator.FullMergeImpl FullMerge { get; init; } + public MergeOperators.FullMergeFunc FullMerge { get; set; } = null!; /// /// Gets the partial merge callback delegate. /// - public required MergeOperator.PartialMergeImpl PartialMerge { get; init; } + public MergeOperators.PartialMergeFunc PartialMerge { get; set; } = null!; /// /// Gets the value serializer for deserializing and serializing values. /// - public required object ValueSerializer { get; init; } + public object ValueSerializer { get; set; } = null!; } diff --git a/src/RocksDb.Extensions/RocksDbBuilder.cs b/src/RocksDb.Extensions/RocksDbBuilder.cs index 0d32023..187760d 100644 --- a/src/RocksDb.Extensions/RocksDbBuilder.cs +++ b/src/RocksDb.Extensions/RocksDbBuilder.cs @@ -74,26 +74,36 @@ private static MergeOperatorConfig CreateMergeOperatorConfig(IMergeOpera { Name = mergeOperator.Name, ValueSerializer = valueSerializer, - FullMerge = (key, existingValue, operands) => FullMergeCallback(key, existingValue, operands, mergeOperator, valueSerializer), - PartialMerge = (key, operands) => PartialMergeCallback(key, operands, mergeOperator, valueSerializer) + FullMerge = (ReadOnlySpan key, bool hasExistingValue, ReadOnlySpan existingValue, MergeOperators.OperandsEnumerator operands, out bool success) => + { + return FullMergeCallback(key, hasExistingValue, existingValue, operands, mergeOperator, valueSerializer, out success); + }, + PartialMerge = (ReadOnlySpan key, MergeOperators.OperandsEnumerator operands, out bool success) => + { + return PartialMergeCallback(key, operands, mergeOperator, valueSerializer, out success); + } }; } private static byte[] FullMergeCallback( ReadOnlySpan key, + bool hasExistingValue, ReadOnlySpan existingValue, - ReadOnlySpan operands, + MergeOperators.OperandsEnumerator operands, IMergeOperator mergeOperator, - ISerializer valueSerializer) + ISerializer valueSerializer, + out bool success) { + success = true; + // Deserialize existing value if present - TValue? existing = existingValue.IsEmpty ? default : valueSerializer.Deserialize(existingValue); + TValue? existing = hasExistingValue ? valueSerializer.Deserialize(existingValue) : default; // Deserialize all operands - var operandList = new List(operands.Length); - foreach (var operand in operands) + var operandList = new List(operands.Count); + for (int i = 0; i < operands.Count; i++) { - operandList.Add(valueSerializer.Deserialize(operand.AsSpan())); + operandList.Add(valueSerializer.Deserialize(operands.Get(i))); } // Call the user's merge operator @@ -103,28 +113,31 @@ private static byte[] FullMergeCallback( return SerializeValue(result, valueSerializer); } - private static byte[]? PartialMergeCallback( + private static byte[] PartialMergeCallback( ReadOnlySpan key, - ReadOnlySpan operands, + MergeOperators.OperandsEnumerator operands, IMergeOperator mergeOperator, - ISerializer valueSerializer) + ISerializer valueSerializer, + out bool success) { // Deserialize all operands - var operandList = new List(operands.Length); - foreach (var operand in operands) + var operandList = new List(operands.Count); + for (int i = 0; i < operands.Count; i++) { - operandList.Add(valueSerializer.Deserialize(operand.AsSpan())); + operandList.Add(valueSerializer.Deserialize(operands.Get(i))); } // Call the user's partial merge operator var result = mergeOperator.PartialMerge(key, operandList); - // If partial merge is not supported, return null + // If partial merge is not supported, return failure if (result == null) { - return null; + success = false; + return Array.Empty(); } + success = true; // Serialize the result return SerializeValue(result, valueSerializer); } diff --git a/src/RocksDb.Extensions/RocksDbContext.cs b/src/RocksDb.Extensions/RocksDbContext.cs index 7c89faf..0e418bc 100644 --- a/src/RocksDb.Extensions/RocksDbContext.cs +++ b/src/RocksDb.Extensions/RocksDbContext.cs @@ -108,10 +108,10 @@ private ColumnFamilies CreateColumnFamilies( cfOptions.EnableStatistics(); // Create and set the merge operator - var mergeOp = MergeOperators.CreateAssociative( + var mergeOp = MergeOperators.Create( mergeOperatorConfig.Name, - mergeOperatorConfig.FullMerge, - mergeOperatorConfig.PartialMerge); + mergeOperatorConfig.PartialMerge, + mergeOperatorConfig.FullMerge); // Keep reference to prevent GC _mergeOperators.Add(mergeOp); diff --git a/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs b/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs new file mode 100644 index 0000000..60efcb3 --- /dev/null +++ b/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs @@ -0,0 +1,153 @@ +using NUnit.Framework; +using RocksDb.Extensions.MergeOperators; +using RocksDb.Extensions.Tests.Utils; + +namespace RocksDb.Extensions.Tests; + +/// +/// Store that exposes the protected Merge method for testing counters. +/// +public class CounterStore : RocksDbStore +{ + public CounterStore(IRocksDbAccessor rocksDbAccessor) : base(rocksDbAccessor) + { + } + + public void Increment(string key, long delta = 1) => Merge(key, delta); +} + +/// +/// Store that exposes the protected Merge method for testing list appends. +/// +public class EventLogStore : RocksDbStore> +{ + public EventLogStore(IRocksDbAccessor> rocksDbAccessor) : base(rocksDbAccessor) + { + } + + public void AppendEvent(string key, string eventData) => Merge(key, new List { eventData }); +} + +public class MergeOperatorTests +{ + [Test] + public void should_increment_counter_using_merge_operation() + { + // Arrange + using var testFixture = TestFixture.Create(rockDb => + { + rockDb.AddStore("counters", new Int64AddMergeOperator()); + }); + + var store = testFixture.GetStore(); + var key = "page-views"; + + // Act + store.Increment(key, 1); + store.Increment(key, 5); + store.Increment(key, 10); + + // Assert + Assert.That(store.TryGet(key, out var value), Is.True); + Assert.That(value, Is.EqualTo(16)); + } + + [Test] + public void should_handle_counter_with_initial_value() + { + // Arrange + using var testFixture = TestFixture.Create(rockDb => + { + rockDb.AddStore("counters", new Int64AddMergeOperator()); + }); + + var store = testFixture.GetStore(); + var key = "page-views"; + + // Act - Put initial value, then merge + store.Put(key, 100); + store.Increment(key, 50); + + // Assert + Assert.That(store.TryGet(key, out var value), Is.True); + Assert.That(value, Is.EqualTo(150)); + } + + [Test] + public void should_append_to_list_using_merge_operation() + { + // Arrange + using var testFixture = TestFixture.Create(rockDb => + { + rockDb.AddStore, EventLogStore>("events", new ListAppendMergeOperator()); + }); + + var store = testFixture.GetStore(); + var key = "user-actions"; + + // Act + store.AppendEvent(key, "login"); + store.AppendEvent(key, "view-page"); + store.AppendEvent(key, "logout"); + + // Assert + Assert.That(store.TryGet(key, out var events), Is.True); + Assert.That(events, Is.Not.Null); + Assert.That(events.Count, Is.EqualTo(3)); + Assert.That(events[0], Is.EqualTo("login")); + Assert.That(events[1], Is.EqualTo("view-page")); + Assert.That(events[2], Is.EqualTo("logout")); + } + + [Test] + public void should_append_to_existing_list_using_merge_operation() + { + // Arrange + using var testFixture = TestFixture.Create(rockDb => + { + rockDb.AddStore, EventLogStore>("events", new ListAppendMergeOperator()); + }); + + var store = testFixture.GetStore(); + var key = "user-actions"; + + // Act - Put initial value, then merge + store.Put(key, new List { "initial-event" }); + store.AppendEvent(key, "new-event"); + + // Assert + Assert.That(store.TryGet(key, out var events), Is.True); + Assert.That(events, Is.Not.Null); + Assert.That(events.Count, Is.EqualTo(2)); + Assert.That(events[0], Is.EqualTo("initial-event")); + Assert.That(events[1], Is.EqualTo("new-event")); + } + + [Test] + public void should_handle_multiple_keys_with_merge_operations() + { + // Arrange + using var testFixture = TestFixture.Create(rockDb => + { + rockDb.AddStore("counters", new Int64AddMergeOperator()); + }); + + var store = testFixture.GetStore(); + + // Act + store.Increment("key1", 10); + store.Increment("key2", 20); + store.Increment("key1", 5); + store.Increment("key2", 10); + + // Assert + Assert.Multiple(() => + { + Assert.That(store.TryGet("key1", out var value1), Is.True); + Assert.That(value1, Is.EqualTo(15)); + + Assert.That(store.TryGet("key2", out var value2), Is.True); + Assert.That(value2, Is.EqualTo(30)); + }); + } +} From 8563757b6e4f433209e61f18a7cbd8bcf9367856 Mon Sep 17 00:00:00 2001 From: Havret Date: Thu, 4 Dec 2025 13:07:26 +0100 Subject: [PATCH 03/19] wip --- src/RocksDb.Extensions/IMergeOperator.cs | 8 ++++---- src/RocksDb.Extensions/MergeOperatorConfig.cs | 4 ++-- .../MergeOperators/Int64AddMergeOperator.cs | 6 +++--- .../MergeOperators/ListAppendMergeOperator.cs | 4 ++-- src/RocksDb.Extensions/RocksDbBuilder.cs | 17 +++++------------ src/RocksDb.Extensions/RocksDbContext.cs | 6 +++--- 6 files changed, 19 insertions(+), 26 deletions(-) diff --git a/src/RocksDb.Extensions/IMergeOperator.cs b/src/RocksDb.Extensions/IMergeOperator.cs index 5453255..ec24583 100644 --- a/src/RocksDb.Extensions/IMergeOperator.cs +++ b/src/RocksDb.Extensions/IMergeOperator.cs @@ -19,10 +19,10 @@ public interface IMergeOperator /// Called when a Get operation encounters merge operands and needs to produce the final value. /// /// The key being merged. - /// The existing value in the database, or null if no value exists. + /// The existing value in the database. For value types, this will be default if no value exists (check hasExistingValue). /// The list of merge operands to apply, in order. /// The merged value. - TValue FullMerge(ReadOnlySpan key, TValue? existingValue, IReadOnlyList operands); + TValue FullMerge(ReadOnlySpan key, TValue existingValue, IReadOnlyList operands); /// /// Performs a partial merge of multiple operands without the existing value. @@ -32,8 +32,8 @@ public interface IMergeOperator /// The key being merged. /// The list of merge operands to combine, in order. /// - /// The combined operand if partial merge is possible; otherwise, null to indicate + /// The combined operand if partial merge is possible; otherwise, default to indicate /// that partial merge is not supported and operands should be kept separate. /// - TValue? PartialMerge(ReadOnlySpan key, IReadOnlyList operands); + TValue PartialMerge(ReadOnlySpan key, IReadOnlyList operands); } diff --git a/src/RocksDb.Extensions/MergeOperatorConfig.cs b/src/RocksDb.Extensions/MergeOperatorConfig.cs index 86e54ad..ac20b8e 100644 --- a/src/RocksDb.Extensions/MergeOperatorConfig.cs +++ b/src/RocksDb.Extensions/MergeOperatorConfig.cs @@ -15,12 +15,12 @@ internal class MergeOperatorConfig /// /// Gets the full merge callback delegate. /// - public MergeOperators.FullMergeFunc FullMerge { get; set; } = null!; + public global::RocksDbSharp.MergeOperators.FullMergeFunc FullMerge { get; set; } = null!; /// /// Gets the partial merge callback delegate. /// - public MergeOperators.PartialMergeFunc PartialMerge { get; set; } = null!; + public global::RocksDbSharp.MergeOperators.PartialMergeFunc PartialMerge { get; set; } = null!; /// /// Gets the value serializer for deserializing and serializing values. diff --git a/src/RocksDb.Extensions/MergeOperators/Int64AddMergeOperator.cs b/src/RocksDb.Extensions/MergeOperators/Int64AddMergeOperator.cs index 38ec7d2..5682183 100644 --- a/src/RocksDb.Extensions/MergeOperators/Int64AddMergeOperator.cs +++ b/src/RocksDb.Extensions/MergeOperators/Int64AddMergeOperator.cs @@ -23,9 +23,9 @@ public class Int64AddMergeOperator : IMergeOperator public string Name => "Int64AddMergeOperator"; /// - public long FullMerge(ReadOnlySpan key, long? existingValue, IReadOnlyList operands) + public long FullMerge(ReadOnlySpan key, long existingValue, IReadOnlyList operands) { - var result = existingValue ?? 0; + var result = existingValue; foreach (var operand in operands) { result += operand; @@ -34,7 +34,7 @@ public long FullMerge(ReadOnlySpan key, long? existingValue, IReadOnlyList } /// - public long? PartialMerge(ReadOnlySpan key, IReadOnlyList operands) + public long PartialMerge(ReadOnlySpan key, IReadOnlyList operands) { long result = 0; foreach (var operand in operands) diff --git a/src/RocksDb.Extensions/MergeOperators/ListAppendMergeOperator.cs b/src/RocksDb.Extensions/MergeOperators/ListAppendMergeOperator.cs index 3d6d6fc..4da0194 100644 --- a/src/RocksDb.Extensions/MergeOperators/ListAppendMergeOperator.cs +++ b/src/RocksDb.Extensions/MergeOperators/ListAppendMergeOperator.cs @@ -27,7 +27,7 @@ public class ListAppendMergeOperator : IMergeOperator> public string Name => $"ListAppendMergeOperator<{typeof(T).Name}>"; /// - public IList FullMerge(ReadOnlySpan key, IList? existingValue, IReadOnlyList> operands) + public IList FullMerge(ReadOnlySpan key, IList existingValue, IReadOnlyList> operands) { var result = existingValue != null ? new List(existingValue) : new List(); @@ -43,7 +43,7 @@ public IList FullMerge(ReadOnlySpan key, IList? existingValue, IRead } /// - public IList? PartialMerge(ReadOnlySpan key, IReadOnlyList> operands) + public IList PartialMerge(ReadOnlySpan key, IReadOnlyList> operands) { var result = new List(); diff --git a/src/RocksDb.Extensions/RocksDbBuilder.cs b/src/RocksDb.Extensions/RocksDbBuilder.cs index 187760d..9fd20dc 100644 --- a/src/RocksDb.Extensions/RocksDbBuilder.cs +++ b/src/RocksDb.Extensions/RocksDbBuilder.cs @@ -74,11 +74,11 @@ private static MergeOperatorConfig CreateMergeOperatorConfig(IMergeOpera { Name = mergeOperator.Name, ValueSerializer = valueSerializer, - FullMerge = (ReadOnlySpan key, bool hasExistingValue, ReadOnlySpan existingValue, MergeOperators.OperandsEnumerator operands, out bool success) => + FullMerge = (ReadOnlySpan key, bool hasExistingValue, ReadOnlySpan existingValue, global::RocksDbSharp.MergeOperators.OperandsEnumerator operands, out bool success) => { return FullMergeCallback(key, hasExistingValue, existingValue, operands, mergeOperator, valueSerializer, out success); }, - PartialMerge = (ReadOnlySpan key, MergeOperators.OperandsEnumerator operands, out bool success) => + PartialMerge = (ReadOnlySpan key, global::RocksDbSharp.MergeOperators.OperandsEnumerator operands, out bool success) => { return PartialMergeCallback(key, operands, mergeOperator, valueSerializer, out success); } @@ -89,7 +89,7 @@ private static byte[] FullMergeCallback( ReadOnlySpan key, bool hasExistingValue, ReadOnlySpan existingValue, - MergeOperators.OperandsEnumerator operands, + global::RocksDbSharp.MergeOperators.OperandsEnumerator operands, IMergeOperator mergeOperator, ISerializer valueSerializer, out bool success) @@ -97,7 +97,7 @@ private static byte[] FullMergeCallback( success = true; // Deserialize existing value if present - TValue? existing = hasExistingValue ? valueSerializer.Deserialize(existingValue) : default; + TValue existing = hasExistingValue ? valueSerializer.Deserialize(existingValue) : default!; // Deserialize all operands var operandList = new List(operands.Count); @@ -115,7 +115,7 @@ private static byte[] FullMergeCallback( private static byte[] PartialMergeCallback( ReadOnlySpan key, - MergeOperators.OperandsEnumerator operands, + global::RocksDbSharp.MergeOperators.OperandsEnumerator operands, IMergeOperator mergeOperator, ISerializer valueSerializer, out bool success) @@ -130,13 +130,6 @@ private static byte[] PartialMergeCallback( // Call the user's partial merge operator var result = mergeOperator.PartialMerge(key, operandList); - // If partial merge is not supported, return failure - if (result == null) - { - success = false; - return Array.Empty(); - } - success = true; // Serialize the result return SerializeValue(result, valueSerializer); diff --git a/src/RocksDb.Extensions/RocksDbContext.cs b/src/RocksDb.Extensions/RocksDbContext.cs index 0e418bc..d1e0085 100644 --- a/src/RocksDb.Extensions/RocksDbContext.cs +++ b/src/RocksDb.Extensions/RocksDbContext.cs @@ -30,7 +30,7 @@ public RocksDbContext(IOptions options) _userSpecifiedOptions.SetWriteBufferSize(WriteBufferSize); _userSpecifiedOptions.SetCompression(Compression.No); _userSpecifiedOptions.SetCompactionStyle(Compaction.Universal); - _userSpecifiedOptions.SetMaxWriteBufferNumberToMaintain(MaxWriteBuffers); + _userSpecifiedOptions.SetMaxWriteBufferNumber(MaxWriteBuffers); _userSpecifiedOptions.SetCreateIfMissing(); _userSpecifiedOptions.SetCreateMissingColumnFamilies(); _userSpecifiedOptions.SetErrorIfExists(false); @@ -100,7 +100,7 @@ private ColumnFamilies CreateColumnFamilies( cfOptions.SetWriteBufferSize(WriteBufferSize); cfOptions.SetCompression(Compression.No); cfOptions.SetCompactionStyle(Compaction.Universal); - cfOptions.SetMaxWriteBufferNumberToMaintain(MaxWriteBuffers); + cfOptions.SetMaxWriteBufferNumber(MaxWriteBuffers); cfOptions.SetCreateIfMissing(); cfOptions.SetCreateMissingColumnFamilies(); cfOptions.SetErrorIfExists(false); @@ -108,7 +108,7 @@ private ColumnFamilies CreateColumnFamilies( cfOptions.EnableStatistics(); // Create and set the merge operator - var mergeOp = MergeOperators.Create( + var mergeOp = global::RocksDbSharp.MergeOperators.Create( mergeOperatorConfig.Name, mergeOperatorConfig.PartialMerge, mergeOperatorConfig.FullMerge); From 0ba1404a0d02a81df0c3f930186d049886669d6f Mon Sep 17 00:00:00 2001 From: Havret Date: Thu, 4 Dec 2025 14:39:59 +0100 Subject: [PATCH 04/19] WIP --- .../IMergeableRocksDbStore.cs | 19 ++ src/RocksDb.Extensions/IRocksDbBuilder.cs | 21 +++ .../ListOperationSerializer.cs | 113 ++++++++++++ .../MergeOperators/ListMergeOperator.cs | 119 ++++++++++++ .../MergeOperators/ListOperation.cs | 65 +++++++ .../MergeableRocksDbStore.cs | 53 ++++++ src/RocksDb.Extensions/RocksDbBuilder.cs | 24 ++- .../MergeOperatorTests.cs | 172 +++++++++++++++++- 8 files changed, 575 insertions(+), 11 deletions(-) create mode 100644 src/RocksDb.Extensions/IMergeableRocksDbStore.cs create mode 100644 src/RocksDb.Extensions/ListOperationSerializer.cs create mode 100644 src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs create mode 100644 src/RocksDb.Extensions/MergeOperators/ListOperation.cs create mode 100644 src/RocksDb.Extensions/MergeableRocksDbStore.cs diff --git a/src/RocksDb.Extensions/IMergeableRocksDbStore.cs b/src/RocksDb.Extensions/IMergeableRocksDbStore.cs new file mode 100644 index 0000000..c4570db --- /dev/null +++ b/src/RocksDb.Extensions/IMergeableRocksDbStore.cs @@ -0,0 +1,19 @@ +namespace RocksDb.Extensions; + +/// +/// Interface for a RocksDB store that supports merge operations. +/// +/// The type of the store's keys. +/// The type of the store's values. +public interface IMergeableRocksDbStore +{ + /// + /// Performs an atomic merge operation on the value associated with the specified key. + /// This operation uses RocksDB's merge operator to combine the operand with the existing value + /// without requiring a separate read operation, which is more efficient than Get+Put for + /// accumulative operations like counters, list appends, or set unions. + /// + /// The key to merge the operand with. + /// The operand to merge with the existing value. + void Merge(TKey key, TValue operand); +} diff --git a/src/RocksDb.Extensions/IRocksDbBuilder.cs b/src/RocksDb.Extensions/IRocksDbBuilder.cs index 861bac3..45680d1 100644 --- a/src/RocksDb.Extensions/IRocksDbBuilder.cs +++ b/src/RocksDb.Extensions/IRocksDbBuilder.cs @@ -44,4 +44,25 @@ public interface IRocksDbBuilder /// Use GetRequiredKeyedService<TStore>(columnFamily) to retrieve a specific store instance. /// IRocksDbBuilder AddStore(string columnFamily, IMergeOperator mergeOperator) where TStore : RocksDbStore; + + /// + /// Adds a mergeable RocksDB store to the builder for the specified column family. + /// This method enforces that the store inherits from + /// and requires a merge operator, providing compile-time safety for merge operations. + /// + /// The name of the column family to associate with the store. + /// The merge operator to use for atomic read-modify-write operations. + /// The type of the store's key. + /// The type of the store's value. + /// The type of the store to add. Must inherit from . + /// The builder instance for method chaining. + /// Thrown if the specified column family is already registered. + /// + /// Use this method when your store needs merge operations. The constraint + /// ensures that only stores designed for merge operations can be registered with this method. + /// + /// For stores that don't need merge operations, use instead. + /// + IRocksDbBuilder AddMergeableStore(string columnFamily, IMergeOperator mergeOperator) + where TStore : MergeableRocksDbStore; } \ No newline at end of file diff --git a/src/RocksDb.Extensions/ListOperationSerializer.cs b/src/RocksDb.Extensions/ListOperationSerializer.cs new file mode 100644 index 0000000..fff759b --- /dev/null +++ b/src/RocksDb.Extensions/ListOperationSerializer.cs @@ -0,0 +1,113 @@ +using System.Buffers; +using RocksDb.Extensions.MergeOperators; + +namespace RocksDb.Extensions; + +/// +/// Serializes ListOperation<T> which contains an operation type (Add/Remove) and a list of items. +/// +/// +/// The serialized format consists of: +/// - 1 byte: Operation type (0 = Add, 1 = Remove) +/// - 4 bytes: Number of items +/// - For each item: +/// - 4 bytes: Size of the serialized item +/// - N bytes: Serialized item data +/// +internal class ListOperationSerializer : ISerializer> +{ + private readonly ISerializer _itemSerializer; + + public ListOperationSerializer(ISerializer itemSerializer) + { + _itemSerializer = itemSerializer; + } + + public bool TryCalculateSize(ref ListOperation value, out int size) + { + // 1 byte for operation type + 4 bytes for count + size = sizeof(byte) + sizeof(int); + + for (int i = 0; i < value.Items.Count; i++) + { + var item = value.Items[i]; + if (_itemSerializer.TryCalculateSize(ref item, out var itemSize)) + { + size += sizeof(int); // size prefix for each item + size += itemSize; + } + } + + return true; + } + + public void WriteTo(ref ListOperation value, ref Span span) + { + int offset = 0; + + // Write operation type (1 byte) + span[offset] = (byte)value.Type; + offset += sizeof(byte); + + // Write count + var slice = span.Slice(offset, sizeof(int)); + BitConverter.TryWriteBytes(slice, value.Items.Count); + offset += sizeof(int); + + // Write each item with size prefix + for (int i = 0; i < value.Items.Count; i++) + { + var item = value.Items[i]; + if (_itemSerializer.TryCalculateSize(ref item, out var itemSize)) + { + slice = span.Slice(offset, sizeof(int)); + BitConverter.TryWriteBytes(slice, itemSize); + offset += sizeof(int); + + slice = span.Slice(offset, itemSize); + _itemSerializer.WriteTo(ref item, ref slice); + offset += itemSize; + } + } + } + + public void WriteTo(ref ListOperation value, IBufferWriter buffer) + { + if (TryCalculateSize(ref value, out var size)) + { + var span = buffer.GetSpan(size); + WriteTo(ref value, ref span); + buffer.Advance(size); + } + } + + public ListOperation Deserialize(ReadOnlySpan buffer) + { + int offset = 0; + + // Read operation type + var operationType = (OperationType)buffer[offset]; + offset += sizeof(byte); + + // Read count + var slice = buffer.Slice(offset, sizeof(int)); + var count = BitConverter.ToInt32(slice); + offset += sizeof(int); + + // Read items + var items = new List(count); + for (int i = 0; i < count; i++) + { + slice = buffer.Slice(offset, sizeof(int)); + var itemSize = BitConverter.ToInt32(slice); + offset += sizeof(int); + + slice = buffer.Slice(offset, itemSize); + var item = _itemSerializer.Deserialize(slice); + items.Add(item); + offset += itemSize; + } + + return new ListOperation(operationType, items); + } +} diff --git a/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs b/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs new file mode 100644 index 0000000..d9c0205 --- /dev/null +++ b/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs @@ -0,0 +1,119 @@ +namespace RocksDb.Extensions.MergeOperators; + +/// +/// A merge operator that supports both adding and removing items from a list. +/// Each merge operand is a ListOperation that specifies whether to add or remove items. +/// Operations are applied in order, enabling atomic list modifications without read-before-write. +/// +/// The type of elements in the list. +/// +/// +/// public class TagsStore : RocksDbStore<string, IList<string>> +/// { +/// public TagsStore(IRocksDbAccessor<string, IList<string>> accessor) : base(accessor) { } +/// +/// public void AddTags(string key, params string[] tags) +/// { +/// Merge(key, new List<ListOperation<string>> { ListOperation<string>.Add(tags) }); +/// } +/// +/// public void RemoveTags(string key, params string[] tags) +/// { +/// Merge(key, new List<ListOperation<string>> { ListOperation<string>.Remove(tags) }); +/// } +/// } +/// +/// // Registration: +/// builder.AddStore<string, IList<string>, TagsStore>("tags", new ListMergeOperator<string>()); +/// +/// +/// +/// +/// The value type stored in RocksDB is IList<T> (the actual list contents), +/// but merge operands are IList<ListOperation<T>> (the operations to apply). +/// +/// +/// Remove operations delete the first occurrence of each item (same as ). +/// If an item to remove doesn't exist in the list, the operation is silently ignored. +/// +/// +/// For append-only use cases where removes are not needed, prefer +/// which has less serialization overhead. +/// +/// +public class ListMergeOperator : IMergeOperator>> +{ + /// + public string Name => $"ListMergeOperator<{typeof(T).Name}>"; + + /// + public IList> FullMerge( + ReadOnlySpan key, + IList> existingValue, + IReadOnlyList>> operands) + { + // Start with existing items or empty list + var result = new List(); + + // If there's an existing value, it contains the accumulated operations from previous merges + // We need to apply those operations first + if (existingValue != null) + { + ApplyOperations(result, existingValue); + } + + // Apply all new operands in order + foreach (var operandBatch in operands) + { + ApplyOperations(result, operandBatch); + } + + // Return the final list wrapped as a single Add operation + // This collapses all operations into the final state + return new List> { ListOperation.Add(result) }; + } + + /// + public IList> PartialMerge( + ReadOnlySpan key, + IReadOnlyList>> operands) + { + // Combine all operations into a single list + // We preserve all operations rather than trying to resolve them + // because removes can't be safely combined without knowing the base state + var combined = new List>(); + + foreach (var operandBatch in operands) + { + foreach (var op in operandBatch) + { + combined.Add(op); + } + } + + return combined; + } + + private static void ApplyOperations(List result, IList> operations) + { + foreach (var operation in operations) + { + switch (operation.Type) + { + case OperationType.Add: + foreach (var item in operation.Items) + { + result.Add(item); + } + break; + + case OperationType.Remove: + foreach (var item in operation.Items) + { + result.Remove(item); // Removes first occurrence + } + break; + } + } + } +} diff --git a/src/RocksDb.Extensions/MergeOperators/ListOperation.cs b/src/RocksDb.Extensions/MergeOperators/ListOperation.cs new file mode 100644 index 0000000..89db008 --- /dev/null +++ b/src/RocksDb.Extensions/MergeOperators/ListOperation.cs @@ -0,0 +1,65 @@ +namespace RocksDb.Extensions.MergeOperators; + +/// +/// Specifies the type of operation to perform on a list. +/// +public enum OperationType +{ + /// + /// Add items to the list. + /// + Add, + + /// + /// Remove items from the list (first occurrence of each item). + /// + Remove +} + +/// +/// Represents an operation (add or remove) to apply to a list via merge. +/// +/// The type of elements in the list. +public class ListOperation +{ + /// + /// Gets the type of operation to perform. + /// + public OperationType Type { get; } + + /// + /// Gets the items to add or remove. + /// + public IList Items { get; } + + /// + /// Creates a new list operation. + /// + /// The type of operation. + /// The items to add or remove. + public ListOperation(OperationType type, IList items) + { + Type = type; + Items = items ?? throw new ArgumentNullException(nameof(items)); + } + + /// + /// Creates an Add operation for the specified items. + /// + public static ListOperation Add(params T[] items) => new(OperationType.Add, items); + + /// + /// Creates an Add operation for the specified items. + /// + public static ListOperation Add(IList items) => new(OperationType.Add, items); + + /// + /// Creates a Remove operation for the specified items. + /// + public static ListOperation Remove(params T[] items) => new(OperationType.Remove, items); + + /// + /// Creates a Remove operation for the specified items. + /// + public static ListOperation Remove(IList items) => new(OperationType.Remove, items); +} diff --git a/src/RocksDb.Extensions/MergeableRocksDbStore.cs b/src/RocksDb.Extensions/MergeableRocksDbStore.cs new file mode 100644 index 0000000..4e2cb22 --- /dev/null +++ b/src/RocksDb.Extensions/MergeableRocksDbStore.cs @@ -0,0 +1,53 @@ +namespace RocksDb.Extensions; + +/// +/// Base class for a RocksDB store that supports merge operations. +/// Inherit from this class when you need to use RocksDB's merge operator functionality +/// for efficient atomic read-modify-write operations. +/// +/// The type of the store's keys. +/// The type of the store's values. +/// +/// +/// Merge operations are useful for: +/// - Counters: Increment/decrement without reading current value +/// - Lists: Append items without reading the entire list +/// - Sets: Add/remove items atomically +/// +/// +/// When using this base class, you must register the store with a merge operator using +/// . +/// +/// +/// +/// +/// public class CounterStore : MergeableRocksDbStore<string, long> +/// { +/// public CounterStore(IRocksDbAccessor<string, long> accessor) : base(accessor) { } +/// +/// public void Increment(string key, long delta = 1) => Merge(key, delta); +/// } +/// +/// // Registration: +/// builder.AddMergeableStore<string, long, CounterStore>("counters", new Int64AddMergeOperator()); +/// +/// +public abstract class MergeableRocksDbStore : RocksDbStore, IMergeableRocksDbStore +{ + /// + /// Initializes a new instance of the class. + /// + /// The RocksDB accessor to use for database operations. + protected MergeableRocksDbStore(IRocksDbAccessor rocksDbAccessor) : base(rocksDbAccessor) + { + } + + /// + /// Performs an atomic merge operation on the value associated with the specified key. + /// This operation uses RocksDB's merge operator to combine the operand with the existing value + /// without requiring a separate read operation. + /// + /// The key to merge the operand with. + /// The operand to merge with the existing value. + public new void Merge(TKey key, TValue operand) => base.Merge(key, operand); +} diff --git a/src/RocksDb.Extensions/RocksDbBuilder.cs b/src/RocksDb.Extensions/RocksDbBuilder.cs index 9fd20dc..f266ede 100644 --- a/src/RocksDb.Extensions/RocksDbBuilder.cs +++ b/src/RocksDb.Extensions/RocksDbBuilder.cs @@ -28,6 +28,12 @@ public IRocksDbBuilder AddStore(string columnFamily, IMerg return AddStoreInternal(columnFamily, mergeOperator); } + public IRocksDbBuilder AddMergeableStore(string columnFamily, IMergeOperator mergeOperator) + where TStore : MergeableRocksDbStore + { + return AddStoreInternal(columnFamily, mergeOperator); + } + private IRocksDbBuilder AddStoreInternal(string columnFamily, IMergeOperator? mergeOperator) where TStore : RocksDbStore { if (!_columnFamilyLookup.Add(columnFamily)) @@ -177,11 +183,25 @@ private static ISerializer CreateSerializer(IReadOnlyList) Activator.CreateInstance(typeof(FixedSizeListSerializer<>).MakeGenericType(elementType), scalarSerializer); + return (ISerializer) Activator.CreateInstance(typeof(FixedSizeListSerializer<>).MakeGenericType(elementType), scalarSerializer)!; } // Use variable size list serializer for non-primitive types - return (ISerializer) Activator.CreateInstance(typeof(VariableSizeListSerializer<>).MakeGenericType(elementType), scalarSerializer); + return (ISerializer) Activator.CreateInstance(typeof(VariableSizeListSerializer<>).MakeGenericType(elementType), scalarSerializer)!; + } + + // Handle ListOperation for the ListMergeOperator + if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(MergeOperators.ListOperation<>)) + { + var itemType = type.GetGenericArguments()[0]; + + // Create the item serializer + var itemSerializer = typeof(RocksDbBuilder).GetMethod(nameof(CreateSerializer), BindingFlags.NonPublic | BindingFlags.Static) + ?.MakeGenericMethod(itemType) + .Invoke(null, new object[] { serializerFactories }); + + // Create ListOperationSerializer + return (ISerializer) Activator.CreateInstance(typeof(ListOperationSerializer<>).MakeGenericType(itemType), itemSerializer)!; } throw new InvalidOperationException($"Type {type.FullName} cannot be used as RocksDbStore key/value. " + diff --git a/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs b/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs index 60efcb3..4655898 100644 --- a/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs +++ b/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs @@ -5,9 +5,9 @@ namespace RocksDb.Extensions.Tests; /// -/// Store that exposes the protected Merge method for testing counters. +/// Store that uses merge operations for testing counters. /// -public class CounterStore : RocksDbStore +public class CounterStore : MergeableRocksDbStore { public CounterStore(IRocksDbAccessor rocksDbAccessor) : base(rocksDbAccessor) { @@ -17,9 +17,9 @@ public CounterStore(IRocksDbAccessor rocksDbAccessor) : base(rocks } /// -/// Store that exposes the protected Merge method for testing list appends. +/// Store that uses merge operations for testing list appends. /// -public class EventLogStore : RocksDbStore> +public class EventLogStore : MergeableRocksDbStore> { public EventLogStore(IRocksDbAccessor> rocksDbAccessor) : base(rocksDbAccessor) { @@ -28,6 +28,36 @@ public EventLogStore(IRocksDbAccessor> rocksDbAccessor) : public void AppendEvent(string key, string eventData) => Merge(key, new List { eventData }); } +/// +/// Store that uses merge operations for testing list operations (add/remove). +/// +public class TagsStore : MergeableRocksDbStore>> +{ + public TagsStore(IRocksDbAccessor>> rocksDbAccessor) : base(rocksDbAccessor) + { + } + + public void AddTags(string key, params string[] tags) + { + Merge(key, new List> { ListOperation.Add(tags) }); + } + + public void RemoveTags(string key, params string[] tags) + { + Merge(key, new List> { ListOperation.Remove(tags) }); + } + + public IList? GetTags(string key) + { + if (TryGet(key, out var operations) && operations != null && operations.Count > 0) + { + // The merged result is a single Add operation containing all items + return operations[0].Items; + } + return null; + } +} + public class MergeOperatorTests { [Test] @@ -36,7 +66,7 @@ public void should_increment_counter_using_merge_operation() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddStore("counters", new Int64AddMergeOperator()); + rockDb.AddMergeableStore("counters", new Int64AddMergeOperator()); }); var store = testFixture.GetStore(); @@ -58,7 +88,7 @@ public void should_handle_counter_with_initial_value() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddStore("counters", new Int64AddMergeOperator()); + rockDb.AddMergeableStore("counters", new Int64AddMergeOperator()); }); var store = testFixture.GetStore(); @@ -79,7 +109,7 @@ public void should_append_to_list_using_merge_operation() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddStore, EventLogStore>("events", new ListAppendMergeOperator()); + rockDb.AddMergeableStore, EventLogStore>("events", new ListAppendMergeOperator()); }); var store = testFixture.GetStore(); @@ -105,7 +135,7 @@ public void should_append_to_existing_list_using_merge_operation() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddStore, EventLogStore>("events", new ListAppendMergeOperator()); + rockDb.AddMergeableStore, EventLogStore>("events", new ListAppendMergeOperator()); }); var store = testFixture.GetStore(); @@ -129,7 +159,7 @@ public void should_handle_multiple_keys_with_merge_operations() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddStore("counters", new Int64AddMergeOperator()); + rockDb.AddMergeableStore("counters", new Int64AddMergeOperator()); }); var store = testFixture.GetStore(); @@ -150,4 +180,128 @@ public void should_handle_multiple_keys_with_merge_operations() Assert.That(value2, Is.EqualTo(30)); }); } + + [Test] + public void should_add_items_to_list_using_list_merge_operator() + { + // Arrange + using var testFixture = TestFixture.Create(rockDb => + { + rockDb.AddMergeableStore>, TagsStore>("tags", new ListMergeOperator()); + }); + + var store = testFixture.GetStore(); + var key = "article-1"; + + // Act + store.AddTags(key, "csharp", "dotnet"); + store.AddTags(key, "rocksdb"); + + // Assert + var tags = store.GetTags(key); + Assert.That(tags, Is.Not.Null); + Assert.That(tags!.Count, Is.EqualTo(3)); + Assert.That(tags, Does.Contain("csharp")); + Assert.That(tags, Does.Contain("dotnet")); + Assert.That(tags, Does.Contain("rocksdb")); + } + + [Test] + public void should_remove_items_from_list_using_list_merge_operator() + { + // Arrange + using var testFixture = TestFixture.Create(rockDb => + { + rockDb.AddMergeableStore>, TagsStore>("tags", new ListMergeOperator()); + }); + + var store = testFixture.GetStore(); + var key = "article-1"; + + // Act - Add items, then remove some + store.AddTags(key, "csharp", "dotnet", "java", "python"); + store.RemoveTags(key, "java", "python"); + + // Assert + var tags = store.GetTags(key); + Assert.That(tags, Is.Not.Null); + Assert.That(tags!.Count, Is.EqualTo(2)); + Assert.That(tags, Does.Contain("csharp")); + Assert.That(tags, Does.Contain("dotnet")); + Assert.That(tags, Does.Not.Contain("java")); + Assert.That(tags, Does.Not.Contain("python")); + } + + [Test] + public void should_handle_mixed_add_and_remove_operations() + { + // Arrange + using var testFixture = TestFixture.Create(rockDb => + { + rockDb.AddMergeableStore>, TagsStore>("tags", new ListMergeOperator()); + }); + + var store = testFixture.GetStore(); + var key = "article-1"; + + // Act - Interleave adds and removes + store.AddTags(key, "a", "b", "c"); + store.RemoveTags(key, "b"); + store.AddTags(key, "d", "e"); + store.RemoveTags(key, "a", "e"); + + // Assert - Should have: c, d + var tags = store.GetTags(key); + Assert.That(tags, Is.Not.Null); + Assert.That(tags!.Count, Is.EqualTo(2)); + Assert.That(tags, Does.Contain("c")); + Assert.That(tags, Does.Contain("d")); + } + + [Test] + public void should_handle_remove_nonexistent_item_gracefully() + { + // Arrange + using var testFixture = TestFixture.Create(rockDb => + { + rockDb.AddMergeableStore>, TagsStore>("tags", new ListMergeOperator()); + }); + + var store = testFixture.GetStore(); + var key = "article-1"; + + // Act - Try to remove items that don't exist + store.AddTags(key, "csharp"); + store.RemoveTags(key, "nonexistent"); + + // Assert - Original item should still be there + var tags = store.GetTags(key); + Assert.That(tags, Is.Not.Null); + Assert.That(tags!.Count, Is.EqualTo(1)); + Assert.That(tags, Does.Contain("csharp")); + } + + [Test] + public void should_remove_only_first_occurrence_of_duplicate_items() + { + // Arrange + using var testFixture = TestFixture.Create(rockDb => + { + rockDb.AddMergeableStore>, TagsStore>("tags", new ListMergeOperator()); + }); + + var store = testFixture.GetStore(); + var key = "article-1"; + + // Act - Add duplicate items, then remove one + store.AddTags(key, "tag", "tag", "tag"); + store.RemoveTags(key, "tag"); + + // Assert - Should have 2 remaining + var tags = store.GetTags(key); + Assert.That(tags, Is.Not.Null); + Assert.That(tags!.Count, Is.EqualTo(2)); + Assert.That(tags[0], Is.EqualTo("tag")); + Assert.That(tags[1], Is.EqualTo("tag")); + } } From 36a7381b2219c7c3f67a41eded5864ad674149cb Mon Sep 17 00:00:00 2001 From: Havret Date: Thu, 4 Dec 2025 15:04:27 +0100 Subject: [PATCH 05/19] wip --- src/RocksDb.Extensions/IMergeAccessor.cs | 17 +++ src/RocksDb.Extensions/IMergeOperator.cs | 26 ++-- .../IMergeableRocksDbStore.cs | 5 +- src/RocksDb.Extensions/IRocksDbBuilder.cs | 46 +++---- src/RocksDb.Extensions/MergeAccessor.cs | 96 ++++++++++++++ .../MergeOperators/Int64AddMergeOperator.cs | 9 +- .../MergeOperators/ListAppendMergeOperator.cs | 9 +- .../MergeOperators/ListMergeOperator.cs | 120 +++++++++--------- .../MergeableRocksDbStore.cs | 39 ++++-- src/RocksDb.Extensions/RocksDbBuilder.cs | 110 ++++++++++------ .../MergeOperatorTests.cs | 59 ++++----- 11 files changed, 346 insertions(+), 190 deletions(-) create mode 100644 src/RocksDb.Extensions/IMergeAccessor.cs create mode 100644 src/RocksDb.Extensions/MergeAccessor.cs diff --git a/src/RocksDb.Extensions/IMergeAccessor.cs b/src/RocksDb.Extensions/IMergeAccessor.cs new file mode 100644 index 0000000..3abf085 --- /dev/null +++ b/src/RocksDb.Extensions/IMergeAccessor.cs @@ -0,0 +1,17 @@ +using System.ComponentModel; + +namespace RocksDb.Extensions; + +#pragma warning disable CS1591 + +/// +/// This interface is not intended to be used directly by the clients of the library. +/// It provides merge operation support with a separate operand type. +/// +[EditorBrowsable(EditorBrowsableState.Never)] +public interface IMergeAccessor +{ + void Merge(TKey key, TOperand operand); +} + +#pragma warning restore CS1591 diff --git a/src/RocksDb.Extensions/IMergeOperator.cs b/src/RocksDb.Extensions/IMergeOperator.cs index ec24583..28206af 100644 --- a/src/RocksDb.Extensions/IMergeOperator.cs +++ b/src/RocksDb.Extensions/IMergeOperator.cs @@ -5,8 +5,17 @@ namespace RocksDb.Extensions; /// Merge operators allow efficient updates without requiring a separate read before write, /// which is particularly useful for counters, list appends, set unions, and other accumulative operations. /// -/// The type of the value being merged. -public interface IMergeOperator +/// The type of the value stored in the database. +/// The type of the merge operand (the delta/change to apply). +/// +/// The separation of and allows for flexible merge patterns: +/// +/// For counters: TValue=long, TOperand=long (same type) +/// For list append: TValue=IList<T>, TOperand=IList<T> (same type) +/// For list with add/remove: TValue=IList<T>, TOperand=ListOperation<T> (different types) +/// +/// +public interface IMergeOperator { /// /// Gets the name of the merge operator. This name is stored in the database @@ -19,10 +28,10 @@ public interface IMergeOperator /// Called when a Get operation encounters merge operands and needs to produce the final value. /// /// The key being merged. - /// The existing value in the database. For value types, this will be default if no value exists (check hasExistingValue). + /// The existing value in the database. For value types, this will be default if no value exists. /// The list of merge operands to apply, in order. - /// The merged value. - TValue FullMerge(ReadOnlySpan key, TValue existingValue, IReadOnlyList operands); + /// The merged value to store. + TValue FullMerge(ReadOnlySpan key, TValue existingValue, IReadOnlyList operands); /// /// Performs a partial merge of multiple operands without the existing value. @@ -31,9 +40,6 @@ public interface IMergeOperator /// /// The key being merged. /// The list of merge operands to combine, in order. - /// - /// The combined operand if partial merge is possible; otherwise, default to indicate - /// that partial merge is not supported and operands should be kept separate. - /// - TValue PartialMerge(ReadOnlySpan key, IReadOnlyList operands); + /// The combined operand. + TOperand PartialMerge(ReadOnlySpan key, IReadOnlyList operands); } diff --git a/src/RocksDb.Extensions/IMergeableRocksDbStore.cs b/src/RocksDb.Extensions/IMergeableRocksDbStore.cs index c4570db..de84e34 100644 --- a/src/RocksDb.Extensions/IMergeableRocksDbStore.cs +++ b/src/RocksDb.Extensions/IMergeableRocksDbStore.cs @@ -5,7 +5,8 @@ namespace RocksDb.Extensions; /// /// The type of the store's keys. /// The type of the store's values. -public interface IMergeableRocksDbStore +/// The type of the merge operand. +public interface IMergeableRocksDbStore { /// /// Performs an atomic merge operation on the value associated with the specified key. @@ -15,5 +16,5 @@ public interface IMergeableRocksDbStore /// /// The key to merge the operand with. /// The operand to merge with the existing value. - void Merge(TKey key, TValue operand); + void Merge(TKey key, TOperand operand); } diff --git a/src/RocksDb.Extensions/IRocksDbBuilder.cs b/src/RocksDb.Extensions/IRocksDbBuilder.cs index 45680d1..a4a3f12 100644 --- a/src/RocksDb.Extensions/IRocksDbBuilder.cs +++ b/src/RocksDb.Extensions/IRocksDbBuilder.cs @@ -23,46 +23,36 @@ public interface IRocksDbBuilder /// IRocksDbBuilder AddStore(string columnFamily) where TStore : RocksDbStore; - /// - /// Adds a RocksDB store to the builder for the specified column family with a merge operator. - /// - /// The name of the column family to associate with the store. - /// The merge operator to use for atomic read-modify-write operations. - /// The type of the store's key. - /// The type of the store's value. - /// The type of the store to add. - /// The builder instance for method chaining. - /// Thrown if the specified column family is already registered. - /// - /// The type must be a concrete implementation of the abstract class - /// . Each store is registered uniquely based on its column family name. - /// - /// The merge operator enables efficient atomic updates without requiring a separate read operation. - /// This is useful for counters, list appends, set unions, and other accumulative operations. - /// - /// Stores can also be resolved as keyed services using their associated column family name. - /// Use GetRequiredKeyedService<TStore>(columnFamily) to retrieve a specific store instance. - /// - IRocksDbBuilder AddStore(string columnFamily, IMergeOperator mergeOperator) where TStore : RocksDbStore; - /// /// Adds a mergeable RocksDB store to the builder for the specified column family. - /// This method enforces that the store inherits from + /// This method enforces that the store inherits from /// and requires a merge operator, providing compile-time safety for merge operations. /// /// The name of the column family to associate with the store. /// The merge operator to use for atomic read-modify-write operations. /// The type of the store's key. - /// The type of the store's value. - /// The type of the store to add. Must inherit from . + /// The type of the store's values (the stored state). + /// The type of the merge operand (the delta/change to apply). + /// The type of the store to add. Must inherit from . /// The builder instance for method chaining. /// Thrown if the specified column family is already registered. /// + /// /// Use this method when your store needs merge operations. The constraint /// ensures that only stores designed for merge operations can be registered with this method. - /// + /// + /// + /// The separation of and allows for flexible patterns: + /// + /// Counters: TValue=long, TOperand=long (same type) + /// List append: TValue=IList<T>, TOperand=IList<T> (same type) + /// List with add/remove: TValue=IList<T>, TOperand=ListOperation<T> (different types) + /// + /// + /// /// For stores that don't need merge operations, use instead. + /// /// - IRocksDbBuilder AddMergeableStore(string columnFamily, IMergeOperator mergeOperator) - where TStore : MergeableRocksDbStore; + IRocksDbBuilder AddMergeableStore(string columnFamily, IMergeOperator mergeOperator) + where TStore : MergeableRocksDbStore; } \ No newline at end of file diff --git a/src/RocksDb.Extensions/MergeAccessor.cs b/src/RocksDb.Extensions/MergeAccessor.cs new file mode 100644 index 0000000..155f96d --- /dev/null +++ b/src/RocksDb.Extensions/MergeAccessor.cs @@ -0,0 +1,96 @@ +using System.Buffers; +using CommunityToolkit.HighPerformance.Buffers; +using RocksDbSharp; + +namespace RocksDb.Extensions; + +internal class MergeAccessor : IMergeAccessor +{ + private const int MaxStackSize = 256; + + private readonly ISerializer _keySerializer; + private readonly ISerializer _operandSerializer; + private readonly RocksDbSharp.RocksDb _db; + private readonly ColumnFamilyHandle _columnFamilyHandle; + + public MergeAccessor( + RocksDbSharp.RocksDb db, + ColumnFamilyHandle columnFamilyHandle, + ISerializer keySerializer, + ISerializer operandSerializer) + { + _db = db; + _columnFamilyHandle = columnFamilyHandle; + _keySerializer = keySerializer; + _operandSerializer = operandSerializer; + } + + public void Merge(TKey key, TOperand operand) + { + byte[]? rentedKeyBuffer = null; + bool useSpanAsKey; + // ReSharper disable once AssignmentInConditionalExpression + Span keyBuffer = (useSpanAsKey = _keySerializer.TryCalculateSize(ref key, out var keySize)) + ? keySize < MaxStackSize + ? stackalloc byte[keySize] + : (rentedKeyBuffer = ArrayPool.Shared.Rent(keySize)).AsSpan(0, keySize) + : Span.Empty; + + ReadOnlySpan keySpan = keyBuffer; + ArrayPoolBufferWriter? keyBufferWriter = null; + + byte[]? rentedOperandBuffer = null; + bool useSpanAsOperand; + // ReSharper disable once AssignmentInConditionalExpression + Span operandBuffer = (useSpanAsOperand = _operandSerializer.TryCalculateSize(ref operand, out var operandSize)) + ? operandSize < MaxStackSize + ? stackalloc byte[operandSize] + : (rentedOperandBuffer = ArrayPool.Shared.Rent(operandSize)).AsSpan(0, operandSize) + : Span.Empty; + + + ReadOnlySpan operandSpan = operandBuffer; + ArrayPoolBufferWriter? operandBufferWriter = null; + + try + { + if (useSpanAsKey) + { + _keySerializer.WriteTo(ref key, ref keyBuffer); + } + else + { + keyBufferWriter = new ArrayPoolBufferWriter(); + _keySerializer.WriteTo(ref key, keyBufferWriter); + keySpan = keyBufferWriter.WrittenSpan; + } + + if (useSpanAsOperand) + { + _operandSerializer.WriteTo(ref operand, ref operandBuffer); + } + else + { + operandBufferWriter = new ArrayPoolBufferWriter(); + _operandSerializer.WriteTo(ref operand, operandBufferWriter); + operandSpan = operandBufferWriter.WrittenSpan; + } + + _db.Merge(keySpan, operandSpan, _columnFamilyHandle); + } + finally + { + keyBufferWriter?.Dispose(); + operandBufferWriter?.Dispose(); + if (rentedKeyBuffer is not null) + { + ArrayPool.Shared.Return(rentedKeyBuffer); + } + + if (rentedOperandBuffer is not null) + { + ArrayPool.Shared.Return(rentedOperandBuffer); + } + } + } +} diff --git a/src/RocksDb.Extensions/MergeOperators/Int64AddMergeOperator.cs b/src/RocksDb.Extensions/MergeOperators/Int64AddMergeOperator.cs index 5682183..0954bc9 100644 --- a/src/RocksDb.Extensions/MergeOperators/Int64AddMergeOperator.cs +++ b/src/RocksDb.Extensions/MergeOperators/Int64AddMergeOperator.cs @@ -6,18 +6,19 @@ namespace RocksDb.Extensions.MergeOperators; /// /// /// -/// public class CounterStore : RocksDbStore<string, long> +/// public class CounterStore : MergeableRocksDbStore<string, long, long> /// { -/// public CounterStore(IRocksDbAccessor<string, long> accessor) : base(accessor) { } +/// public CounterStore(IRocksDbAccessor<string, long> accessor, IMergeAccessor<string, long> mergeAccessor) +/// : base(accessor, mergeAccessor) { } /// /// public void Increment(string key, long delta = 1) => Merge(key, delta); /// } /// /// // Registration: -/// builder.AddStore<string, long, CounterStore>("counters", new Int64AddMergeOperator()); +/// builder.AddMergeableStore<string, long, long, CounterStore>("counters", new Int64AddMergeOperator()); /// /// -public class Int64AddMergeOperator : IMergeOperator +public class Int64AddMergeOperator : IMergeOperator { /// public string Name => "Int64AddMergeOperator"; diff --git a/src/RocksDb.Extensions/MergeOperators/ListAppendMergeOperator.cs b/src/RocksDb.Extensions/MergeOperators/ListAppendMergeOperator.cs index 4da0194..4a2fce7 100644 --- a/src/RocksDb.Extensions/MergeOperators/ListAppendMergeOperator.cs +++ b/src/RocksDb.Extensions/MergeOperators/ListAppendMergeOperator.cs @@ -7,9 +7,10 @@ namespace RocksDb.Extensions.MergeOperators; /// The type of elements in the list. /// /// -/// public class EventLogStore : RocksDbStore<string, IList<string>> +/// public class EventLogStore : MergeableRocksDbStore<string, IList<string>, IList<string>> /// { -/// public EventLogStore(IRocksDbAccessor<string, IList<string>> accessor) : base(accessor) { } +/// public EventLogStore(IRocksDbAccessor<string, IList<string>> accessor, IMergeAccessor<string, IList<string>> mergeAccessor) +/// : base(accessor, mergeAccessor) { } /// /// public void AppendEvent(string key, string eventData) /// { @@ -18,10 +19,10 @@ namespace RocksDb.Extensions.MergeOperators; /// } /// /// // Registration: -/// builder.AddStore<string, IList<string>, EventLogStore>("events", new ListAppendMergeOperator<string>()); +/// builder.AddMergeableStore<string, IList<string>, IList<string>, EventLogStore>("events", new ListAppendMergeOperator<string>()); /// /// -public class ListAppendMergeOperator : IMergeOperator> +public class ListAppendMergeOperator : IMergeOperator, IList> { /// public string Name => $"ListAppendMergeOperator<{typeof(T).Name}>"; diff --git a/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs b/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs index d9c0205..8ee62ec 100644 --- a/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs +++ b/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs @@ -8,29 +8,23 @@ namespace RocksDb.Extensions.MergeOperators; /// The type of elements in the list. /// /// -/// public class TagsStore : RocksDbStore<string, IList<string>> +/// public class TagsStore : MergeableRocksDbStore<string, IList<string>, ListOperation<string>> /// { -/// public TagsStore(IRocksDbAccessor<string, IList<string>> accessor) : base(accessor) { } +/// public TagsStore(IRocksDbAccessor<string, IList<string>> accessor, IMergeAccessor<string, ListOperation<string>> mergeAccessor) +/// : base(accessor, mergeAccessor) { } /// -/// public void AddTags(string key, params string[] tags) -/// { -/// Merge(key, new List<ListOperation<string>> { ListOperation<string>.Add(tags) }); -/// } -/// -/// public void RemoveTags(string key, params string[] tags) -/// { -/// Merge(key, new List<ListOperation<string>> { ListOperation<string>.Remove(tags) }); -/// } +/// public void AddTags(string key, params string[] tags) => Merge(key, ListOperation<string>.Add(tags)); +/// public void RemoveTags(string key, params string[] tags) => Merge(key, ListOperation<string>.Remove(tags)); /// } /// /// // Registration: -/// builder.AddStore<string, IList<string>, TagsStore>("tags", new ListMergeOperator<string>()); +/// builder.AddMergeableStore<string, IList<string>, ListOperation<string>, TagsStore>("tags", new ListMergeOperator<string>()); /// /// /// /// /// The value type stored in RocksDB is IList<T> (the actual list contents), -/// but merge operands are IList<ListOperation<T>> (the operations to apply). +/// while merge operands are ListOperation<T> (the operations to apply). /// /// /// Remove operations delete the first occurrence of each item (same as ). @@ -41,79 +35,91 @@ namespace RocksDb.Extensions.MergeOperators; /// which has less serialization overhead. /// /// -public class ListMergeOperator : IMergeOperator>> +public class ListMergeOperator : IMergeOperator, ListOperation> { /// public string Name => $"ListMergeOperator<{typeof(T).Name}>"; /// - public IList> FullMerge( + public IList FullMerge( ReadOnlySpan key, - IList> existingValue, - IReadOnlyList>> operands) + IList existingValue, + IReadOnlyList> operands) { // Start with existing items or empty list - var result = new List(); - - // If there's an existing value, it contains the accumulated operations from previous merges - // We need to apply those operations first - if (existingValue != null) - { - ApplyOperations(result, existingValue); - } + var result = existingValue != null ? new List(existingValue) : new List(); - // Apply all new operands in order - foreach (var operandBatch in operands) + // Apply all operands in order + foreach (var operand in operands) { - ApplyOperations(result, operandBatch); + ApplyOperation(result, operand); } - // Return the final list wrapped as a single Add operation - // This collapses all operations into the final state - return new List> { ListOperation.Add(result) }; + return result; } /// - public IList> PartialMerge( + public ListOperation PartialMerge( ReadOnlySpan key, - IReadOnlyList>> operands) + IReadOnlyList> operands) { - // Combine all operations into a single list - // We preserve all operations rather than trying to resolve them - // because removes can't be safely combined without knowing the base state - var combined = new List>(); + // Combine all operations into a single compound operation + // We preserve all items in order - adds followed by removes + // This is a simplification; a more sophisticated implementation could optimize + var allAdds = new List(); + var allRemoves = new List(); - foreach (var operandBatch in operands) - { - foreach (var op in operandBatch) - { - combined.Add(op); - } - } - - return combined; - } - - private static void ApplyOperations(List result, IList> operations) - { - foreach (var operation in operations) + foreach (var operand in operands) { - switch (operation.Type) + switch (operand.Type) { case OperationType.Add: - foreach (var item in operation.Items) + foreach (var item in operand.Items) { - result.Add(item); + allAdds.Add(item); } break; - case OperationType.Remove: - foreach (var item in operation.Items) + foreach (var item in operand.Items) { - result.Remove(item); // Removes first occurrence + allRemoves.Add(item); } break; } } + + // For partial merge, we can only safely combine if there are no removes + // (since removes depend on the order of operations and existing state) + // For now, just return the first operand's type with combined items + // A better approach would be to return a compound operation, but that complicates the type + if (allRemoves.Count == 0) + { + return ListOperation.Add(allAdds); + } + + // If we have removes, we can't safely combine without knowing the state + // Return the combined adds (removes will be re-applied during full merge) + // This is a simplification - in practice, operands are kept separate if partial merge fails + return ListOperation.Add(allAdds); + } + + private static void ApplyOperation(List result, ListOperation operation) + { + switch (operation.Type) + { + case OperationType.Add: + foreach (var item in operation.Items) + { + result.Add(item); + } + break; + + case OperationType.Remove: + foreach (var item in operation.Items) + { + result.Remove(item); // Removes first occurrence + } + break; + } } } diff --git a/src/RocksDb.Extensions/MergeableRocksDbStore.cs b/src/RocksDb.Extensions/MergeableRocksDbStore.cs index 4e2cb22..52cfe85 100644 --- a/src/RocksDb.Extensions/MergeableRocksDbStore.cs +++ b/src/RocksDb.Extensions/MergeableRocksDbStore.cs @@ -7,39 +7,54 @@ namespace RocksDb.Extensions; /// /// The type of the store's keys. /// The type of the store's values. +/// The type of the merge operand. /// /// /// Merge operations are useful for: -/// - Counters: Increment/decrement without reading current value -/// - Lists: Append items without reading the entire list -/// - Sets: Add/remove items atomically +/// - Counters: Increment/decrement without reading current value (TValue=long, TOperand=long) +/// - Lists: Append items without reading the entire list (TValue=IList<T>, TOperand=IList<T>) +/// - Lists with add/remove: Modify lists atomically (TValue=IList<T>, TOperand=ListOperation<T>) /// /// /// When using this base class, you must register the store with a merge operator using -/// . +/// . /// /// /// /// -/// public class CounterStore : MergeableRocksDbStore<string, long> +/// // Counter store where value and operand are the same type +/// public class CounterStore : MergeableRocksDbStore<string, long, long> /// { -/// public CounterStore(IRocksDbAccessor<string, long> accessor) : base(accessor) { } +/// public CounterStore(IRocksDbAccessor<string, long> accessor, IMergeAccessor<string, long> mergeAccessor) +/// : base(accessor, mergeAccessor) { } /// /// public void Increment(string key, long delta = 1) => Merge(key, delta); /// } /// -/// // Registration: -/// builder.AddMergeableStore<string, long, CounterStore>("counters", new Int64AddMergeOperator()); +/// // Tags store where value is IList<string> but operand is ListOperation<string> +/// public class TagsStore : MergeableRocksDbStore<string, IList<string>, ListOperation<string>> +/// { +/// public TagsStore(IRocksDbAccessor<string, IList<string>> accessor, IMergeAccessor<string, ListOperation<string>> mergeAccessor) +/// : base(accessor, mergeAccessor) { } +/// +/// public void AddTag(string key, string tag) => Merge(key, ListOperation<string>.Add(tag)); +/// public void RemoveTag(string key, string tag) => Merge(key, ListOperation<string>.Remove(tag)); +/// } /// /// -public abstract class MergeableRocksDbStore : RocksDbStore, IMergeableRocksDbStore +public abstract class MergeableRocksDbStore : RocksDbStore, IMergeableRocksDbStore { + private readonly IMergeAccessor _mergeAccessor; + /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// The RocksDB accessor to use for database operations. - protected MergeableRocksDbStore(IRocksDbAccessor rocksDbAccessor) : base(rocksDbAccessor) + /// The merge accessor to use for merge operations. + protected MergeableRocksDbStore(IRocksDbAccessor rocksDbAccessor, IMergeAccessor mergeAccessor) + : base(rocksDbAccessor) { + _mergeAccessor = mergeAccessor; } /// @@ -49,5 +64,5 @@ protected MergeableRocksDbStore(IRocksDbAccessor rocksDbAccessor) /// /// The key to merge the operand with. /// The operand to merge with the existing value. - public new void Merge(TKey key, TValue operand) => base.Merge(key, operand); + public void Merge(TKey key, TOperand operand) => _mergeAccessor.Merge(key, operand); } diff --git a/src/RocksDb.Extensions/RocksDbBuilder.cs b/src/RocksDb.Extensions/RocksDbBuilder.cs index f266ede..1bf9370 100644 --- a/src/RocksDb.Extensions/RocksDbBuilder.cs +++ b/src/RocksDb.Extensions/RocksDbBuilder.cs @@ -20,21 +20,39 @@ public RocksDbBuilder(IServiceCollection serviceCollection) public IRocksDbBuilder AddStore(string columnFamily) where TStore : RocksDbStore { - return AddStoreInternal(columnFamily, null); - } - - public IRocksDbBuilder AddStore(string columnFamily, IMergeOperator mergeOperator) where TStore : RocksDbStore - { - return AddStoreInternal(columnFamily, mergeOperator); - } + if (!_columnFamilyLookup.Add(columnFamily)) + { + throw new InvalidOperationException($"{columnFamily} is already registered."); + } - public IRocksDbBuilder AddMergeableStore(string columnFamily, IMergeOperator mergeOperator) - where TStore : MergeableRocksDbStore - { - return AddStoreInternal(columnFamily, mergeOperator); + _ = _serviceCollection.Configure(options => + { + options.ColumnFamilies.Add(columnFamily); + }); + + _serviceCollection.AddKeyedSingleton(columnFamily, (provider, _) => + { + var rocksDbContext = provider.GetRequiredService(); + var columnFamilyHandle = rocksDbContext.Db.GetColumnFamily(columnFamily); + var rocksDbOptions = provider.GetRequiredService>(); + var keySerializer = CreateSerializer(rocksDbOptions.Value.SerializerFactories); + var valueSerializer = CreateSerializer(rocksDbOptions.Value.SerializerFactories); + var rocksDbAccessor = new RocksDbAccessor( + rocksDbContext, + new ColumnFamily(columnFamilyHandle, columnFamily), + keySerializer, + valueSerializer + ); + return ActivatorUtilities.CreateInstance(provider, rocksDbAccessor); + }); + + _serviceCollection.TryAddSingleton(typeof(TStore), provider => provider.GetRequiredKeyedService(columnFamily)); + + return this; } - private IRocksDbBuilder AddStoreInternal(string columnFamily, IMergeOperator? mergeOperator) where TStore : RocksDbStore + public IRocksDbBuilder AddMergeableStore(string columnFamily, IMergeOperator mergeOperator) + where TStore : MergeableRocksDbStore { if (!_columnFamilyLookup.Add(columnFamily)) { @@ -45,12 +63,10 @@ private IRocksDbBuilder AddStoreInternal(string columnFami { options.ColumnFamilies.Add(columnFamily); - if (mergeOperator != null) - { - var valueSerializer = CreateSerializer(options.SerializerFactories); - var config = CreateMergeOperatorConfig(mergeOperator, valueSerializer); - options.MergeOperators[columnFamily] = config; - } + var valueSerializer = CreateSerializer(options.SerializerFactories); + var operandSerializer = CreateSerializer(options.SerializerFactories); + var config = CreateMergeOperatorConfig(mergeOperator, valueSerializer, operandSerializer); + options.MergeOperators[columnFamily] = config; }); _serviceCollection.AddKeyedSingleton(columnFamily, (provider, _) => @@ -60,13 +76,23 @@ private IRocksDbBuilder AddStoreInternal(string columnFami var rocksDbOptions = provider.GetRequiredService>(); var keySerializer = CreateSerializer(rocksDbOptions.Value.SerializerFactories); var valueSerializer = CreateSerializer(rocksDbOptions.Value.SerializerFactories); + var operandSerializer = CreateSerializer(rocksDbOptions.Value.SerializerFactories); + var rocksDbAccessor = new RocksDbAccessor( rocksDbContext, new ColumnFamily(columnFamilyHandle, columnFamily), keySerializer, valueSerializer ); - return ActivatorUtilities.CreateInstance(provider, rocksDbAccessor); + + var mergeAccessor = new MergeAccessor( + rocksDbContext.Db, + columnFamilyHandle, + keySerializer, + operandSerializer + ); + + return ActivatorUtilities.CreateInstance(provider, rocksDbAccessor, mergeAccessor); }); _serviceCollection.TryAddSingleton(typeof(TStore), provider => provider.GetRequiredKeyedService(columnFamily)); @@ -74,7 +100,10 @@ private IRocksDbBuilder AddStoreInternal(string columnFami return this; } - private static MergeOperatorConfig CreateMergeOperatorConfig(IMergeOperator mergeOperator, ISerializer valueSerializer) + private static MergeOperatorConfig CreateMergeOperatorConfig( + IMergeOperator mergeOperator, + ISerializer valueSerializer, + ISerializer operandSerializer) { return new MergeOperatorConfig { @@ -82,22 +111,23 @@ private static MergeOperatorConfig CreateMergeOperatorConfig(IMergeOpera ValueSerializer = valueSerializer, FullMerge = (ReadOnlySpan key, bool hasExistingValue, ReadOnlySpan existingValue, global::RocksDbSharp.MergeOperators.OperandsEnumerator operands, out bool success) => { - return FullMergeCallback(key, hasExistingValue, existingValue, operands, mergeOperator, valueSerializer, out success); + return FullMergeCallback(key, hasExistingValue, existingValue, operands, mergeOperator, valueSerializer, operandSerializer, out success); }, PartialMerge = (ReadOnlySpan key, global::RocksDbSharp.MergeOperators.OperandsEnumerator operands, out bool success) => { - return PartialMergeCallback(key, operands, mergeOperator, valueSerializer, out success); + return PartialMergeCallback(key, operands, mergeOperator, operandSerializer, out success); } }; } - private static byte[] FullMergeCallback( + private static byte[] FullMergeCallback( ReadOnlySpan key, bool hasExistingValue, ReadOnlySpan existingValue, global::RocksDbSharp.MergeOperators.OperandsEnumerator operands, - IMergeOperator mergeOperator, + IMergeOperator mergeOperator, ISerializer valueSerializer, + ISerializer operandSerializer, out bool success) { success = true; @@ -106,54 +136,54 @@ private static byte[] FullMergeCallback( TValue existing = hasExistingValue ? valueSerializer.Deserialize(existingValue) : default!; // Deserialize all operands - var operandList = new List(operands.Count); + var operandList = new List(operands.Count); for (int i = 0; i < operands.Count; i++) { - operandList.Add(valueSerializer.Deserialize(operands.Get(i))); + operandList.Add(operandSerializer.Deserialize(operands.Get(i))); } - // Call the user's merge operator + // Call the user's merge operator - returns TValue var result = mergeOperator.FullMerge(key, existing, operandList); - // Serialize the result + // Serialize the result as TValue return SerializeValue(result, valueSerializer); } - private static byte[] PartialMergeCallback( + private static byte[] PartialMergeCallback( ReadOnlySpan key, global::RocksDbSharp.MergeOperators.OperandsEnumerator operands, - IMergeOperator mergeOperator, - ISerializer valueSerializer, + IMergeOperator mergeOperator, + ISerializer operandSerializer, out bool success) { // Deserialize all operands - var operandList = new List(operands.Count); + var operandList = new List(operands.Count); for (int i = 0; i < operands.Count; i++) { - operandList.Add(valueSerializer.Deserialize(operands.Get(i))); + operandList.Add(operandSerializer.Deserialize(operands.Get(i))); } - // Call the user's partial merge operator + // Call the user's partial merge operator - returns TOperand var result = mergeOperator.PartialMerge(key, operandList); success = true; - // Serialize the result - return SerializeValue(result, valueSerializer); + // Serialize the result as TOperand + return SerializeValue(result, operandSerializer); } - private static byte[] SerializeValue(TValue value, ISerializer valueSerializer) + private static byte[] SerializeValue(T value, ISerializer serializer) { - if (valueSerializer.TryCalculateSize(ref value, out var size)) + if (serializer.TryCalculateSize(ref value, out var size)) { var buffer = new byte[size]; var span = buffer.AsSpan(); - valueSerializer.WriteTo(ref value, ref span); + serializer.WriteTo(ref value, ref span); return buffer; } else { using var bufferWriter = new ArrayPoolBufferWriter(); - valueSerializer.WriteTo(ref value, bufferWriter); + serializer.WriteTo(ref value, bufferWriter); return bufferWriter.WrittenSpan.ToArray(); } } diff --git a/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs b/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs index 4655898..5f66710 100644 --- a/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs +++ b/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs @@ -7,9 +7,10 @@ namespace RocksDb.Extensions.Tests; /// /// Store that uses merge operations for testing counters. /// -public class CounterStore : MergeableRocksDbStore +public class CounterStore : MergeableRocksDbStore { - public CounterStore(IRocksDbAccessor rocksDbAccessor) : base(rocksDbAccessor) + public CounterStore(IRocksDbAccessor rocksDbAccessor, IMergeAccessor mergeAccessor) + : base(rocksDbAccessor, mergeAccessor) { } @@ -19,9 +20,10 @@ public CounterStore(IRocksDbAccessor rocksDbAccessor) : base(rocks /// /// Store that uses merge operations for testing list appends. /// -public class EventLogStore : MergeableRocksDbStore> +public class EventLogStore : MergeableRocksDbStore, IList> { - public EventLogStore(IRocksDbAccessor> rocksDbAccessor) : base(rocksDbAccessor) + public EventLogStore(IRocksDbAccessor> rocksDbAccessor, IMergeAccessor> mergeAccessor) + : base(rocksDbAccessor, mergeAccessor) { } @@ -31,30 +33,21 @@ public EventLogStore(IRocksDbAccessor> rocksDbAccessor) : /// /// Store that uses merge operations for testing list operations (add/remove). /// -public class TagsStore : MergeableRocksDbStore>> +public class TagsStore : MergeableRocksDbStore, ListOperation> { - public TagsStore(IRocksDbAccessor>> rocksDbAccessor) : base(rocksDbAccessor) + public TagsStore(IRocksDbAccessor> rocksDbAccessor, IMergeAccessor> mergeAccessor) + : base(rocksDbAccessor, mergeAccessor) { } public void AddTags(string key, params string[] tags) { - Merge(key, new List> { ListOperation.Add(tags) }); + Merge(key, ListOperation.Add(tags)); } public void RemoveTags(string key, params string[] tags) { - Merge(key, new List> { ListOperation.Remove(tags) }); - } - - public IList? GetTags(string key) - { - if (TryGet(key, out var operations) && operations != null && operations.Count > 0) - { - // The merged result is a single Add operation containing all items - return operations[0].Items; - } - return null; + Merge(key, ListOperation.Remove(tags)); } } @@ -66,7 +59,7 @@ public void should_increment_counter_using_merge_operation() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddMergeableStore("counters", new Int64AddMergeOperator()); + rockDb.AddMergeableStore("counters", new Int64AddMergeOperator()); }); var store = testFixture.GetStore(); @@ -88,7 +81,7 @@ public void should_handle_counter_with_initial_value() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddMergeableStore("counters", new Int64AddMergeOperator()); + rockDb.AddMergeableStore("counters", new Int64AddMergeOperator()); }); var store = testFixture.GetStore(); @@ -109,7 +102,7 @@ public void should_append_to_list_using_merge_operation() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddMergeableStore, EventLogStore>("events", new ListAppendMergeOperator()); + rockDb.AddMergeableStore, IList, EventLogStore>("events", new ListAppendMergeOperator()); }); var store = testFixture.GetStore(); @@ -135,7 +128,7 @@ public void should_append_to_existing_list_using_merge_operation() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddMergeableStore, EventLogStore>("events", new ListAppendMergeOperator()); + rockDb.AddMergeableStore, IList, EventLogStore>("events", new ListAppendMergeOperator()); }); var store = testFixture.GetStore(); @@ -159,7 +152,7 @@ public void should_handle_multiple_keys_with_merge_operations() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddMergeableStore("counters", new Int64AddMergeOperator()); + rockDb.AddMergeableStore("counters", new Int64AddMergeOperator()); }); var store = testFixture.GetStore(); @@ -187,7 +180,7 @@ public void should_add_items_to_list_using_list_merge_operator() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddMergeableStore>, TagsStore>("tags", new ListMergeOperator()); + rockDb.AddMergeableStore, ListOperation, TagsStore>("tags", new ListMergeOperator()); }); var store = testFixture.GetStore(); @@ -198,7 +191,7 @@ public void should_add_items_to_list_using_list_merge_operator() store.AddTags(key, "rocksdb"); // Assert - var tags = store.GetTags(key); + Assert.That(store.TryGet(key, out var tags), Is.True); Assert.That(tags, Is.Not.Null); Assert.That(tags!.Count, Is.EqualTo(3)); Assert.That(tags, Does.Contain("csharp")); @@ -212,7 +205,7 @@ public void should_remove_items_from_list_using_list_merge_operator() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddMergeableStore>, TagsStore>("tags", new ListMergeOperator()); + rockDb.AddMergeableStore, ListOperation, TagsStore>("tags", new ListMergeOperator()); }); var store = testFixture.GetStore(); @@ -223,7 +216,7 @@ public void should_remove_items_from_list_using_list_merge_operator() store.RemoveTags(key, "java", "python"); // Assert - var tags = store.GetTags(key); + Assert.That(store.TryGet(key, out var tags), Is.True); Assert.That(tags, Is.Not.Null); Assert.That(tags!.Count, Is.EqualTo(2)); Assert.That(tags, Does.Contain("csharp")); @@ -238,7 +231,7 @@ public void should_handle_mixed_add_and_remove_operations() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddMergeableStore>, TagsStore>("tags", new ListMergeOperator()); + rockDb.AddMergeableStore, ListOperation, TagsStore>("tags", new ListMergeOperator()); }); var store = testFixture.GetStore(); @@ -251,7 +244,7 @@ public void should_handle_mixed_add_and_remove_operations() store.RemoveTags(key, "a", "e"); // Assert - Should have: c, d - var tags = store.GetTags(key); + Assert.That(store.TryGet(key, out var tags), Is.True); Assert.That(tags, Is.Not.Null); Assert.That(tags!.Count, Is.EqualTo(2)); Assert.That(tags, Does.Contain("c")); @@ -264,7 +257,7 @@ public void should_handle_remove_nonexistent_item_gracefully() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddMergeableStore>, TagsStore>("tags", new ListMergeOperator()); + rockDb.AddMergeableStore, ListOperation, TagsStore>("tags", new ListMergeOperator()); }); var store = testFixture.GetStore(); @@ -275,7 +268,7 @@ public void should_handle_remove_nonexistent_item_gracefully() store.RemoveTags(key, "nonexistent"); // Assert - Original item should still be there - var tags = store.GetTags(key); + Assert.That(store.TryGet(key, out var tags), Is.True); Assert.That(tags, Is.Not.Null); Assert.That(tags!.Count, Is.EqualTo(1)); Assert.That(tags, Does.Contain("csharp")); @@ -287,7 +280,7 @@ public void should_remove_only_first_occurrence_of_duplicate_items() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddMergeableStore>, TagsStore>("tags", new ListMergeOperator()); + rockDb.AddMergeableStore, ListOperation, TagsStore>("tags", new ListMergeOperator()); }); var store = testFixture.GetStore(); @@ -298,7 +291,7 @@ public void should_remove_only_first_occurrence_of_duplicate_items() store.RemoveTags(key, "tag"); // Assert - Should have 2 remaining - var tags = store.GetTags(key); + Assert.That(store.TryGet(key, out var tags), Is.True); Assert.That(tags, Is.Not.Null); Assert.That(tags!.Count, Is.EqualTo(2)); Assert.That(tags[0], Is.EqualTo("tag")); From 1143ec4eb4faf575dc901ba9312700cc7ad2f1be Mon Sep 17 00:00:00 2001 From: Havret Date: Thu, 4 Dec 2025 16:32:54 +0100 Subject: [PATCH 06/19] wip --- src/RocksDb.Extensions/IMergeOperator.cs | 6 ++---- .../MergeOperators/Int64AddMergeOperator.cs | 4 ++-- .../MergeOperators/ListAppendMergeOperator.cs | 4 ++-- src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs | 2 -- src/RocksDb.Extensions/RocksDbBuilder.cs | 4 ++-- 5 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/RocksDb.Extensions/IMergeOperator.cs b/src/RocksDb.Extensions/IMergeOperator.cs index 28206af..172694b 100644 --- a/src/RocksDb.Extensions/IMergeOperator.cs +++ b/src/RocksDb.Extensions/IMergeOperator.cs @@ -27,19 +27,17 @@ public interface IMergeOperator /// Performs a full merge of the existing value with one or more operands. /// Called when a Get operation encounters merge operands and needs to produce the final value. /// - /// The key being merged. /// The existing value in the database. For value types, this will be default if no value exists. /// The list of merge operands to apply, in order. /// The merged value to store. - TValue FullMerge(ReadOnlySpan key, TValue existingValue, IReadOnlyList operands); + TValue FullMerge(TValue existingValue, IReadOnlyList operands); /// /// Performs a partial merge of multiple operands without the existing value. /// Called during compaction to combine multiple merge operands into a single operand. /// This is an optimization that reduces the number of operands that need to be stored. /// - /// The key being merged. /// The list of merge operands to combine, in order. /// The combined operand. - TOperand PartialMerge(ReadOnlySpan key, IReadOnlyList operands); + TOperand PartialMerge(IReadOnlyList operands); } diff --git a/src/RocksDb.Extensions/MergeOperators/Int64AddMergeOperator.cs b/src/RocksDb.Extensions/MergeOperators/Int64AddMergeOperator.cs index 0954bc9..26805f0 100644 --- a/src/RocksDb.Extensions/MergeOperators/Int64AddMergeOperator.cs +++ b/src/RocksDb.Extensions/MergeOperators/Int64AddMergeOperator.cs @@ -24,7 +24,7 @@ public class Int64AddMergeOperator : IMergeOperator public string Name => "Int64AddMergeOperator"; /// - public long FullMerge(ReadOnlySpan key, long existingValue, IReadOnlyList operands) + public long FullMerge(long existingValue, IReadOnlyList operands) { var result = existingValue; foreach (var operand in operands) @@ -35,7 +35,7 @@ public long FullMerge(ReadOnlySpan key, long existingValue, IReadOnlyList< } /// - public long PartialMerge(ReadOnlySpan key, IReadOnlyList operands) + public long PartialMerge(IReadOnlyList operands) { long result = 0; foreach (var operand in operands) diff --git a/src/RocksDb.Extensions/MergeOperators/ListAppendMergeOperator.cs b/src/RocksDb.Extensions/MergeOperators/ListAppendMergeOperator.cs index 4a2fce7..6342661 100644 --- a/src/RocksDb.Extensions/MergeOperators/ListAppendMergeOperator.cs +++ b/src/RocksDb.Extensions/MergeOperators/ListAppendMergeOperator.cs @@ -28,7 +28,7 @@ public class ListAppendMergeOperator : IMergeOperator, IList> public string Name => $"ListAppendMergeOperator<{typeof(T).Name}>"; /// - public IList FullMerge(ReadOnlySpan key, IList existingValue, IReadOnlyList> operands) + public IList FullMerge(IList existingValue, IReadOnlyList> operands) { var result = existingValue != null ? new List(existingValue) : new List(); @@ -44,7 +44,7 @@ public IList FullMerge(ReadOnlySpan key, IList existingValue, IReadO } /// - public IList PartialMerge(ReadOnlySpan key, IReadOnlyList> operands) + public IList PartialMerge(IReadOnlyList> operands) { var result = new List(); diff --git a/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs b/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs index 8ee62ec..665feb0 100644 --- a/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs +++ b/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs @@ -42,7 +42,6 @@ public class ListMergeOperator : IMergeOperator, ListOperation> /// public IList FullMerge( - ReadOnlySpan key, IList existingValue, IReadOnlyList> operands) { @@ -60,7 +59,6 @@ public IList FullMerge( /// public ListOperation PartialMerge( - ReadOnlySpan key, IReadOnlyList> operands) { // Combine all operations into a single compound operation diff --git a/src/RocksDb.Extensions/RocksDbBuilder.cs b/src/RocksDb.Extensions/RocksDbBuilder.cs index 1bf9370..e6bf69f 100644 --- a/src/RocksDb.Extensions/RocksDbBuilder.cs +++ b/src/RocksDb.Extensions/RocksDbBuilder.cs @@ -143,7 +143,7 @@ private static byte[] FullMergeCallback( } // Call the user's merge operator - returns TValue - var result = mergeOperator.FullMerge(key, existing, operandList); + var result = mergeOperator.FullMerge(existing, operandList); // Serialize the result as TValue return SerializeValue(result, valueSerializer); @@ -164,7 +164,7 @@ private static byte[] PartialMergeCallback( } // Call the user's partial merge operator - returns TOperand - var result = mergeOperator.PartialMerge(key, operandList); + var result = mergeOperator.PartialMerge(operandList); success = true; // Serialize the result as TOperand From da936371c9e2f445a8dfd4f99805e49671c20dd7 Mon Sep 17 00:00:00 2001 From: Havret Date: Thu, 4 Dec 2025 16:50:49 +0100 Subject: [PATCH 07/19] wip --- .../IMergeableRocksDbStore.cs | 20 --------------- src/RocksDb.Extensions/IRocksDbStore.cs | 19 -------------- .../MergeableRocksDbStore.cs | 2 +- src/RocksDb.Extensions/RocksDbBuilder.cs | 25 ++++++------------- 4 files changed, 9 insertions(+), 57 deletions(-) delete mode 100644 src/RocksDb.Extensions/IMergeableRocksDbStore.cs diff --git a/src/RocksDb.Extensions/IMergeableRocksDbStore.cs b/src/RocksDb.Extensions/IMergeableRocksDbStore.cs deleted file mode 100644 index de84e34..0000000 --- a/src/RocksDb.Extensions/IMergeableRocksDbStore.cs +++ /dev/null @@ -1,20 +0,0 @@ -namespace RocksDb.Extensions; - -/// -/// Interface for a RocksDB store that supports merge operations. -/// -/// The type of the store's keys. -/// The type of the store's values. -/// The type of the merge operand. -public interface IMergeableRocksDbStore -{ - /// - /// Performs an atomic merge operation on the value associated with the specified key. - /// This operation uses RocksDB's merge operator to combine the operand with the existing value - /// without requiring a separate read operation, which is more efficient than Get+Put for - /// accumulative operations like counters, list appends, or set unions. - /// - /// The key to merge the operand with. - /// The operand to merge with the existing value. - void Merge(TKey key, TOperand operand); -} diff --git a/src/RocksDb.Extensions/IRocksDbStore.cs b/src/RocksDb.Extensions/IRocksDbStore.cs index ea7af2d..c39f6b9 100644 --- a/src/RocksDb.Extensions/IRocksDbStore.cs +++ b/src/RocksDb.Extensions/IRocksDbStore.cs @@ -107,23 +107,4 @@ public abstract class RocksDbStore /// /// An enumerable collection of all the keys in the store. public IEnumerable GetAllKeys() => _rocksDbAccessor.GetAllKeys(); - - /// - /// Performs an atomic merge operation on the value associated with the specified key. - /// This operation uses RocksDB's merge operator to combine the operand with the existing value - /// without requiring a separate read operation, which is more efficient than Get+Put for - /// accumulative operations like counters, list appends, or set unions. - /// - /// The key to merge the operand with. - /// The operand to merge with the existing value. - /// - /// This method requires a merge operator to be configured for the store's column family. - /// If no merge operator is configured, RocksDB will throw an error when reading the value. - /// - /// Common use cases include: - /// - Counters: Merge operand represents the increment value - /// - Lists: Merge operand represents items to append - /// - Sets: Merge operand represents items to add to the set - /// - protected void Merge(TKey key, TValue operand) => _rocksDbAccessor.Merge(key, operand); } diff --git a/src/RocksDb.Extensions/MergeableRocksDbStore.cs b/src/RocksDb.Extensions/MergeableRocksDbStore.cs index 52cfe85..22f08a6 100644 --- a/src/RocksDb.Extensions/MergeableRocksDbStore.cs +++ b/src/RocksDb.Extensions/MergeableRocksDbStore.cs @@ -42,7 +42,7 @@ namespace RocksDb.Extensions; /// } /// /// -public abstract class MergeableRocksDbStore : RocksDbStore, IMergeableRocksDbStore +public abstract class MergeableRocksDbStore : RocksDbStore { private readonly IMergeAccessor _mergeAccessor; diff --git a/src/RocksDb.Extensions/RocksDbBuilder.cs b/src/RocksDb.Extensions/RocksDbBuilder.cs index e6bf69f..d0f9d9d 100644 --- a/src/RocksDb.Extensions/RocksDbBuilder.cs +++ b/src/RocksDb.Extensions/RocksDbBuilder.cs @@ -109,20 +109,18 @@ private static MergeOperatorConfig CreateMergeOperatorConfig( { Name = mergeOperator.Name, ValueSerializer = valueSerializer, - FullMerge = (ReadOnlySpan key, bool hasExistingValue, ReadOnlySpan existingValue, global::RocksDbSharp.MergeOperators.OperandsEnumerator operands, out bool success) => + FullMerge = (ReadOnlySpan _, bool hasExistingValue, ReadOnlySpan existingValue, global::RocksDbSharp.MergeOperators.OperandsEnumerator operands, out bool success) => { - return FullMergeCallback(key, hasExistingValue, existingValue, operands, mergeOperator, valueSerializer, operandSerializer, out success); + return FullMergeCallback(hasExistingValue, existingValue, operands, mergeOperator, valueSerializer, operandSerializer, out success); }, - PartialMerge = (ReadOnlySpan key, global::RocksDbSharp.MergeOperators.OperandsEnumerator operands, out bool success) => + PartialMerge = (ReadOnlySpan _, global::RocksDbSharp.MergeOperators.OperandsEnumerator operands, out bool success) => { - return PartialMergeCallback(key, operands, mergeOperator, operandSerializer, out success); + return PartialMergeCallback(operands, mergeOperator, operandSerializer, out success); } }; } - private static byte[] FullMergeCallback( - ReadOnlySpan key, - bool hasExistingValue, + private static byte[] FullMergeCallback(bool hasExistingValue, ReadOnlySpan existingValue, global::RocksDbSharp.MergeOperators.OperandsEnumerator operands, IMergeOperator mergeOperator, @@ -132,10 +130,8 @@ private static byte[] FullMergeCallback( { success = true; - // Deserialize existing value if present - TValue existing = hasExistingValue ? valueSerializer.Deserialize(existingValue) : default!; + var existing = hasExistingValue ? valueSerializer.Deserialize(existingValue) : default!; - // Deserialize all operands var operandList = new List(operands.Count); for (int i = 0; i < operands.Count; i++) { @@ -145,29 +141,24 @@ private static byte[] FullMergeCallback( // Call the user's merge operator - returns TValue var result = mergeOperator.FullMerge(existing, operandList); - // Serialize the result as TValue return SerializeValue(result, valueSerializer); } - private static byte[] PartialMergeCallback( - ReadOnlySpan key, - global::RocksDbSharp.MergeOperators.OperandsEnumerator operands, + private static byte[] PartialMergeCallback(global::RocksDbSharp.MergeOperators.OperandsEnumerator operands, IMergeOperator mergeOperator, ISerializer operandSerializer, out bool success) { - // Deserialize all operands var operandList = new List(operands.Count); for (int i = 0; i < operands.Count; i++) { operandList.Add(operandSerializer.Deserialize(operands.Get(i))); } - // Call the user's partial merge operator - returns TOperand var result = mergeOperator.PartialMerge(operandList); success = true; - // Serialize the result as TOperand + return SerializeValue(result, operandSerializer); } From 95dbd3ae39b6ea8a87dcb30c87f93d48e32d0429 Mon Sep 17 00:00:00 2001 From: Havret Date: Thu, 4 Dec 2025 17:25:21 +0100 Subject: [PATCH 08/19] wip --- src/RocksDb.Extensions/IMergeOperator.cs | 10 ++- .../ListOperationSerializer.cs | 7 +- .../MergeOperators/Int64AddMergeOperator.cs | 47 ------------ .../MergeOperators/ListAppendMergeOperator.cs | 2 +- .../MergeOperators/ListMergeOperator.cs | 49 ++++++------- .../MergeOperators/ListOperation.cs | 16 ----- .../MergeOperators/OperationType.cs | 17 +++++ src/RocksDb.Extensions/RocksDbBuilder.cs | 7 +- .../MergeOperatorTests.cs | 72 +------------------ 9 files changed, 53 insertions(+), 174 deletions(-) delete mode 100644 src/RocksDb.Extensions/MergeOperators/Int64AddMergeOperator.cs create mode 100644 src/RocksDb.Extensions/MergeOperators/OperationType.cs diff --git a/src/RocksDb.Extensions/IMergeOperator.cs b/src/RocksDb.Extensions/IMergeOperator.cs index 172694b..6ba14b4 100644 --- a/src/RocksDb.Extensions/IMergeOperator.cs +++ b/src/RocksDb.Extensions/IMergeOperator.cs @@ -30,7 +30,7 @@ public interface IMergeOperator /// The existing value in the database. For value types, this will be default if no value exists. /// The list of merge operands to apply, in order. /// The merged value to store. - TValue FullMerge(TValue existingValue, IReadOnlyList operands); + TValue FullMerge(TValue? existingValue, IReadOnlyList operands); /// /// Performs a partial merge of multiple operands without the existing value. @@ -38,6 +38,10 @@ public interface IMergeOperator /// This is an optimization that reduces the number of operands that need to be stored. /// /// The list of merge operands to combine, in order. - /// The combined operand. - TOperand PartialMerge(IReadOnlyList operands); + /// The combined operand, or null if partial merge is not safe for these operands. + /// + /// Return null when it's not safe to combine operands without knowing the existing value. + /// When null is returned, RocksDB will keep the operands separate and call FullMerge later. + /// + TOperand? PartialMerge(IReadOnlyList operands); } diff --git a/src/RocksDb.Extensions/ListOperationSerializer.cs b/src/RocksDb.Extensions/ListOperationSerializer.cs index fff759b..2fbf1d4 100644 --- a/src/RocksDb.Extensions/ListOperationSerializer.cs +++ b/src/RocksDb.Extensions/ListOperationSerializer.cs @@ -73,12 +73,7 @@ public void WriteTo(ref ListOperation value, ref Span span) public void WriteTo(ref ListOperation value, IBufferWriter buffer) { - if (TryCalculateSize(ref value, out var size)) - { - var span = buffer.GetSpan(size); - WriteTo(ref value, ref span); - buffer.Advance(size); - } + throw new NotImplementedException(); } public ListOperation Deserialize(ReadOnlySpan buffer) diff --git a/src/RocksDb.Extensions/MergeOperators/Int64AddMergeOperator.cs b/src/RocksDb.Extensions/MergeOperators/Int64AddMergeOperator.cs deleted file mode 100644 index 26805f0..0000000 --- a/src/RocksDb.Extensions/MergeOperators/Int64AddMergeOperator.cs +++ /dev/null @@ -1,47 +0,0 @@ -namespace RocksDb.Extensions.MergeOperators; - -/// -/// A merge operator that adds Int64 values together. -/// Useful for implementing atomic counters. -/// -/// -/// -/// public class CounterStore : MergeableRocksDbStore<string, long, long> -/// { -/// public CounterStore(IRocksDbAccessor<string, long> accessor, IMergeAccessor<string, long> mergeAccessor) -/// : base(accessor, mergeAccessor) { } -/// -/// public void Increment(string key, long delta = 1) => Merge(key, delta); -/// } -/// -/// // Registration: -/// builder.AddMergeableStore<string, long, long, CounterStore>("counters", new Int64AddMergeOperator()); -/// -/// -public class Int64AddMergeOperator : IMergeOperator -{ - /// - public string Name => "Int64AddMergeOperator"; - - /// - public long FullMerge(long existingValue, IReadOnlyList operands) - { - var result = existingValue; - foreach (var operand in operands) - { - result += operand; - } - return result; - } - - /// - public long PartialMerge(IReadOnlyList operands) - { - long result = 0; - foreach (var operand in operands) - { - result += operand; - } - return result; - } -} diff --git a/src/RocksDb.Extensions/MergeOperators/ListAppendMergeOperator.cs b/src/RocksDb.Extensions/MergeOperators/ListAppendMergeOperator.cs index 6342661..66bd839 100644 --- a/src/RocksDb.Extensions/MergeOperators/ListAppendMergeOperator.cs +++ b/src/RocksDb.Extensions/MergeOperators/ListAppendMergeOperator.cs @@ -28,7 +28,7 @@ public class ListAppendMergeOperator : IMergeOperator, IList> public string Name => $"ListAppendMergeOperator<{typeof(T).Name}>"; /// - public IList FullMerge(IList existingValue, IReadOnlyList> operands) + public IList FullMerge(IList? existingValue, IReadOnlyList> operands) { var result = existingValue != null ? new List(existingValue) : new List(); diff --git a/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs b/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs index 665feb0..7dbad5c 100644 --- a/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs +++ b/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs @@ -42,7 +42,7 @@ public class ListMergeOperator : IMergeOperator, ListOperation> /// public IList FullMerge( - IList existingValue, + IList? existingValue, IReadOnlyList> operands) { // Start with existing items or empty list @@ -58,46 +58,37 @@ public IList FullMerge( } /// - public ListOperation PartialMerge( - IReadOnlyList> operands) + public ListOperation? PartialMerge(IReadOnlyList> operands) { - // Combine all operations into a single compound operation - // We preserve all items in order - adds followed by removes - // This is a simplification; a more sophisticated implementation could optimize + // Check if any operands contain removes + bool hasRemoves = false; var allAdds = new List(); - var allRemoves = new List(); foreach (var operand in operands) { - switch (operand.Type) + if (operand.Type == OperationType.Remove) + { + hasRemoves = true; + break; + } + + if (operand.Type == OperationType.Add) { - case OperationType.Add: - foreach (var item in operand.Items) - { - allAdds.Add(item); - } - break; - case OperationType.Remove: - foreach (var item in operand.Items) - { - allRemoves.Add(item); - } - break; + foreach (var item in operand.Items) + { + allAdds.Add(item); + } } } - // For partial merge, we can only safely combine if there are no removes - // (since removes depend on the order of operations and existing state) - // For now, just return the first operand's type with combined items - // A better approach would be to return a compound operation, but that complicates the type - if (allRemoves.Count == 0) + // If there are any removes, we can't safely combine without knowing the existing state + // Return null to signal that RocksDB should keep operands separate + if (hasRemoves) { - return ListOperation.Add(allAdds); + return null; } - // If we have removes, we can't safely combine without knowing the state - // Return the combined adds (removes will be re-applied during full merge) - // This is a simplification - in practice, operands are kept separate if partial merge fails + // Only adds present - safe to combine return ListOperation.Add(allAdds); } diff --git a/src/RocksDb.Extensions/MergeOperators/ListOperation.cs b/src/RocksDb.Extensions/MergeOperators/ListOperation.cs index 89db008..711649c 100644 --- a/src/RocksDb.Extensions/MergeOperators/ListOperation.cs +++ b/src/RocksDb.Extensions/MergeOperators/ListOperation.cs @@ -1,21 +1,5 @@ namespace RocksDb.Extensions.MergeOperators; -/// -/// Specifies the type of operation to perform on a list. -/// -public enum OperationType -{ - /// - /// Add items to the list. - /// - Add, - - /// - /// Remove items from the list (first occurrence of each item). - /// - Remove -} - /// /// Represents an operation (add or remove) to apply to a list via merge. /// diff --git a/src/RocksDb.Extensions/MergeOperators/OperationType.cs b/src/RocksDb.Extensions/MergeOperators/OperationType.cs new file mode 100644 index 0000000..a500bef --- /dev/null +++ b/src/RocksDb.Extensions/MergeOperators/OperationType.cs @@ -0,0 +1,17 @@ +namespace RocksDb.Extensions.MergeOperators; + +/// +/// Specifies the type of operation to perform on a list. +/// +public enum OperationType +{ + /// + /// Add items to the list. + /// + Add, + + /// + /// Remove items from the list (first occurrence of each item). + /// + Remove +} \ No newline at end of file diff --git a/src/RocksDb.Extensions/RocksDbBuilder.cs b/src/RocksDb.Extensions/RocksDbBuilder.cs index d0f9d9d..86106a4 100644 --- a/src/RocksDb.Extensions/RocksDbBuilder.cs +++ b/src/RocksDb.Extensions/RocksDbBuilder.cs @@ -157,8 +157,13 @@ private static byte[] PartialMergeCallback(global::RocksDbShar var result = mergeOperator.PartialMerge(operandList); + if (result == null) + { + success = false; + return Array.Empty(); + } + success = true; - return SerializeValue(result, operandSerializer); } diff --git a/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs b/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs index 5f66710..6d29c4c 100644 --- a/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs +++ b/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs @@ -53,49 +53,7 @@ public void RemoveTags(string key, params string[] tags) public class MergeOperatorTests { - [Test] - public void should_increment_counter_using_merge_operation() - { - // Arrange - using var testFixture = TestFixture.Create(rockDb => - { - rockDb.AddMergeableStore("counters", new Int64AddMergeOperator()); - }); - - var store = testFixture.GetStore(); - var key = "page-views"; - - // Act - store.Increment(key, 1); - store.Increment(key, 5); - store.Increment(key, 10); - - // Assert - Assert.That(store.TryGet(key, out var value), Is.True); - Assert.That(value, Is.EqualTo(16)); - } - - [Test] - public void should_handle_counter_with_initial_value() - { - // Arrange - using var testFixture = TestFixture.Create(rockDb => - { - rockDb.AddMergeableStore("counters", new Int64AddMergeOperator()); - }); - - var store = testFixture.GetStore(); - var key = "page-views"; - - // Act - Put initial value, then merge - store.Put(key, 100); - store.Increment(key, 50); - - // Assert - Assert.That(store.TryGet(key, out var value), Is.True); - Assert.That(value, Is.EqualTo(150)); - } - + [Test] public void should_append_to_list_using_merge_operation() { @@ -146,34 +104,6 @@ public void should_append_to_existing_list_using_merge_operation() Assert.That(events[1], Is.EqualTo("new-event")); } - [Test] - public void should_handle_multiple_keys_with_merge_operations() - { - // Arrange - using var testFixture = TestFixture.Create(rockDb => - { - rockDb.AddMergeableStore("counters", new Int64AddMergeOperator()); - }); - - var store = testFixture.GetStore(); - - // Act - store.Increment("key1", 10); - store.Increment("key2", 20); - store.Increment("key1", 5); - store.Increment("key2", 10); - - // Assert - Assert.Multiple(() => - { - Assert.That(store.TryGet("key1", out var value1), Is.True); - Assert.That(value1, Is.EqualTo(15)); - - Assert.That(store.TryGet("key2", out var value2), Is.True); - Assert.That(value2, Is.EqualTo(30)); - }); - } - [Test] public void should_add_items_to_list_using_list_merge_operator() { From 3b1398f9a90aabaab1b5a41aff4296a758571474 Mon Sep 17 00:00:00 2001 From: Havret Date: Fri, 5 Dec 2025 00:08:25 +0100 Subject: [PATCH 09/19] wip --- src/RocksDb.Extensions/IMergeAccessor.cs | 2 +- src/RocksDb.Extensions/IRocksDbAccessor.cs | 1 - src/RocksDb.Extensions/IRocksDbBuilder.cs | 2 +- src/RocksDb.Extensions/MergeAccessor.cs | 20 ++-- .../MergeableRocksDbStore.cs | 98 +++++++++++++++++-- src/RocksDb.Extensions/RocksDbAccessor.cs | 20 ++-- src/RocksDb.Extensions/RocksDbBuilder.cs | 14 +-- src/RocksDb.Extensions/RocksDbContext.cs | 3 - .../{IRocksDbStore.cs => RocksDbStore.cs} | 0 .../MergeOperatorTests.cs | 40 +++----- 10 files changed, 127 insertions(+), 73 deletions(-) rename src/RocksDb.Extensions/{IRocksDbStore.cs => RocksDbStore.cs} (100%) diff --git a/src/RocksDb.Extensions/IMergeAccessor.cs b/src/RocksDb.Extensions/IMergeAccessor.cs index 3abf085..9c5605e 100644 --- a/src/RocksDb.Extensions/IMergeAccessor.cs +++ b/src/RocksDb.Extensions/IMergeAccessor.cs @@ -9,7 +9,7 @@ namespace RocksDb.Extensions; /// It provides merge operation support with a separate operand type. /// [EditorBrowsable(EditorBrowsableState.Never)] -public interface IMergeAccessor +public interface IMergeAccessor : IRocksDbAccessor { void Merge(TKey key, TOperand operand); } diff --git a/src/RocksDb.Extensions/IRocksDbAccessor.cs b/src/RocksDb.Extensions/IRocksDbAccessor.cs index 90d3710..e3c0b3f 100644 --- a/src/RocksDb.Extensions/IRocksDbAccessor.cs +++ b/src/RocksDb.Extensions/IRocksDbAccessor.cs @@ -23,7 +23,6 @@ public interface IRocksDbAccessor bool HasKey(TKey key); void Clear(); int Count(); - void Merge(TKey key, TValue operand); } #pragma warning restore CS1591 \ No newline at end of file diff --git a/src/RocksDb.Extensions/IRocksDbBuilder.cs b/src/RocksDb.Extensions/IRocksDbBuilder.cs index a4a3f12..cb0b67c 100644 --- a/src/RocksDb.Extensions/IRocksDbBuilder.cs +++ b/src/RocksDb.Extensions/IRocksDbBuilder.cs @@ -53,6 +53,6 @@ public interface IRocksDbBuilder /// For stores that don't need merge operations, use instead. /// /// - IRocksDbBuilder AddMergeableStore(string columnFamily, IMergeOperator mergeOperator) + IRocksDbBuilder AddMergeableStore(string columnFamily, IMergeOperator mergeOperator) where TStore : MergeableRocksDbStore; } \ No newline at end of file diff --git a/src/RocksDb.Extensions/MergeAccessor.cs b/src/RocksDb.Extensions/MergeAccessor.cs index 155f96d..4c94250 100644 --- a/src/RocksDb.Extensions/MergeAccessor.cs +++ b/src/RocksDb.Extensions/MergeAccessor.cs @@ -1,27 +1,19 @@ using System.Buffers; using CommunityToolkit.HighPerformance.Buffers; -using RocksDbSharp; namespace RocksDb.Extensions; -internal class MergeAccessor : IMergeAccessor +internal class MergeAccessor : RocksDbAccessor, IMergeAccessor { - private const int MaxStackSize = 256; - - private readonly ISerializer _keySerializer; private readonly ISerializer _operandSerializer; - private readonly RocksDbSharp.RocksDb _db; - private readonly ColumnFamilyHandle _columnFamilyHandle; public MergeAccessor( - RocksDbSharp.RocksDb db, - ColumnFamilyHandle columnFamilyHandle, + RocksDbContext db, + ColumnFamily columnFamily, ISerializer keySerializer, - ISerializer operandSerializer) + ISerializer valueSerializer, + ISerializer operandSerializer) : base(db, columnFamily, keySerializer, valueSerializer) { - _db = db; - _columnFamilyHandle = columnFamilyHandle; - _keySerializer = keySerializer; _operandSerializer = operandSerializer; } @@ -76,7 +68,7 @@ public void Merge(TKey key, TOperand operand) operandSpan = operandBufferWriter.WrittenSpan; } - _db.Merge(keySpan, operandSpan, _columnFamilyHandle); + _rocksDbContext.Db.Merge(keySpan, operandSpan, _columnFamily.Handle); } finally { diff --git a/src/RocksDb.Extensions/MergeableRocksDbStore.cs b/src/RocksDb.Extensions/MergeableRocksDbStore.cs index 22f08a6..c36ec3d 100644 --- a/src/RocksDb.Extensions/MergeableRocksDbStore.cs +++ b/src/RocksDb.Extensions/MergeableRocksDbStore.cs @@ -1,3 +1,5 @@ +using System.Diagnostics.CodeAnalysis; + namespace RocksDb.Extensions; /// @@ -42,19 +44,17 @@ namespace RocksDb.Extensions; /// } /// /// -public abstract class MergeableRocksDbStore : RocksDbStore +public abstract class MergeableRocksDbStore { - private readonly IMergeAccessor _mergeAccessor; + private readonly IMergeAccessor _rocksDbAccessor; /// /// Initializes a new instance of the class. /// /// The RocksDB accessor to use for database operations. - /// The merge accessor to use for merge operations. - protected MergeableRocksDbStore(IRocksDbAccessor rocksDbAccessor, IMergeAccessor mergeAccessor) - : base(rocksDbAccessor) + protected MergeableRocksDbStore(IMergeAccessor rocksDbAccessor) { - _mergeAccessor = mergeAccessor; + _rocksDbAccessor = rocksDbAccessor; } /// @@ -64,5 +64,89 @@ protected MergeableRocksDbStore(IRocksDbAccessor rocksDbAccessor, /// /// The key to merge the operand with. /// The operand to merge with the existing value. - public void Merge(TKey key, TOperand operand) => _mergeAccessor.Merge(key, operand); + public void Merge(TKey key, TOperand operand) => _rocksDbAccessor.Merge(key, operand); + + /// + /// Removes the specified key and its associated value from the store. + /// + /// The key to remove. + public void Remove(TKey key) => _rocksDbAccessor.Remove(key); + + /// + /// Adds or updates the specified key-value pair in the store. + /// + /// The key to add or update. + /// The value to add or update. + public void Put(TKey key, TValue value) => _rocksDbAccessor.Put(key, value); + + /// + /// Tries to get the value associated with the specified key in the store. + /// + /// The key of the value to get. + /// The value associated with the specified key, if the key is found; otherwise, the default value for the type of the value parameter. + /// true if the key is found; otherwise, false. + public bool TryGet(TKey key, [MaybeNullWhen(false)] out TValue value) => _rocksDbAccessor.TryGet(key, out value); + + /// + /// Puts the specified keys and values in the store. + /// + /// The keys to put in the store. + /// The values to put in the store. + public void PutRange(ReadOnlySpan keys, ReadOnlySpan values) => _rocksDbAccessor.PutRange(keys, values); + + /// + /// Puts the specified values in the store using the specified key selector function to generate keys. + /// + /// The values to put in the store. + /// The function to use to generate keys for the values. + public void PutRange(ReadOnlySpan values, Func keySelector) => _rocksDbAccessor.PutRange(values, keySelector); + + /// + /// Adds or updates a collection of key-value pairs in the store. + /// + /// The collection of key-value pairs to add or update. + public void PutRange(IReadOnlyList<(TKey key, TValue value)> items) => _rocksDbAccessor.PutRange(items); + + /// + /// Gets all the values in the store. + /// + /// An enumerable collection of all the values in the store. + public IEnumerable GetAllValues() => _rocksDbAccessor.GetAllValues(); + + /// + /// Determines whether the store contains a value for a specific key. + /// + /// The key to check in the store for an associated value. + /// true if the store contains an element with the specified key; otherwise, false. + public bool HasKey(TKey key) => _rocksDbAccessor.HasKey(key); + + /// + /// Resets the column family associated with the store. + /// This operation destroys the current column family and creates a new one, + /// effectively removing all stored key-value pairs. + /// + /// Note: This method is intended for scenarios where a complete reset of the column family + /// is required. The operation may involve internal reallocation and metadata changes, which + /// can impact performance during execution. Use with caution in high-frequency workflows. + /// + public void Clear() => _rocksDbAccessor.Clear(); + + /// + /// Gets the number of key-value pairs currently stored. + /// + /// + /// This method is not a constant-time operation. Internally, it iterates over all entries in the store + /// to compute the count. While the keys and values are not deserialized during iteration, this process may still + /// be expensive for large datasets. + /// + /// Use this method with caution in performance-critical paths, especially if the store contains a high number of entries. + /// + /// The total count of items in the store. + public int Count() => _rocksDbAccessor.Count(); + + /// + /// Gets all the keys in the store. + /// + /// An enumerable collection of all the keys in the store. + public IEnumerable GetAllKeys() => _rocksDbAccessor.GetAllKeys(); } diff --git a/src/RocksDb.Extensions/RocksDbAccessor.cs b/src/RocksDb.Extensions/RocksDbAccessor.cs index 5915a55..393224e 100644 --- a/src/RocksDb.Extensions/RocksDbAccessor.cs +++ b/src/RocksDb.Extensions/RocksDbAccessor.cs @@ -7,12 +7,12 @@ namespace RocksDb.Extensions; internal class RocksDbAccessor : IRocksDbAccessor, ISpanDeserializer { - private const int MaxStackSize = 256; + private protected const int MaxStackSize = 256; - private readonly ISerializer _keySerializer; - private readonly ISerializer _valueSerializer; - private readonly RocksDbContext _rocksDbContext; - private readonly ColumnFamily _columnFamily; + protected readonly ISerializer _keySerializer; + protected private readonly ISerializer _valueSerializer; + protected private readonly RocksDbContext _rocksDbContext; + private protected readonly ColumnFamily _columnFamily; private readonly bool _checkIfExists; public RocksDbAccessor(RocksDbContext rocksDbContext, @@ -280,7 +280,7 @@ private void AddToBatch(TKey key, TValue value, WriteBatch batch) _valueSerializer.WriteTo(ref value, valueBufferWriter); valueSpan = valueBufferWriter.WrittenSpan; } - + _ = batch.Put(keySpan, valueSpan, _columnFamily.Handle); } finally @@ -298,7 +298,7 @@ private void AddToBatch(TKey key, TValue value, WriteBatch batch) } } } - + public IEnumerable GetAllKeys() { using var iterator = _rocksDbContext.Db.NewIterator(_columnFamily.Handle); @@ -320,7 +320,7 @@ public IEnumerable GetAllValues() _ = iterator.Next(); } } - + public int Count() { using var iterator = _rocksDbContext.Db.NewIterator(_columnFamily.Handle); @@ -374,13 +374,13 @@ public bool HasKey(TKey key) } } } - + public void Clear() { var prevColumnFamilyHandle = _columnFamily.Handle; _rocksDbContext.Db.DropColumnFamily(_columnFamily.Name); _columnFamily.Handle = _rocksDbContext.Db.CreateColumnFamily(_rocksDbContext.ColumnFamilyOptions, _columnFamily.Name); - + Native.Instance.rocksdb_column_family_handle_destroy(prevColumnFamilyHandle.Handle); } diff --git a/src/RocksDb.Extensions/RocksDbBuilder.cs b/src/RocksDb.Extensions/RocksDbBuilder.cs index 86106a4..4c2a31d 100644 --- a/src/RocksDb.Extensions/RocksDbBuilder.cs +++ b/src/RocksDb.Extensions/RocksDbBuilder.cs @@ -51,7 +51,7 @@ public IRocksDbBuilder AddStore(string columnFamily) where return this; } - public IRocksDbBuilder AddMergeableStore(string columnFamily, IMergeOperator mergeOperator) + public IRocksDbBuilder AddMergeableStore(string columnFamily, IMergeOperator mergeOperator) where TStore : MergeableRocksDbStore { if (!_columnFamilyLookup.Add(columnFamily)) @@ -78,21 +78,15 @@ public IRocksDbBuilder AddMergeableStore(string var valueSerializer = CreateSerializer(rocksDbOptions.Value.SerializerFactories); var operandSerializer = CreateSerializer(rocksDbOptions.Value.SerializerFactories); - var rocksDbAccessor = new RocksDbAccessor( + var rocksDbAccessor = new MergeAccessor( rocksDbContext, new ColumnFamily(columnFamilyHandle, columnFamily), keySerializer, - valueSerializer - ); - - var mergeAccessor = new MergeAccessor( - rocksDbContext.Db, - columnFamilyHandle, - keySerializer, + valueSerializer, operandSerializer ); - return ActivatorUtilities.CreateInstance(provider, rocksDbAccessor, mergeAccessor); + return ActivatorUtilities.CreateInstance(provider, rocksDbAccessor); }); _serviceCollection.TryAddSingleton(typeof(TStore), provider => provider.GetRequiredKeyedService(columnFamily)); diff --git a/src/RocksDb.Extensions/RocksDbContext.cs b/src/RocksDb.Extensions/RocksDbContext.cs index d1e0085..f4ef5e2 100644 --- a/src/RocksDb.Extensions/RocksDbContext.cs +++ b/src/RocksDb.Extensions/RocksDbContext.cs @@ -8,7 +8,6 @@ internal class RocksDbContext : IDisposable private readonly RocksDbSharp.RocksDb _rocksDb; private readonly Cache _cache; private readonly ColumnFamilyOptions _userSpecifiedOptions; - private readonly List _mergeOperators = new(); private const long BlockCacheSize = 50 * 1024 * 1024L; private const long BlockSize = 4096L; @@ -113,8 +112,6 @@ private ColumnFamilies CreateColumnFamilies( mergeOperatorConfig.PartialMerge, mergeOperatorConfig.FullMerge); - // Keep reference to prevent GC - _mergeOperators.Add(mergeOp); cfOptions.SetMergeOperator(mergeOp); columnFamilies.Add(columnFamilyName, cfOptions); diff --git a/src/RocksDb.Extensions/IRocksDbStore.cs b/src/RocksDb.Extensions/RocksDbStore.cs similarity index 100% rename from src/RocksDb.Extensions/IRocksDbStore.cs rename to src/RocksDb.Extensions/RocksDbStore.cs diff --git a/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs b/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs index 6d29c4c..85b38d4 100644 --- a/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs +++ b/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs @@ -4,26 +4,13 @@ namespace RocksDb.Extensions.Tests; -/// -/// Store that uses merge operations for testing counters. -/// -public class CounterStore : MergeableRocksDbStore -{ - public CounterStore(IRocksDbAccessor rocksDbAccessor, IMergeAccessor mergeAccessor) - : base(rocksDbAccessor, mergeAccessor) - { - } - - public void Increment(string key, long delta = 1) => Merge(key, delta); -} - /// /// Store that uses merge operations for testing list appends. /// public class EventLogStore : MergeableRocksDbStore, IList> { - public EventLogStore(IRocksDbAccessor> rocksDbAccessor, IMergeAccessor> mergeAccessor) - : base(rocksDbAccessor, mergeAccessor) + public EventLogStore(IMergeAccessor, IList> mergeAccessor) + : base(mergeAccessor) { } @@ -35,8 +22,8 @@ public EventLogStore(IRocksDbAccessor> rocksDbAccessor, IM /// public class TagsStore : MergeableRocksDbStore, ListOperation> { - public TagsStore(IRocksDbAccessor> rocksDbAccessor, IMergeAccessor> mergeAccessor) - : base(rocksDbAccessor, mergeAccessor) + public TagsStore(IMergeAccessor, ListOperation> mergeAccessor) + : base(mergeAccessor) { } @@ -53,14 +40,14 @@ public void RemoveTags(string key, params string[] tags) public class MergeOperatorTests { - [Test] public void should_append_to_list_using_merge_operation() { // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddMergeableStore, IList, EventLogStore>("events", new ListAppendMergeOperator()); + rockDb.AddMergeableStore, EventLogStore, IList>("events", + new ListAppendMergeOperator()); }); var store = testFixture.GetStore(); @@ -86,7 +73,8 @@ public void should_append_to_existing_list_using_merge_operation() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddMergeableStore, IList, EventLogStore>("events", new ListAppendMergeOperator()); + rockDb.AddMergeableStore, EventLogStore, IList>("events", + new ListAppendMergeOperator()); }); var store = testFixture.GetStore(); @@ -110,7 +98,7 @@ public void should_add_items_to_list_using_list_merge_operator() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddMergeableStore, ListOperation, TagsStore>("tags", new ListMergeOperator()); + rockDb.AddMergeableStore, TagsStore, ListOperation>("tags", new ListMergeOperator()); }); var store = testFixture.GetStore(); @@ -135,7 +123,7 @@ public void should_remove_items_from_list_using_list_merge_operator() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddMergeableStore, ListOperation, TagsStore>("tags", new ListMergeOperator()); + rockDb.AddMergeableStore, TagsStore, ListOperation>("tags", new ListMergeOperator()); }); var store = testFixture.GetStore(); @@ -161,7 +149,7 @@ public void should_handle_mixed_add_and_remove_operations() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddMergeableStore, ListOperation, TagsStore>("tags", new ListMergeOperator()); + rockDb.AddMergeableStore, TagsStore, ListOperation>("tags", new ListMergeOperator()); }); var store = testFixture.GetStore(); @@ -187,7 +175,7 @@ public void should_handle_remove_nonexistent_item_gracefully() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddMergeableStore, ListOperation, TagsStore>("tags", new ListMergeOperator()); + rockDb.AddMergeableStore, TagsStore, ListOperation>("tags", new ListMergeOperator()); }); var store = testFixture.GetStore(); @@ -210,7 +198,7 @@ public void should_remove_only_first_occurrence_of_duplicate_items() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddMergeableStore, ListOperation, TagsStore>("tags", new ListMergeOperator()); + rockDb.AddMergeableStore, TagsStore, ListOperation>("tags", new ListMergeOperator()); }); var store = testFixture.GetStore(); @@ -227,4 +215,4 @@ public void should_remove_only_first_occurrence_of_duplicate_items() Assert.That(tags[0], Is.EqualTo("tag")); Assert.That(tags[1], Is.EqualTo("tag")); } -} +} \ No newline at end of file From c9854142a2f04c2d09dc6e1d700e7387db982886 Mon Sep 17 00:00:00 2001 From: Havret Date: Thu, 11 Dec 2025 00:09:13 +0100 Subject: [PATCH 10/19] wip --- src/RocksDb.Extensions/IMergeOperator.cs | 8 +-- .../MergeOperators/ListAppendMergeOperator.cs | 61 ---------------- .../MergeOperators/ListMergeOperator.cs | 4 +- .../RocksDb.Extensions.csproj | 2 +- src/RocksDb.Extensions/RocksDbBuilder.cs | 56 ++++++++++----- .../MergeOperatorTests.cs | 69 ++++--------------- 6 files changed, 57 insertions(+), 143 deletions(-) delete mode 100644 src/RocksDb.Extensions/MergeOperators/ListAppendMergeOperator.cs diff --git a/src/RocksDb.Extensions/IMergeOperator.cs b/src/RocksDb.Extensions/IMergeOperator.cs index 6ba14b4..7cb0a32 100644 --- a/src/RocksDb.Extensions/IMergeOperator.cs +++ b/src/RocksDb.Extensions/IMergeOperator.cs @@ -28,20 +28,20 @@ public interface IMergeOperator /// Called when a Get operation encounters merge operands and needs to produce the final value. /// /// The existing value in the database. For value types, this will be default if no value exists. - /// The list of merge operands to apply, in order. + /// The span of merge operands to apply, in order. /// The merged value to store. - TValue FullMerge(TValue? existingValue, IReadOnlyList operands); + TValue FullMerge(TValue? existingValue, ReadOnlySpan operands); /// /// Performs a partial merge of multiple operands without the existing value. /// Called during compaction to combine multiple merge operands into a single operand. /// This is an optimization that reduces the number of operands that need to be stored. /// - /// The list of merge operands to combine, in order. + /// The span of merge operands to combine, in order. /// The combined operand, or null if partial merge is not safe for these operands. /// /// Return null when it's not safe to combine operands without knowing the existing value. /// When null is returned, RocksDB will keep the operands separate and call FullMerge later. /// - TOperand? PartialMerge(IReadOnlyList operands); + TOperand? PartialMerge(ReadOnlySpan operands); } diff --git a/src/RocksDb.Extensions/MergeOperators/ListAppendMergeOperator.cs b/src/RocksDb.Extensions/MergeOperators/ListAppendMergeOperator.cs deleted file mode 100644 index 66bd839..0000000 --- a/src/RocksDb.Extensions/MergeOperators/ListAppendMergeOperator.cs +++ /dev/null @@ -1,61 +0,0 @@ -namespace RocksDb.Extensions.MergeOperators; - -/// -/// A merge operator that appends items to a list. -/// Useful for implementing atomic list append operations without requiring a read before write. -/// -/// The type of elements in the list. -/// -/// -/// public class EventLogStore : MergeableRocksDbStore<string, IList<string>, IList<string>> -/// { -/// public EventLogStore(IRocksDbAccessor<string, IList<string>> accessor, IMergeAccessor<string, IList<string>> mergeAccessor) -/// : base(accessor, mergeAccessor) { } -/// -/// public void AppendEvent(string key, string eventData) -/// { -/// Merge(key, new List<string> { eventData }); -/// } -/// } -/// -/// // Registration: -/// builder.AddMergeableStore<string, IList<string>, IList<string>, EventLogStore>("events", new ListAppendMergeOperator<string>()); -/// -/// -public class ListAppendMergeOperator : IMergeOperator, IList> -{ - /// - public string Name => $"ListAppendMergeOperator<{typeof(T).Name}>"; - - /// - public IList FullMerge(IList? existingValue, IReadOnlyList> operands) - { - var result = existingValue != null ? new List(existingValue) : new List(); - - foreach (var operand in operands) - { - foreach (var item in operand) - { - result.Add(item); - } - } - - return result; - } - - /// - public IList PartialMerge(IReadOnlyList> operands) - { - var result = new List(); - - foreach (var operand in operands) - { - foreach (var item in operand) - { - result.Add(item); - } - } - - return result; - } -} diff --git a/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs b/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs index 7dbad5c..6ac0718 100644 --- a/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs +++ b/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs @@ -43,7 +43,7 @@ public class ListMergeOperator : IMergeOperator, ListOperation> /// public IList FullMerge( IList? existingValue, - IReadOnlyList> operands) + ReadOnlySpan> operands) { // Start with existing items or empty list var result = existingValue != null ? new List(existingValue) : new List(); @@ -58,7 +58,7 @@ public IList FullMerge( } /// - public ListOperation? PartialMerge(IReadOnlyList> operands) + public ListOperation? PartialMerge(ReadOnlySpan> operands) { // Check if any operands contain removes bool hasRemoves = false; diff --git a/src/RocksDb.Extensions/RocksDb.Extensions.csproj b/src/RocksDb.Extensions/RocksDb.Extensions.csproj index f94f8d8..87ef619 100644 --- a/src/RocksDb.Extensions/RocksDb.Extensions.csproj +++ b/src/RocksDb.Extensions/RocksDb.Extensions.csproj @@ -27,6 +27,6 @@ - + diff --git a/src/RocksDb.Extensions/RocksDbBuilder.cs b/src/RocksDb.Extensions/RocksDbBuilder.cs index 4c2a31d..e097872 100644 --- a/src/RocksDb.Extensions/RocksDbBuilder.cs +++ b/src/RocksDb.Extensions/RocksDbBuilder.cs @@ -1,5 +1,6 @@ using System.Buffers; using System.Reflection; +using System.Runtime.CompilerServices; using CommunityToolkit.HighPerformance.Buffers; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; @@ -125,17 +126,24 @@ private static byte[] FullMergeCallback(bool hasExistingValue, success = true; var existing = hasExistingValue ? valueSerializer.Deserialize(existingValue) : default!; + + var operandArray = ArrayPool.Shared.Rent(operands.Count); + try + { + for (int i = 0; i < operands.Count; i++) + { + operandArray[i] = operandSerializer.Deserialize(operands.Get(i)); + } + + var operandSpan = operandArray.AsSpan(0, operands.Count); + var result = mergeOperator.FullMerge(existing, operandSpan); - var operandList = new List(operands.Count); - for (int i = 0; i < operands.Count; i++) + return SerializeValue(result, valueSerializer); + } + finally { - operandList.Add(operandSerializer.Deserialize(operands.Get(i))); + ArrayPool.Shared.Return(operandArray, clearArray: RuntimeHelpers.IsReferenceOrContainsReferences()); } - - // Call the user's merge operator - returns TValue - var result = mergeOperator.FullMerge(existing, operandList); - - return SerializeValue(result, valueSerializer); } private static byte[] PartialMergeCallback(global::RocksDbSharp.MergeOperators.OperandsEnumerator operands, @@ -143,22 +151,32 @@ private static byte[] PartialMergeCallback(global::RocksDbShar ISerializer operandSerializer, out bool success) { - var operandList = new List(operands.Count); - for (int i = 0; i < operands.Count; i++) + // Rent array from pool instead of allocating List + var operandArray = ArrayPool.Shared.Rent(operands.Count); + try { - operandList.Add(operandSerializer.Deserialize(operands.Get(i))); - } + for (int i = 0; i < operands.Count; i++) + { + operandArray[i] = operandSerializer.Deserialize(operands.Get(i)); + } - var result = mergeOperator.PartialMerge(operandList); + // Call the user's merge operator with ReadOnlySpan - zero allocation + var operandSpan = operandArray.AsSpan(0, operands.Count); + var result = mergeOperator.PartialMerge(operandSpan); + + if (result == null) + { + success = false; + return Array.Empty(); + } - if (result == null) + success = true; + return SerializeValue(result, operandSerializer); + } + finally { - success = false; - return Array.Empty(); + ArrayPool.Shared.Return(operandArray, clearArray: RuntimeHelpers.IsReferenceOrContainsReferences()); } - - success = true; - return SerializeValue(result, operandSerializer); } private static byte[] SerializeValue(T value, ISerializer serializer) diff --git a/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs b/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs index 85b38d4..b7d1988 100644 --- a/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs +++ b/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs @@ -4,22 +4,6 @@ namespace RocksDb.Extensions.Tests; -/// -/// Store that uses merge operations for testing list appends. -/// -public class EventLogStore : MergeableRocksDbStore, IList> -{ - public EventLogStore(IMergeAccessor, IList> mergeAccessor) - : base(mergeAccessor) - { - } - - public void AppendEvent(string key, string eventData) => Merge(key, new List { eventData }); -} - -/// -/// Store that uses merge operations for testing list operations (add/remove). -/// public class TagsStore : MergeableRocksDbStore, ListOperation> { public TagsStore(IMergeAccessor, ListOperation> mergeAccessor) @@ -41,55 +25,28 @@ public void RemoveTags(string key, params string[] tags) public class MergeOperatorTests { [Test] - public void should_append_to_list_using_merge_operation() + public void should_add_to_existing_list_using_merge_operation() { // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddMergeableStore, EventLogStore, IList>("events", - new ListAppendMergeOperator()); + rockDb.AddMergeableStore, TagsStore, ListOperation>("tags", new ListMergeOperator()); }); - var store = testFixture.GetStore(); - var key = "user-actions"; + var store = testFixture.GetStore(); + var key = "article-1"; // Act - store.AppendEvent(key, "login"); - store.AppendEvent(key, "view-page"); - store.AppendEvent(key, "logout"); - - // Assert - Assert.That(store.TryGet(key, out var events), Is.True); - Assert.That(events, Is.Not.Null); - Assert.That(events.Count, Is.EqualTo(3)); - Assert.That(events[0], Is.EqualTo("login")); - Assert.That(events[1], Is.EqualTo("view-page")); - Assert.That(events[2], Is.EqualTo("logout")); - } - - [Test] - public void should_append_to_existing_list_using_merge_operation() - { - // Arrange - using var testFixture = TestFixture.Create(rockDb => - { - rockDb.AddMergeableStore, EventLogStore, IList>("events", - new ListAppendMergeOperator()); - }); - - var store = testFixture.GetStore(); - var key = "user-actions"; - - // Act - Put initial value, then merge - store.Put(key, new List { "initial-event" }); - store.AppendEvent(key, "new-event"); + store.Put(key, new List { "csharp", "dotnet" }); + store.AddTags(key, "rocksdb"); // Assert - Assert.That(store.TryGet(key, out var events), Is.True); - Assert.That(events, Is.Not.Null); - Assert.That(events.Count, Is.EqualTo(2)); - Assert.That(events[0], Is.EqualTo("initial-event")); - Assert.That(events[1], Is.EqualTo("new-event")); + Assert.That(store.TryGet(key, out var tags), Is.True); + Assert.That(tags, Is.Not.Null); + Assert.That(tags!.Count, Is.EqualTo(3)); + Assert.That(tags, Does.Contain("csharp")); + Assert.That(tags, Does.Contain("dotnet")); + Assert.That(tags, Does.Contain("rocksdb")); } [Test] @@ -130,7 +87,7 @@ public void should_remove_items_from_list_using_list_merge_operator() var key = "article-1"; // Act - Add items, then remove some - store.AddTags(key, "csharp", "dotnet", "java", "python"); + store.Merge(key, ListOperation.Add("csharp", "dotnet", "java", "python")); store.RemoveTags(key, "java", "python"); // Assert From 63b97fca6a8a255bc37de9ab0b5bf01567dcd1c5 Mon Sep 17 00:00:00 2001 From: Havret Date: Thu, 11 Dec 2025 00:21:33 +0100 Subject: [PATCH 11/19] wip --- src/RocksDb.Extensions/MergeAccessor.cs | 8 +-- src/RocksDb.Extensions/RocksDbAccessor.cs | 84 +++++++++++------------ 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/src/RocksDb.Extensions/MergeAccessor.cs b/src/RocksDb.Extensions/MergeAccessor.cs index 4c94250..da880d0 100644 --- a/src/RocksDb.Extensions/MergeAccessor.cs +++ b/src/RocksDb.Extensions/MergeAccessor.cs @@ -22,7 +22,7 @@ public void Merge(TKey key, TOperand operand) byte[]? rentedKeyBuffer = null; bool useSpanAsKey; // ReSharper disable once AssignmentInConditionalExpression - Span keyBuffer = (useSpanAsKey = _keySerializer.TryCalculateSize(ref key, out var keySize)) + Span keyBuffer = (useSpanAsKey = KeySerializer.TryCalculateSize(ref key, out var keySize)) ? keySize < MaxStackSize ? stackalloc byte[keySize] : (rentedKeyBuffer = ArrayPool.Shared.Rent(keySize)).AsSpan(0, keySize) @@ -48,12 +48,12 @@ public void Merge(TKey key, TOperand operand) { if (useSpanAsKey) { - _keySerializer.WriteTo(ref key, ref keyBuffer); + KeySerializer.WriteTo(ref key, ref keyBuffer); } else { keyBufferWriter = new ArrayPoolBufferWriter(); - _keySerializer.WriteTo(ref key, keyBufferWriter); + KeySerializer.WriteTo(ref key, keyBufferWriter); keySpan = keyBufferWriter.WrittenSpan; } @@ -68,7 +68,7 @@ public void Merge(TKey key, TOperand operand) operandSpan = operandBufferWriter.WrittenSpan; } - _rocksDbContext.Db.Merge(keySpan, operandSpan, _columnFamily.Handle); + RocksDbContext.Db.Merge(keySpan, operandSpan, ColumnFamily.Handle); } finally { diff --git a/src/RocksDb.Extensions/RocksDbAccessor.cs b/src/RocksDb.Extensions/RocksDbAccessor.cs index 393224e..2c2ea35 100644 --- a/src/RocksDb.Extensions/RocksDbAccessor.cs +++ b/src/RocksDb.Extensions/RocksDbAccessor.cs @@ -9,10 +9,10 @@ internal class RocksDbAccessor : IRocksDbAccessor, I { private protected const int MaxStackSize = 256; - protected readonly ISerializer _keySerializer; - protected private readonly ISerializer _valueSerializer; - protected private readonly RocksDbContext _rocksDbContext; - private protected readonly ColumnFamily _columnFamily; + protected readonly ISerializer KeySerializer; + private readonly ISerializer _valueSerializer; + private protected readonly RocksDbContext RocksDbContext; + private protected readonly ColumnFamily ColumnFamily; private readonly bool _checkIfExists; public RocksDbAccessor(RocksDbContext rocksDbContext, @@ -20,9 +20,9 @@ public RocksDbAccessor(RocksDbContext rocksDbContext, ISerializer keySerializer, ISerializer valueSerializer) { - _rocksDbContext = rocksDbContext; - _columnFamily = columnFamily; - _keySerializer = keySerializer; + RocksDbContext = rocksDbContext; + ColumnFamily = columnFamily; + KeySerializer = keySerializer; _valueSerializer = valueSerializer; _checkIfExists = typeof(TValue).IsValueType; @@ -34,7 +34,7 @@ public void Remove(TKey key) bool useSpan; // ReSharper disable once AssignmentInConditionalExpression - Span keyBuffer = (useSpan = _keySerializer.TryCalculateSize(ref key, out var keySize)) + Span keyBuffer = (useSpan = KeySerializer.TryCalculateSize(ref key, out var keySize)) ? keySize < MaxStackSize ? stackalloc byte[keySize] : (rentedKeyBuffer = ArrayPool.Shared.Rent(keySize)).AsSpan(0, keySize) @@ -47,16 +47,16 @@ public void Remove(TKey key) { if (useSpan) { - _keySerializer.WriteTo(ref key, ref keyBuffer); + KeySerializer.WriteTo(ref key, ref keyBuffer); } else { keyBufferWriter = new ArrayPoolBufferWriter(); - _keySerializer.WriteTo(ref key, keyBufferWriter); + KeySerializer.WriteTo(ref key, keyBufferWriter); keySpan = keyBufferWriter.WrittenSpan; } - _rocksDbContext.Db.Remove(keySpan, _columnFamily.Handle); + RocksDbContext.Db.Remove(keySpan, ColumnFamily.Handle); } finally { @@ -73,7 +73,7 @@ public void Put(TKey key, TValue value) byte[]? rentedKeyBuffer = null; bool useSpanAsKey; // ReSharper disable once AssignmentInConditionalExpression - Span keyBuffer = (useSpanAsKey = _keySerializer.TryCalculateSize(ref key, out var keySize)) + Span keyBuffer = (useSpanAsKey = KeySerializer.TryCalculateSize(ref key, out var keySize)) ? keySize < MaxStackSize ? stackalloc byte[keySize] : (rentedKeyBuffer = ArrayPool.Shared.Rent(keySize)).AsSpan(0, keySize) @@ -99,12 +99,12 @@ public void Put(TKey key, TValue value) { if (useSpanAsKey) { - _keySerializer.WriteTo(ref key, ref keyBuffer); + KeySerializer.WriteTo(ref key, ref keyBuffer); } else { keyBufferWriter = new ArrayPoolBufferWriter(); - _keySerializer.WriteTo(ref key, keyBufferWriter); + KeySerializer.WriteTo(ref key, keyBufferWriter); keySpan = keyBufferWriter.WrittenSpan; } @@ -119,7 +119,7 @@ public void Put(TKey key, TValue value) valueSpan = valueBufferWriter.WrittenSpan; } - _rocksDbContext.Db.Put(keySpan, valueSpan, _columnFamily.Handle); + RocksDbContext.Db.Put(keySpan, valueSpan, ColumnFamily.Handle); } finally { @@ -143,7 +143,7 @@ public bool TryGet(TKey key, [MaybeNullWhen(false)] out TValue value) bool useSpan; // ReSharper disable once AssignmentInConditionalExpression - Span keyBuffer = (useSpan = _keySerializer.TryCalculateSize(ref key, out var keySize)) + Span keyBuffer = (useSpan = KeySerializer.TryCalculateSize(ref key, out var keySize)) ? keySize < MaxStackSize ? stackalloc byte[keySize] : (rentedKeyBuffer = ArrayPool.Shared.Rent(keySize)).AsSpan(0, keySize) @@ -156,22 +156,22 @@ public bool TryGet(TKey key, [MaybeNullWhen(false)] out TValue value) { if (useSpan) { - _keySerializer.WriteTo(ref key, ref keyBuffer); + KeySerializer.WriteTo(ref key, ref keyBuffer); } else { keyBufferWriter = new ArrayPoolBufferWriter(); - _keySerializer.WriteTo(ref key, keyBufferWriter); + KeySerializer.WriteTo(ref key, keyBufferWriter); keySpan = keyBufferWriter.WrittenSpan; } - if (_checkIfExists && _rocksDbContext.Db.HasKey(keySpan, _columnFamily.Handle) == false) + if (_checkIfExists && RocksDbContext.Db.HasKey(keySpan, ColumnFamily.Handle) == false) { value = default; return false; } - value = _rocksDbContext.Db.Get(keySpan, this, _columnFamily.Handle); + value = RocksDbContext.Db.Get(keySpan, this, ColumnFamily.Handle); return value != null; } finally @@ -202,7 +202,7 @@ public void PutRange(ReadOnlySpan keys, ReadOnlySpan values) AddToBatch(keys[i], values[i], batch); } - _rocksDbContext.Db.Write(batch); + RocksDbContext.Db.Write(batch); } public void PutRange(ReadOnlySpan values, Func keySelector) @@ -215,7 +215,7 @@ public void PutRange(ReadOnlySpan values, Func keySelector AddToBatch(key, value, batch); } - _rocksDbContext.Db.Write(batch); + RocksDbContext.Db.Write(batch); } public void PutRange(IReadOnlyList<(TKey key, TValue value)> items) @@ -227,7 +227,7 @@ public void PutRange(IReadOnlyList<(TKey key, TValue value)> items) AddToBatch(key, value, batch); } - _rocksDbContext.Db.Write(batch); + RocksDbContext.Db.Write(batch); } private void AddToBatch(TKey key, TValue value, WriteBatch batch) @@ -235,7 +235,7 @@ private void AddToBatch(TKey key, TValue value, WriteBatch batch) byte[]? rentedKeyBuffer = null; bool useSpanAsKey; // ReSharper disable once AssignmentInConditionalExpression - Span keyBuffer = (useSpanAsKey = _keySerializer.TryCalculateSize(ref key, out var keySize)) + Span keyBuffer = (useSpanAsKey = KeySerializer.TryCalculateSize(ref key, out var keySize)) ? keySize < MaxStackSize ? stackalloc byte[keySize] : (rentedKeyBuffer = ArrayPool.Shared.Rent(keySize)).AsSpan(0, keySize) @@ -261,12 +261,12 @@ private void AddToBatch(TKey key, TValue value, WriteBatch batch) { if (useSpanAsKey) { - _keySerializer.WriteTo(ref key, ref keyBuffer); + KeySerializer.WriteTo(ref key, ref keyBuffer); } else { keyBufferWriter = new ArrayPoolBufferWriter(); - _keySerializer.WriteTo(ref key, keyBufferWriter); + KeySerializer.WriteTo(ref key, keyBufferWriter); keySpan = keyBufferWriter.WrittenSpan; } @@ -281,7 +281,7 @@ private void AddToBatch(TKey key, TValue value, WriteBatch batch) valueSpan = valueBufferWriter.WrittenSpan; } - _ = batch.Put(keySpan, valueSpan, _columnFamily.Handle); + _ = batch.Put(keySpan, valueSpan, ColumnFamily.Handle); } finally { @@ -301,18 +301,18 @@ private void AddToBatch(TKey key, TValue value, WriteBatch batch) public IEnumerable GetAllKeys() { - using var iterator = _rocksDbContext.Db.NewIterator(_columnFamily.Handle); + using var iterator = RocksDbContext.Db.NewIterator(ColumnFamily.Handle); _ = iterator.SeekToFirst(); while (iterator.Valid()) { - yield return _keySerializer.Deserialize(iterator.Key()); + yield return KeySerializer.Deserialize(iterator.Key()); _ = iterator.Next(); } } public IEnumerable GetAllValues() { - using var iterator = _rocksDbContext.Db.NewIterator(_columnFamily.Handle); + using var iterator = RocksDbContext.Db.NewIterator(ColumnFamily.Handle); _ = iterator.SeekToFirst(); while (iterator.Valid()) { @@ -323,7 +323,7 @@ public IEnumerable GetAllValues() public int Count() { - using var iterator = _rocksDbContext.Db.NewIterator(_columnFamily.Handle); + using var iterator = RocksDbContext.Db.NewIterator(ColumnFamily.Handle); _ = iterator.SeekToFirst(); var count = 0; while (iterator.Valid()) @@ -341,7 +341,7 @@ public bool HasKey(TKey key) bool useSpan; // ReSharper disable once AssignmentInConditionalExpression - Span keyBuffer = (useSpan = _keySerializer.TryCalculateSize(ref key, out var keySize)) + Span keyBuffer = (useSpan = KeySerializer.TryCalculateSize(ref key, out var keySize)) ? keySize < MaxStackSize ? stackalloc byte[keySize] : (rentedKeyBuffer = ArrayPool.Shared.Rent(keySize)).AsSpan(0, keySize) @@ -354,16 +354,16 @@ public bool HasKey(TKey key) { if (useSpan) { - _keySerializer.WriteTo(ref key, ref keyBuffer); + KeySerializer.WriteTo(ref key, ref keyBuffer); } else { keyBufferWriter = new ArrayPoolBufferWriter(); - _keySerializer.WriteTo(ref key, keyBufferWriter); + KeySerializer.WriteTo(ref key, keyBufferWriter); keySpan = keyBufferWriter.WrittenSpan; } - return _rocksDbContext.Db.HasKey(keySpan, _columnFamily.Handle); + return RocksDbContext.Db.HasKey(keySpan, ColumnFamily.Handle); } finally { @@ -377,9 +377,9 @@ public bool HasKey(TKey key) public void Clear() { - var prevColumnFamilyHandle = _columnFamily.Handle; - _rocksDbContext.Db.DropColumnFamily(_columnFamily.Name); - _columnFamily.Handle = _rocksDbContext.Db.CreateColumnFamily(_rocksDbContext.ColumnFamilyOptions, _columnFamily.Name); + var prevColumnFamilyHandle = ColumnFamily.Handle; + RocksDbContext.Db.DropColumnFamily(ColumnFamily.Name); + ColumnFamily.Handle = RocksDbContext.Db.CreateColumnFamily(RocksDbContext.ColumnFamilyOptions, ColumnFamily.Name); Native.Instance.rocksdb_column_family_handle_destroy(prevColumnFamilyHandle.Handle); } @@ -389,7 +389,7 @@ public void Merge(TKey key, TValue operand) byte[]? rentedKeyBuffer = null; bool useSpanAsKey; // ReSharper disable once AssignmentInConditionalExpression - Span keyBuffer = (useSpanAsKey = _keySerializer.TryCalculateSize(ref key, out var keySize)) + Span keyBuffer = (useSpanAsKey = KeySerializer.TryCalculateSize(ref key, out var keySize)) ? keySize < MaxStackSize ? stackalloc byte[keySize] : (rentedKeyBuffer = ArrayPool.Shared.Rent(keySize)).AsSpan(0, keySize) @@ -416,12 +416,12 @@ public void Merge(TKey key, TValue operand) { if (useSpanAsKey) { - _keySerializer.WriteTo(ref key, ref keyBuffer); + KeySerializer.WriteTo(ref key, ref keyBuffer); } else { keyBufferWriter = new ArrayPoolBufferWriter(); - _keySerializer.WriteTo(ref key, keyBufferWriter); + KeySerializer.WriteTo(ref key, keyBufferWriter); keySpan = keyBufferWriter.WrittenSpan; } @@ -436,7 +436,7 @@ public void Merge(TKey key, TValue operand) valueSpan = valueBufferWriter.WrittenSpan; } - _rocksDbContext.Db.Merge(keySpan, valueSpan, _columnFamily.Handle); + RocksDbContext.Db.Merge(keySpan, valueSpan, ColumnFamily.Handle); } finally { From bd32b5a027592e7fbadca859c323739de5c31fee Mon Sep 17 00:00:00 2001 From: Havret Date: Thu, 11 Dec 2025 23:18:23 +0100 Subject: [PATCH 12/19] wip --- src/RocksDb.Extensions/MergeOperatorConfig.cs | 2 - .../MergeOperators/ListMergeOperator.cs | 10 +++ src/RocksDb.Extensions/RocksDbAccessor.cs | 70 ------------------- src/RocksDb.Extensions/RocksDbBuilder.cs | 14 +++- src/RocksDb.Extensions/RocksDbContext.cs | 3 +- src/RocksDb.Extensions/RocksDbOptions.cs | 3 +- 6 files changed, 24 insertions(+), 78 deletions(-) diff --git a/src/RocksDb.Extensions/MergeOperatorConfig.cs b/src/RocksDb.Extensions/MergeOperatorConfig.cs index ac20b8e..d4b029c 100644 --- a/src/RocksDb.Extensions/MergeOperatorConfig.cs +++ b/src/RocksDb.Extensions/MergeOperatorConfig.cs @@ -1,5 +1,3 @@ -using RocksDbSharp; - namespace RocksDb.Extensions; /// diff --git a/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs b/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs index 6ac0718..181b9aa 100644 --- a/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs +++ b/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs @@ -63,6 +63,16 @@ public IList FullMerge( // Check if any operands contain removes bool hasRemoves = false; var allAdds = new List(); + + foreach (var operand in operands) + { + if (operand.Type == OperationType.Remove) + { + // If there are any removes, we can't safely combine without knowing the existing state + // Return null to signal that RocksDB should keep operands separate + return null; + } + } foreach (var operand in operands) { diff --git a/src/RocksDb.Extensions/RocksDbAccessor.cs b/src/RocksDb.Extensions/RocksDbAccessor.cs index 2c2ea35..53fef90 100644 --- a/src/RocksDb.Extensions/RocksDbAccessor.cs +++ b/src/RocksDb.Extensions/RocksDbAccessor.cs @@ -383,75 +383,5 @@ public void Clear() Native.Instance.rocksdb_column_family_handle_destroy(prevColumnFamilyHandle.Handle); } - - public void Merge(TKey key, TValue operand) - { - byte[]? rentedKeyBuffer = null; - bool useSpanAsKey; - // ReSharper disable once AssignmentInConditionalExpression - Span keyBuffer = (useSpanAsKey = KeySerializer.TryCalculateSize(ref key, out var keySize)) - ? keySize < MaxStackSize - ? stackalloc byte[keySize] - : (rentedKeyBuffer = ArrayPool.Shared.Rent(keySize)).AsSpan(0, keySize) - : Span.Empty; - - ReadOnlySpan keySpan = keyBuffer; - ArrayPoolBufferWriter? keyBufferWriter = null; - - var value = operand; - byte[]? rentedValueBuffer = null; - bool useSpanAsValue; - // ReSharper disable once AssignmentInConditionalExpression - Span valueBuffer = (useSpanAsValue = _valueSerializer.TryCalculateSize(ref value, out var valueSize)) - ? valueSize < MaxStackSize - ? stackalloc byte[valueSize] - : (rentedValueBuffer = ArrayPool.Shared.Rent(valueSize)).AsSpan(0, valueSize) - : Span.Empty; - - - ReadOnlySpan valueSpan = valueBuffer; - ArrayPoolBufferWriter? valueBufferWriter = null; - - try - { - if (useSpanAsKey) - { - KeySerializer.WriteTo(ref key, ref keyBuffer); - } - else - { - keyBufferWriter = new ArrayPoolBufferWriter(); - KeySerializer.WriteTo(ref key, keyBufferWriter); - keySpan = keyBufferWriter.WrittenSpan; - } - - if (useSpanAsValue) - { - _valueSerializer.WriteTo(ref value, ref valueBuffer); - } - else - { - valueBufferWriter = new ArrayPoolBufferWriter(); - _valueSerializer.WriteTo(ref value, valueBufferWriter); - valueSpan = valueBufferWriter.WrittenSpan; - } - - RocksDbContext.Db.Merge(keySpan, valueSpan, ColumnFamily.Handle); - } - finally - { - keyBufferWriter?.Dispose(); - valueBufferWriter?.Dispose(); - if (rentedKeyBuffer is not null) - { - ArrayPool.Shared.Return(rentedKeyBuffer); - } - - if (rentedValueBuffer is not null) - { - ArrayPool.Shared.Return(rentedValueBuffer); - } - } - } } diff --git a/src/RocksDb.Extensions/RocksDbBuilder.cs b/src/RocksDb.Extensions/RocksDbBuilder.cs index e097872..8e8e644 100644 --- a/src/RocksDb.Extensions/RocksDbBuilder.cs +++ b/src/RocksDb.Extensions/RocksDbBuilder.cs @@ -134,12 +134,17 @@ private static byte[] FullMergeCallback(bool hasExistingValue, { operandArray[i] = operandSerializer.Deserialize(operands.Get(i)); } - + var operandSpan = operandArray.AsSpan(0, operands.Count); var result = mergeOperator.FullMerge(existing, operandSpan); return SerializeValue(result, valueSerializer); } + catch + { + success = false; + return Array.Empty(); + } finally { ArrayPool.Shared.Return(operandArray, clearArray: RuntimeHelpers.IsReferenceOrContainsReferences()); @@ -151,7 +156,6 @@ private static byte[] PartialMergeCallback(global::RocksDbShar ISerializer operandSerializer, out bool success) { - // Rent array from pool instead of allocating List var operandArray = ArrayPool.Shared.Rent(operands.Count); try { @@ -160,7 +164,6 @@ private static byte[] PartialMergeCallback(global::RocksDbShar operandArray[i] = operandSerializer.Deserialize(operands.Get(i)); } - // Call the user's merge operator with ReadOnlySpan - zero allocation var operandSpan = operandArray.AsSpan(0, operands.Count); var result = mergeOperator.PartialMerge(operandSpan); @@ -173,6 +176,11 @@ private static byte[] PartialMergeCallback(global::RocksDbShar success = true; return SerializeValue(result, operandSerializer); } + catch + { + success = false; + return Array.Empty(); + } finally { ArrayPool.Shared.Return(operandArray, clearArray: RuntimeHelpers.IsReferenceOrContainsReferences()); diff --git a/src/RocksDb.Extensions/RocksDbContext.cs b/src/RocksDb.Extensions/RocksDbContext.cs index f4ef5e2..7589329 100644 --- a/src/RocksDb.Extensions/RocksDbContext.cs +++ b/src/RocksDb.Extensions/RocksDbContext.cs @@ -111,8 +111,7 @@ private ColumnFamilies CreateColumnFamilies( mergeOperatorConfig.Name, mergeOperatorConfig.PartialMerge, mergeOperatorConfig.FullMerge); - - + cfOptions.SetMergeOperator(mergeOp); columnFamilies.Add(columnFamilyName, cfOptions); } diff --git a/src/RocksDb.Extensions/RocksDbOptions.cs b/src/RocksDb.Extensions/RocksDbOptions.cs index 0fc5305..09f5202 100644 --- a/src/RocksDb.Extensions/RocksDbOptions.cs +++ b/src/RocksDb.Extensions/RocksDbOptions.cs @@ -35,8 +35,9 @@ public class RocksDbOptions /// /// Internal dictionary of merge operators per column family. + /// Column family names are case-sensitive, matching RocksDB's behavior. /// - internal Dictionary MergeOperators { get; } = new(StringComparer.InvariantCultureIgnoreCase); + internal Dictionary MergeOperators { get; } = new(); /// /// Enables direct I/O mode for reads, which bypasses the OS page cache. From d2670f529bdfa2fbb393376aa2a41d3404345bd4 Mon Sep 17 00:00:00 2001 From: Havret Date: Thu, 11 Dec 2025 23:41:07 +0100 Subject: [PATCH 13/19] wip --- src/RocksDb.Extensions/MergeAccessor.cs | 2 +- src/RocksDb.Extensions/RocksDbAccessor.cs | 10 ++-- src/RocksDb.Extensions/RocksDbContext.cs | 66 +++++++++++------------ 3 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/RocksDb.Extensions/MergeAccessor.cs b/src/RocksDb.Extensions/MergeAccessor.cs index da880d0..6825a2e 100644 --- a/src/RocksDb.Extensions/MergeAccessor.cs +++ b/src/RocksDb.Extensions/MergeAccessor.cs @@ -68,7 +68,7 @@ public void Merge(TKey key, TOperand operand) operandSpan = operandBufferWriter.WrittenSpan; } - RocksDbContext.Db.Merge(keySpan, operandSpan, ColumnFamily.Handle); + RocksDbContext.Db.Merge(keySpan, operandSpan, ColumnFamily.Handle, RocksDbContext.WriteOptions); } finally { diff --git a/src/RocksDb.Extensions/RocksDbAccessor.cs b/src/RocksDb.Extensions/RocksDbAccessor.cs index 53fef90..dfa0016 100644 --- a/src/RocksDb.Extensions/RocksDbAccessor.cs +++ b/src/RocksDb.Extensions/RocksDbAccessor.cs @@ -56,7 +56,7 @@ public void Remove(TKey key) keySpan = keyBufferWriter.WrittenSpan; } - RocksDbContext.Db.Remove(keySpan, ColumnFamily.Handle); + RocksDbContext.Db.Remove(keySpan, ColumnFamily.Handle, RocksDbContext.WriteOptions); } finally { @@ -119,7 +119,7 @@ public void Put(TKey key, TValue value) valueSpan = valueBufferWriter.WrittenSpan; } - RocksDbContext.Db.Put(keySpan, valueSpan, ColumnFamily.Handle); + RocksDbContext.Db.Put(keySpan, valueSpan, ColumnFamily.Handle, RocksDbContext.WriteOptions); } finally { @@ -202,7 +202,7 @@ public void PutRange(ReadOnlySpan keys, ReadOnlySpan values) AddToBatch(keys[i], values[i], batch); } - RocksDbContext.Db.Write(batch); + RocksDbContext.Db.Write(batch, RocksDbContext.WriteOptions); } public void PutRange(ReadOnlySpan values, Func keySelector) @@ -215,7 +215,7 @@ public void PutRange(ReadOnlySpan values, Func keySelector AddToBatch(key, value, batch); } - RocksDbContext.Db.Write(batch); + RocksDbContext.Db.Write(batch, RocksDbContext.WriteOptions); } public void PutRange(IReadOnlyList<(TKey key, TValue value)> items) @@ -227,7 +227,7 @@ public void PutRange(IReadOnlyList<(TKey key, TValue value)> items) AddToBatch(key, value, batch); } - RocksDbContext.Db.Write(batch); + RocksDbContext.Db.Write(batch, RocksDbContext.WriteOptions); } private void AddToBatch(TKey key, TValue value, WriteBatch batch) diff --git a/src/RocksDb.Extensions/RocksDbContext.cs b/src/RocksDb.Extensions/RocksDbContext.cs index 7589329..e616582 100644 --- a/src/RocksDb.Extensions/RocksDbContext.cs +++ b/src/RocksDb.Extensions/RocksDbContext.cs @@ -8,6 +8,7 @@ internal class RocksDbContext : IDisposable private readonly RocksDbSharp.RocksDb _rocksDb; private readonly Cache _cache; private readonly ColumnFamilyOptions _userSpecifiedOptions; + private readonly WriteOptions _writeOptions; private const long BlockCacheSize = 50 * 1024 * 1024L; private const long BlockSize = 4096L; @@ -18,22 +19,9 @@ public RocksDbContext(IOptions options) { var dbOptions = new DbOptions(); _userSpecifiedOptions = new ColumnFamilyOptions(); - var tableConfig = new BlockBasedTableOptions(); _cache = Cache.CreateLru(BlockCacheSize); - tableConfig.SetBlockCache(_cache); - tableConfig.SetBlockSize(BlockSize); - - var filter = BloomFilterPolicy.Create(); - tableConfig.SetFilterPolicy(filter); - _userSpecifiedOptions.SetBlockBasedTableFactory(tableConfig); - _userSpecifiedOptions.SetWriteBufferSize(WriteBufferSize); - _userSpecifiedOptions.SetCompression(Compression.No); - _userSpecifiedOptions.SetCompactionStyle(Compaction.Universal); - _userSpecifiedOptions.SetMaxWriteBufferNumber(MaxWriteBuffers); - _userSpecifiedOptions.SetCreateIfMissing(); - _userSpecifiedOptions.SetCreateMissingColumnFamilies(); - _userSpecifiedOptions.SetErrorIfExists(false); - _userSpecifiedOptions.SetInfoLogLevel(InfoLogLevel.Error); + + ConfigureColumnFamilyOptions(_userSpecifiedOptions, _cache); // this is the recommended way to increase parallelism in RocksDb // note that the current implementation of setIncreaseParallelism affects the number @@ -48,11 +36,9 @@ public RocksDbContext(IOptions options) dbOptions.SetUseDirectReads(options.Value.UseDirectReads); dbOptions.SetUseDirectIoForFlushAndCompaction(options.Value.UseDirectIoForFlushAndCompaction); - var fOptions = new FlushOptions(); - fOptions.SetWaitForFlush(options.Value.WaitForFlush); - var writeOptions = new WriteOptions(); - writeOptions.DisableWal(1); + _writeOptions = new WriteOptions(); + _writeOptions.DisableWal(1); _userSpecifiedOptions.EnableStatistics(); @@ -72,10 +58,34 @@ private static void DestroyDatabase(string path) Native.Instance.rocksdb_destroy_db(dbOptions.Handle, path); } + private static void ConfigureColumnFamilyOptions(ColumnFamilyOptions cfOptions, Cache cache) + { + var tableConfig = new BlockBasedTableOptions(); + tableConfig.SetBlockCache(cache); + tableConfig.SetBlockSize(BlockSize); + + var filter = BloomFilterPolicy.Create(); + tableConfig.SetFilterPolicy(filter); + + cfOptions.SetBlockBasedTableFactory(tableConfig); + cfOptions.SetWriteBufferSize(WriteBufferSize); + cfOptions.SetCompression(Compression.No); + cfOptions.SetCompactionStyle(Compaction.Universal); + cfOptions.SetMaxWriteBufferNumber(MaxWriteBuffers); + cfOptions.SetCreateIfMissing(); + cfOptions.SetCreateMissingColumnFamilies(); + cfOptions.SetErrorIfExists(false); + cfOptions.SetInfoLogLevel(InfoLogLevel.Error); + cfOptions.EnableStatistics(); + } + public RocksDbSharp.RocksDb Db => _rocksDb; public ColumnFamilyOptions ColumnFamilyOptions => _userSpecifiedOptions; + public WriteOptions WriteOptions => _writeOptions; + + private ColumnFamilies CreateColumnFamilies( IReadOnlyList columnFamilyNames, IReadOnlyDictionary mergeOperators, @@ -88,23 +98,7 @@ private ColumnFamilies CreateColumnFamilies( { // Create a copy of the default options for this column family var cfOptions = new ColumnFamilyOptions(); - - // Apply the same settings as the default options - var tableConfig = new BlockBasedTableOptions(); - tableConfig.SetBlockCache(_cache); - tableConfig.SetBlockSize(BlockSize); - var filter = BloomFilterPolicy.Create(); - tableConfig.SetFilterPolicy(filter); - cfOptions.SetBlockBasedTableFactory(tableConfig); - cfOptions.SetWriteBufferSize(WriteBufferSize); - cfOptions.SetCompression(Compression.No); - cfOptions.SetCompactionStyle(Compaction.Universal); - cfOptions.SetMaxWriteBufferNumber(MaxWriteBuffers); - cfOptions.SetCreateIfMissing(); - cfOptions.SetCreateMissingColumnFamilies(); - cfOptions.SetErrorIfExists(false); - cfOptions.SetInfoLogLevel(InfoLogLevel.Error); - cfOptions.EnableStatistics(); + ConfigureColumnFamilyOptions(cfOptions, _cache); // Create and set the merge operator var mergeOp = global::RocksDbSharp.MergeOperators.Create( From ee73d9fa59910a7b23d41d94328d3d39c4cb61f2 Mon Sep 17 00:00:00 2001 From: Havret Date: Thu, 11 Dec 2025 23:47:34 +0100 Subject: [PATCH 14/19] wip --- src/RocksDb.Extensions/MergeOperatorConfig.cs | 5 ----- src/RocksDb.Extensions/RocksDbBuilder.cs | 1 - 2 files changed, 6 deletions(-) diff --git a/src/RocksDb.Extensions/MergeOperatorConfig.cs b/src/RocksDb.Extensions/MergeOperatorConfig.cs index d4b029c..5117d25 100644 --- a/src/RocksDb.Extensions/MergeOperatorConfig.cs +++ b/src/RocksDb.Extensions/MergeOperatorConfig.cs @@ -19,9 +19,4 @@ internal class MergeOperatorConfig /// Gets the partial merge callback delegate. /// public global::RocksDbSharp.MergeOperators.PartialMergeFunc PartialMerge { get; set; } = null!; - - /// - /// Gets the value serializer for deserializing and serializing values. - /// - public object ValueSerializer { get; set; } = null!; } diff --git a/src/RocksDb.Extensions/RocksDbBuilder.cs b/src/RocksDb.Extensions/RocksDbBuilder.cs index 8e8e644..d99d697 100644 --- a/src/RocksDb.Extensions/RocksDbBuilder.cs +++ b/src/RocksDb.Extensions/RocksDbBuilder.cs @@ -103,7 +103,6 @@ private static MergeOperatorConfig CreateMergeOperatorConfig( return new MergeOperatorConfig { Name = mergeOperator.Name, - ValueSerializer = valueSerializer, FullMerge = (ReadOnlySpan _, bool hasExistingValue, ReadOnlySpan existingValue, global::RocksDbSharp.MergeOperators.OperandsEnumerator operands, out bool success) => { return FullMergeCallback(hasExistingValue, existingValue, operands, mergeOperator, valueSerializer, operandSerializer, out success); From ae6036cec81e6b2d4b382300e4e2d17ae6c4edd8 Mon Sep 17 00:00:00 2001 From: Havret Date: Fri, 12 Dec 2025 00:10:43 +0100 Subject: [PATCH 15/19] wip --- src/RocksDb.Extensions/MergeOperatorConfig.cs | 112 +++++++++++++++++ .../MergeOperators/ListMergeOperator.cs | 30 +---- src/RocksDb.Extensions/RocksDbBuilder.cs | 114 +----------------- 3 files changed, 117 insertions(+), 139 deletions(-) diff --git a/src/RocksDb.Extensions/MergeOperatorConfig.cs b/src/RocksDb.Extensions/MergeOperatorConfig.cs index 5117d25..c0bfd71 100644 --- a/src/RocksDb.Extensions/MergeOperatorConfig.cs +++ b/src/RocksDb.Extensions/MergeOperatorConfig.cs @@ -1,3 +1,7 @@ +using System.Buffers; +using System.Runtime.CompilerServices; +using CommunityToolkit.HighPerformance.Buffers; + namespace RocksDb.Extensions; /// @@ -19,4 +23,112 @@ internal class MergeOperatorConfig /// Gets the partial merge callback delegate. /// public global::RocksDbSharp.MergeOperators.PartialMergeFunc PartialMerge { get; set; } = null!; + + internal static MergeOperatorConfig CreateMergeOperatorConfig( + IMergeOperator mergeOperator, + ISerializer valueSerializer, + ISerializer operandSerializer) + { + return new MergeOperatorConfig + { + Name = mergeOperator.Name, + FullMerge = (ReadOnlySpan _, bool hasExistingValue, ReadOnlySpan existingValue, global::RocksDbSharp.MergeOperators.OperandsEnumerator operands, out bool success) => + { + return FullMergeCallback(hasExistingValue, existingValue, operands, mergeOperator, valueSerializer, operandSerializer, out success); + }, + PartialMerge = (ReadOnlySpan _, global::RocksDbSharp.MergeOperators.OperandsEnumerator operands, out bool success) => + { + return PartialMergeCallback(operands, mergeOperator, operandSerializer, out success); + } + }; + } + + private static byte[] FullMergeCallback(bool hasExistingValue, + ReadOnlySpan existingValue, + global::RocksDbSharp.MergeOperators.OperandsEnumerator operands, + IMergeOperator mergeOperator, + ISerializer valueSerializer, + ISerializer operandSerializer, + out bool success) + { + success = true; + + var existing = hasExistingValue ? valueSerializer.Deserialize(existingValue) : default!; + + var operandArray = ArrayPool.Shared.Rent(operands.Count); + try + { + for (int i = 0; i < operands.Count; i++) + { + operandArray[i] = operandSerializer.Deserialize(operands.Get(i)); + } + + var operandSpan = operandArray.AsSpan(0, operands.Count); + var result = mergeOperator.FullMerge(existing, operandSpan); + + return SerializeValue(result, valueSerializer); + } + catch + { + success = false; + return Array.Empty(); + } + finally + { + ArrayPool.Shared.Return(operandArray, clearArray: RuntimeHelpers.IsReferenceOrContainsReferences()); + } + } + + private static byte[] PartialMergeCallback(global::RocksDbSharp.MergeOperators.OperandsEnumerator operands, + IMergeOperator mergeOperator, + ISerializer operandSerializer, + out bool success) + { + var operandArray = ArrayPool.Shared.Rent(operands.Count); + try + { + for (int i = 0; i < operands.Count; i++) + { + operandArray[i] = operandSerializer.Deserialize(operands.Get(i)); + } + + var operandSpan = operandArray.AsSpan(0, operands.Count); + var result = mergeOperator.PartialMerge(operandSpan); + + if (result == null) + { + success = false; + return Array.Empty(); + } + + success = true; + return SerializeValue(result, operandSerializer); + } + catch + { + success = false; + return Array.Empty(); + } + finally + { + ArrayPool.Shared.Return(operandArray, clearArray: RuntimeHelpers.IsReferenceOrContainsReferences()); + } + } + + private static byte[] SerializeValue(T value, ISerializer serializer) + { + if (serializer.TryCalculateSize(ref value, out var size)) + { + var buffer = new byte[size]; + var span = buffer.AsSpan(); + serializer.WriteTo(ref value, ref span); + return buffer; + } + else + { + using var bufferWriter = new ArrayPoolBufferWriter(); + serializer.WriteTo(ref value, bufferWriter); + return bufferWriter.WrittenSpan.ToArray(); + } + } } diff --git a/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs b/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs index 181b9aa..bc67aab 100644 --- a/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs +++ b/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs @@ -10,8 +10,8 @@ namespace RocksDb.Extensions.MergeOperators; /// /// public class TagsStore : MergeableRocksDbStore<string, IList<string>, ListOperation<string>> /// { -/// public TagsStore(IRocksDbAccessor<string, IList<string>> accessor, IMergeAccessor<string, ListOperation<string>> mergeAccessor) -/// : base(accessor, mergeAccessor) { } +/// public TagsStore(IMergeAccessor<string, IList<string>, ListOperation<string>> mergeAccessor) +/// : base(mergeAccessor) { } /// /// public void AddTags(string key, params string[] tags) => Merge(key, ListOperation<string>.Add(tags)); /// public void RemoveTags(string key, params string[] tags) => Merge(key, ListOperation<string>.Remove(tags)); @@ -30,10 +30,6 @@ namespace RocksDb.Extensions.MergeOperators; /// Remove operations delete the first occurrence of each item (same as ). /// If an item to remove doesn't exist in the list, the operation is silently ignored. /// -/// -/// For append-only use cases where removes are not needed, prefer -/// which has less serialization overhead. -/// /// public class ListMergeOperator : IMergeOperator, ListOperation> { @@ -60,8 +56,6 @@ public IList FullMerge( /// public ListOperation? PartialMerge(ReadOnlySpan> operands) { - // Check if any operands contain removes - bool hasRemoves = false; var allAdds = new List(); foreach (var operand in operands) @@ -76,26 +70,10 @@ public IList FullMerge( foreach (var operand in operands) { - if (operand.Type == OperationType.Remove) + foreach (var item in operand.Items) { - hasRemoves = true; - break; + allAdds.Add(item); } - - if (operand.Type == OperationType.Add) - { - foreach (var item in operand.Items) - { - allAdds.Add(item); - } - } - } - - // If there are any removes, we can't safely combine without knowing the existing state - // Return null to signal that RocksDB should keep operands separate - if (hasRemoves) - { - return null; } // Only adds present - safe to combine diff --git a/src/RocksDb.Extensions/RocksDbBuilder.cs b/src/RocksDb.Extensions/RocksDbBuilder.cs index d99d697..d5d111b 100644 --- a/src/RocksDb.Extensions/RocksDbBuilder.cs +++ b/src/RocksDb.Extensions/RocksDbBuilder.cs @@ -1,11 +1,7 @@ -using System.Buffers; using System.Reflection; -using System.Runtime.CompilerServices; -using CommunityToolkit.HighPerformance.Buffers; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; -using RocksDbSharp; namespace RocksDb.Extensions; @@ -66,7 +62,7 @@ public IRocksDbBuilder AddMergeableStore(string var valueSerializer = CreateSerializer(options.SerializerFactories); var operandSerializer = CreateSerializer(options.SerializerFactories); - var config = CreateMergeOperatorConfig(mergeOperator, valueSerializer, operandSerializer); + var config = MergeOperatorConfig.CreateMergeOperatorConfig(mergeOperator, valueSerializer, operandSerializer); options.MergeOperators[columnFamily] = config; }); @@ -95,114 +91,6 @@ public IRocksDbBuilder AddMergeableStore(string return this; } - private static MergeOperatorConfig CreateMergeOperatorConfig( - IMergeOperator mergeOperator, - ISerializer valueSerializer, - ISerializer operandSerializer) - { - return new MergeOperatorConfig - { - Name = mergeOperator.Name, - FullMerge = (ReadOnlySpan _, bool hasExistingValue, ReadOnlySpan existingValue, global::RocksDbSharp.MergeOperators.OperandsEnumerator operands, out bool success) => - { - return FullMergeCallback(hasExistingValue, existingValue, operands, mergeOperator, valueSerializer, operandSerializer, out success); - }, - PartialMerge = (ReadOnlySpan _, global::RocksDbSharp.MergeOperators.OperandsEnumerator operands, out bool success) => - { - return PartialMergeCallback(operands, mergeOperator, operandSerializer, out success); - } - }; - } - - private static byte[] FullMergeCallback(bool hasExistingValue, - ReadOnlySpan existingValue, - global::RocksDbSharp.MergeOperators.OperandsEnumerator operands, - IMergeOperator mergeOperator, - ISerializer valueSerializer, - ISerializer operandSerializer, - out bool success) - { - success = true; - - var existing = hasExistingValue ? valueSerializer.Deserialize(existingValue) : default!; - - var operandArray = ArrayPool.Shared.Rent(operands.Count); - try - { - for (int i = 0; i < operands.Count; i++) - { - operandArray[i] = operandSerializer.Deserialize(operands.Get(i)); - } - - var operandSpan = operandArray.AsSpan(0, operands.Count); - var result = mergeOperator.FullMerge(existing, operandSpan); - - return SerializeValue(result, valueSerializer); - } - catch - { - success = false; - return Array.Empty(); - } - finally - { - ArrayPool.Shared.Return(operandArray, clearArray: RuntimeHelpers.IsReferenceOrContainsReferences()); - } - } - - private static byte[] PartialMergeCallback(global::RocksDbSharp.MergeOperators.OperandsEnumerator operands, - IMergeOperator mergeOperator, - ISerializer operandSerializer, - out bool success) - { - var operandArray = ArrayPool.Shared.Rent(operands.Count); - try - { - for (int i = 0; i < operands.Count; i++) - { - operandArray[i] = operandSerializer.Deserialize(operands.Get(i)); - } - - var operandSpan = operandArray.AsSpan(0, operands.Count); - var result = mergeOperator.PartialMerge(operandSpan); - - if (result == null) - { - success = false; - return Array.Empty(); - } - - success = true; - return SerializeValue(result, operandSerializer); - } - catch - { - success = false; - return Array.Empty(); - } - finally - { - ArrayPool.Shared.Return(operandArray, clearArray: RuntimeHelpers.IsReferenceOrContainsReferences()); - } - } - - private static byte[] SerializeValue(T value, ISerializer serializer) - { - if (serializer.TryCalculateSize(ref value, out var size)) - { - var buffer = new byte[size]; - var span = buffer.AsSpan(); - serializer.WriteTo(ref value, ref span); - return buffer; - } - else - { - using var bufferWriter = new ArrayPoolBufferWriter(); - serializer.WriteTo(ref value, bufferWriter); - return bufferWriter.WrittenSpan.ToArray(); - } - } - private static ISerializer CreateSerializer(IReadOnlyList serializerFactories) { var type = typeof(T); From 4cc4e274483c1955aa55b8b7ea1fc64a198d8663 Mon Sep 17 00:00:00 2001 From: Havret Date: Fri, 12 Dec 2025 00:19:08 +0100 Subject: [PATCH 16/19] wip --- src/RocksDb.Extensions/IMergeOperator.cs | 2 +- src/RocksDb.Extensions/IRocksDbBuilder.cs | 2 +- .../ListOperationSerializer.cs | 14 +++++------ .../MergeOperators/ListMergeOperator.cs | 24 +++++++++---------- .../MergeOperators/ListOperation.cs | 18 +++++++------- .../MergeOperators/OperationType.cs | 6 ++--- .../MergeableRocksDbStore.cs | 12 +++++----- src/RocksDb.Extensions/RocksDbBuilder.cs | 4 ++-- .../MergeOperatorTests.cs | 22 ++++++++--------- 9 files changed, 52 insertions(+), 52 deletions(-) diff --git a/src/RocksDb.Extensions/IMergeOperator.cs b/src/RocksDb.Extensions/IMergeOperator.cs index 7cb0a32..014de46 100644 --- a/src/RocksDb.Extensions/IMergeOperator.cs +++ b/src/RocksDb.Extensions/IMergeOperator.cs @@ -12,7 +12,7 @@ namespace RocksDb.Extensions; /// /// For counters: TValue=long, TOperand=long (same type) /// For list append: TValue=IList<T>, TOperand=IList<T> (same type) -/// For list with add/remove: TValue=IList<T>, TOperand=ListOperation<T> (different types) +/// For list with add/remove: TValue=IList<T>, TOperand=CollectionOperation<T> (different types) /// /// public interface IMergeOperator diff --git a/src/RocksDb.Extensions/IRocksDbBuilder.cs b/src/RocksDb.Extensions/IRocksDbBuilder.cs index cb0b67c..2edf242 100644 --- a/src/RocksDb.Extensions/IRocksDbBuilder.cs +++ b/src/RocksDb.Extensions/IRocksDbBuilder.cs @@ -46,7 +46,7 @@ public interface IRocksDbBuilder /// /// Counters: TValue=long, TOperand=long (same type) /// List append: TValue=IList<T>, TOperand=IList<T> (same type) - /// List with add/remove: TValue=IList<T>, TOperand=ListOperation<T> (different types) + /// List with add/remove: TValue=IList<T>, TOperand=CollectionOperation<T> (different types) /// /// /// diff --git a/src/RocksDb.Extensions/ListOperationSerializer.cs b/src/RocksDb.Extensions/ListOperationSerializer.cs index 2fbf1d4..f8d0c60 100644 --- a/src/RocksDb.Extensions/ListOperationSerializer.cs +++ b/src/RocksDb.Extensions/ListOperationSerializer.cs @@ -4,7 +4,7 @@ namespace RocksDb.Extensions; /// -/// Serializes ListOperation<T> which contains an operation type (Add/Remove) and a list of items. +/// Serializes CollectionOperation<T> which contains an operation type (Add/Remove) and a list of items. /// /// /// The serialized format consists of: @@ -14,7 +14,7 @@ namespace RocksDb.Extensions; /// - 4 bytes: Size of the serialized item /// - N bytes: Serialized item data /// -internal class ListOperationSerializer : ISerializer> +internal class ListOperationSerializer : ISerializer> { private readonly ISerializer _itemSerializer; @@ -23,7 +23,7 @@ public ListOperationSerializer(ISerializer itemSerializer) _itemSerializer = itemSerializer; } - public bool TryCalculateSize(ref ListOperation value, out int size) + public bool TryCalculateSize(ref CollectionOperation value, out int size) { // 1 byte for operation type + 4 bytes for count size = sizeof(byte) + sizeof(int); @@ -41,7 +41,7 @@ public bool TryCalculateSize(ref ListOperation value, out int size) return true; } - public void WriteTo(ref ListOperation value, ref Span span) + public void WriteTo(ref CollectionOperation value, ref Span span) { int offset = 0; @@ -71,12 +71,12 @@ public void WriteTo(ref ListOperation value, ref Span span) } } - public void WriteTo(ref ListOperation value, IBufferWriter buffer) + public void WriteTo(ref CollectionOperation value, IBufferWriter buffer) { throw new NotImplementedException(); } - public ListOperation Deserialize(ReadOnlySpan buffer) + public CollectionOperation Deserialize(ReadOnlySpan buffer) { int offset = 0; @@ -103,6 +103,6 @@ public ListOperation Deserialize(ReadOnlySpan buffer) offset += itemSize; } - return new ListOperation(operationType, items); + return new CollectionOperation(operationType, items); } } diff --git a/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs b/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs index bc67aab..06c42d8 100644 --- a/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs +++ b/src/RocksDb.Extensions/MergeOperators/ListMergeOperator.cs @@ -2,36 +2,36 @@ namespace RocksDb.Extensions.MergeOperators; /// /// A merge operator that supports both adding and removing items from a list. -/// Each merge operand is a ListOperation that specifies whether to add or remove items. +/// Each merge operand is a CollectionOperation that specifies whether to add or remove items. /// Operations are applied in order, enabling atomic list modifications without read-before-write. /// /// The type of elements in the list. /// /// -/// public class TagsStore : MergeableRocksDbStore<string, IList<string>, ListOperation<string>> +/// public class TagsStore : MergeableRocksDbStore<string, IList<string>, CollectionOperation<string>> /// { -/// public TagsStore(IMergeAccessor<string, IList<string>, ListOperation<string>> mergeAccessor) +/// public TagsStore(IMergeAccessor<string, IList<string>, CollectionOperation<string>> mergeAccessor) /// : base(mergeAccessor) { } /// -/// public void AddTags(string key, params string[] tags) => Merge(key, ListOperation<string>.Add(tags)); -/// public void RemoveTags(string key, params string[] tags) => Merge(key, ListOperation<string>.Remove(tags)); +/// public void AddTags(string key, params string[] tags) => Merge(key, CollectionOperation<string>.Add(tags)); +/// public void RemoveTags(string key, params string[] tags) => Merge(key, CollectionOperation<string>.Remove(tags)); /// } /// /// // Registration: -/// builder.AddMergeableStore<string, IList<string>, ListOperation<string>, TagsStore>("tags", new ListMergeOperator<string>()); +/// builder.AddMergeableStore<string, IList<string>, CollectionOperation<string>, TagsStore>("tags", new ListMergeOperator<string>()); /// /// /// /// /// The value type stored in RocksDB is IList<T> (the actual list contents), -/// while merge operands are ListOperation<T> (the operations to apply). +/// while merge operands are CollectionOperation<T> (the operations to apply). /// /// /// Remove operations delete the first occurrence of each item (same as ). /// If an item to remove doesn't exist in the list, the operation is silently ignored. /// /// -public class ListMergeOperator : IMergeOperator, ListOperation> +public class ListMergeOperator : IMergeOperator, CollectionOperation> { /// public string Name => $"ListMergeOperator<{typeof(T).Name}>"; @@ -39,7 +39,7 @@ public class ListMergeOperator : IMergeOperator, ListOperation> /// public IList FullMerge( IList? existingValue, - ReadOnlySpan> operands) + ReadOnlySpan> operands) { // Start with existing items or empty list var result = existingValue != null ? new List(existingValue) : new List(); @@ -54,7 +54,7 @@ public IList FullMerge( } /// - public ListOperation? PartialMerge(ReadOnlySpan> operands) + public CollectionOperation? PartialMerge(ReadOnlySpan> operands) { var allAdds = new List(); @@ -77,10 +77,10 @@ public IList FullMerge( } // Only adds present - safe to combine - return ListOperation.Add(allAdds); + return CollectionOperation.Add(allAdds); } - private static void ApplyOperation(List result, ListOperation operation) + private static void ApplyOperation(List result, CollectionOperation operation) { switch (operation.Type) { diff --git a/src/RocksDb.Extensions/MergeOperators/ListOperation.cs b/src/RocksDb.Extensions/MergeOperators/ListOperation.cs index 711649c..a0c8c34 100644 --- a/src/RocksDb.Extensions/MergeOperators/ListOperation.cs +++ b/src/RocksDb.Extensions/MergeOperators/ListOperation.cs @@ -1,10 +1,10 @@ namespace RocksDb.Extensions.MergeOperators; /// -/// Represents an operation (add or remove) to apply to a list via merge. +/// Represents an operation (add or remove) to apply to a collection via merge. /// -/// The type of elements in the list. -public class ListOperation +/// The type of elements in the collection. +public class CollectionOperation { /// /// Gets the type of operation to perform. @@ -17,11 +17,11 @@ public class ListOperation public IList Items { get; } /// - /// Creates a new list operation. + /// Creates a new collection operation. /// /// The type of operation. /// The items to add or remove. - public ListOperation(OperationType type, IList items) + public CollectionOperation(OperationType type, IList items) { Type = type; Items = items ?? throw new ArgumentNullException(nameof(items)); @@ -30,20 +30,20 @@ public ListOperation(OperationType type, IList items) /// /// Creates an Add operation for the specified items. /// - public static ListOperation Add(params T[] items) => new(OperationType.Add, items); + public static CollectionOperation Add(params T[] items) => new(OperationType.Add, items); /// /// Creates an Add operation for the specified items. /// - public static ListOperation Add(IList items) => new(OperationType.Add, items); + public static CollectionOperation Add(IList items) => new(OperationType.Add, items); /// /// Creates a Remove operation for the specified items. /// - public static ListOperation Remove(params T[] items) => new(OperationType.Remove, items); + public static CollectionOperation Remove(params T[] items) => new(OperationType.Remove, items); /// /// Creates a Remove operation for the specified items. /// - public static ListOperation Remove(IList items) => new(OperationType.Remove, items); + public static CollectionOperation Remove(IList items) => new(OperationType.Remove, items); } diff --git a/src/RocksDb.Extensions/MergeOperators/OperationType.cs b/src/RocksDb.Extensions/MergeOperators/OperationType.cs index a500bef..d6305f6 100644 --- a/src/RocksDb.Extensions/MergeOperators/OperationType.cs +++ b/src/RocksDb.Extensions/MergeOperators/OperationType.cs @@ -1,17 +1,17 @@ namespace RocksDb.Extensions.MergeOperators; /// -/// Specifies the type of operation to perform on a list. +/// Specifies the type of operation to perform on a collection. /// public enum OperationType { /// - /// Add items to the list. + /// Add items to the collection. /// Add, /// - /// Remove items from the list (first occurrence of each item). + /// Remove items from the collection. /// Remove } \ No newline at end of file diff --git a/src/RocksDb.Extensions/MergeableRocksDbStore.cs b/src/RocksDb.Extensions/MergeableRocksDbStore.cs index c36ec3d..510c8fb 100644 --- a/src/RocksDb.Extensions/MergeableRocksDbStore.cs +++ b/src/RocksDb.Extensions/MergeableRocksDbStore.cs @@ -15,7 +15,7 @@ namespace RocksDb.Extensions; /// Merge operations are useful for: /// - Counters: Increment/decrement without reading current value (TValue=long, TOperand=long) /// - Lists: Append items without reading the entire list (TValue=IList<T>, TOperand=IList<T>) -/// - Lists with add/remove: Modify lists atomically (TValue=IList<T>, TOperand=ListOperation<T>) +/// - Lists with add/remove: Modify lists atomically (TValue=IList<T>, TOperand=CollectionOperation<T>) /// /// /// When using this base class, you must register the store with a merge operator using @@ -33,14 +33,14 @@ namespace RocksDb.Extensions; /// public void Increment(string key, long delta = 1) => Merge(key, delta); /// } /// -/// // Tags store where value is IList<string> but operand is ListOperation<string> -/// public class TagsStore : MergeableRocksDbStore<string, IList<string>, ListOperation<string>> +/// // Tags store where value is IList<string> but operand is CollectionOperation<string> +/// public class TagsStore : MergeableRocksDbStore<string, IList<string>, CollectionOperation<string>> /// { -/// public TagsStore(IRocksDbAccessor<string, IList<string>> accessor, IMergeAccessor<string, ListOperation<string>> mergeAccessor) +/// public TagsStore(IRocksDbAccessor<string, IList<string>> accessor, IMergeAccessor<string, CollectionOperation<string>> mergeAccessor) /// : base(accessor, mergeAccessor) { } /// -/// public void AddTag(string key, string tag) => Merge(key, ListOperation<string>.Add(tag)); -/// public void RemoveTag(string key, string tag) => Merge(key, ListOperation<string>.Remove(tag)); +/// public void AddTag(string key, string tag) => Merge(key, CollectionOperation<string>.Add(tag)); +/// public void RemoveTag(string key, string tag) => Merge(key, CollectionOperation<string>.Remove(tag)); /// } /// /// diff --git a/src/RocksDb.Extensions/RocksDbBuilder.cs b/src/RocksDb.Extensions/RocksDbBuilder.cs index d5d111b..82f1c38 100644 --- a/src/RocksDb.Extensions/RocksDbBuilder.cs +++ b/src/RocksDb.Extensions/RocksDbBuilder.cs @@ -123,8 +123,8 @@ private static ISerializer CreateSerializer(IReadOnlyList) Activator.CreateInstance(typeof(VariableSizeListSerializer<>).MakeGenericType(elementType), scalarSerializer)!; } - // Handle ListOperation for the ListMergeOperator - if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(MergeOperators.ListOperation<>)) + // Handle CollectionOperation for the ListMergeOperator + if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(MergeOperators.CollectionOperation<>)) { var itemType = type.GetGenericArguments()[0]; diff --git a/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs b/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs index b7d1988..94cc8a5 100644 --- a/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs +++ b/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs @@ -4,21 +4,21 @@ namespace RocksDb.Extensions.Tests; -public class TagsStore : MergeableRocksDbStore, ListOperation> +public class TagsStore : MergeableRocksDbStore, CollectionOperation> { - public TagsStore(IMergeAccessor, ListOperation> mergeAccessor) + public TagsStore(IMergeAccessor, CollectionOperation> mergeAccessor) : base(mergeAccessor) { } public void AddTags(string key, params string[] tags) { - Merge(key, ListOperation.Add(tags)); + Merge(key, CollectionOperation.Add(tags)); } public void RemoveTags(string key, params string[] tags) { - Merge(key, ListOperation.Remove(tags)); + Merge(key, CollectionOperation.Remove(tags)); } } @@ -30,7 +30,7 @@ public void should_add_to_existing_list_using_merge_operation() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddMergeableStore, TagsStore, ListOperation>("tags", new ListMergeOperator()); + rockDb.AddMergeableStore, TagsStore, CollectionOperation>("tags", new ListMergeOperator()); }); var store = testFixture.GetStore(); @@ -55,7 +55,7 @@ public void should_add_items_to_list_using_list_merge_operator() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddMergeableStore, TagsStore, ListOperation>("tags", new ListMergeOperator()); + rockDb.AddMergeableStore, TagsStore, CollectionOperation>("tags", new ListMergeOperator()); }); var store = testFixture.GetStore(); @@ -80,14 +80,14 @@ public void should_remove_items_from_list_using_list_merge_operator() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddMergeableStore, TagsStore, ListOperation>("tags", new ListMergeOperator()); + rockDb.AddMergeableStore, TagsStore, CollectionOperation>("tags", new ListMergeOperator()); }); var store = testFixture.GetStore(); var key = "article-1"; // Act - Add items, then remove some - store.Merge(key, ListOperation.Add("csharp", "dotnet", "java", "python")); + store.Merge(key, CollectionOperation.Add("csharp", "dotnet", "java", "python")); store.RemoveTags(key, "java", "python"); // Assert @@ -106,7 +106,7 @@ public void should_handle_mixed_add_and_remove_operations() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddMergeableStore, TagsStore, ListOperation>("tags", new ListMergeOperator()); + rockDb.AddMergeableStore, TagsStore, CollectionOperation>("tags", new ListMergeOperator()); }); var store = testFixture.GetStore(); @@ -132,7 +132,7 @@ public void should_handle_remove_nonexistent_item_gracefully() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddMergeableStore, TagsStore, ListOperation>("tags", new ListMergeOperator()); + rockDb.AddMergeableStore, TagsStore, CollectionOperation>("tags", new ListMergeOperator()); }); var store = testFixture.GetStore(); @@ -155,7 +155,7 @@ public void should_remove_only_first_occurrence_of_duplicate_items() // Arrange using var testFixture = TestFixture.Create(rockDb => { - rockDb.AddMergeableStore, TagsStore, ListOperation>("tags", new ListMergeOperator()); + rockDb.AddMergeableStore, TagsStore, CollectionOperation>("tags", new ListMergeOperator()); }); var store = testFixture.GetStore(); From 0d6565c74251437297374a5ef198499a9e780a30 Mon Sep 17 00:00:00 2001 From: Havret Date: Wed, 17 Dec 2025 17:49:29 +0100 Subject: [PATCH 17/19] wip --- .../MergeableRocksDbStore.cs | 104 ++-------------- src/RocksDb.Extensions/RocksDbStore.cs | 94 +-------------- src/RocksDb.Extensions/RocksDbStoreBase.cs | 111 ++++++++++++++++++ 3 files changed, 124 insertions(+), 185 deletions(-) create mode 100644 src/RocksDb.Extensions/RocksDbStoreBase.cs diff --git a/src/RocksDb.Extensions/MergeableRocksDbStore.cs b/src/RocksDb.Extensions/MergeableRocksDbStore.cs index 510c8fb..99d684a 100644 --- a/src/RocksDb.Extensions/MergeableRocksDbStore.cs +++ b/src/RocksDb.Extensions/MergeableRocksDbStore.cs @@ -1,5 +1,3 @@ -using System.Diagnostics.CodeAnalysis; - namespace RocksDb.Extensions; /// @@ -27,8 +25,8 @@ namespace RocksDb.Extensions; /// // Counter store where value and operand are the same type /// public class CounterStore : MergeableRocksDbStore<string, long, long> /// { -/// public CounterStore(IRocksDbAccessor<string, long> accessor, IMergeAccessor<string, long> mergeAccessor) -/// : base(accessor, mergeAccessor) { } +/// public CounterStore(IMergeAccessor<string, long, long> mergeAccessor) +/// : base(mergeAccessor) { } /// /// public void Increment(string key, long delta = 1) => Merge(key, delta); /// } @@ -36,25 +34,25 @@ namespace RocksDb.Extensions; /// // Tags store where value is IList<string> but operand is CollectionOperation<string> /// public class TagsStore : MergeableRocksDbStore<string, IList<string>, CollectionOperation<string>> /// { -/// public TagsStore(IRocksDbAccessor<string, IList<string>> accessor, IMergeAccessor<string, CollectionOperation<string>> mergeAccessor) -/// : base(accessor, mergeAccessor) { } +/// public TagsStore(IMergeAccessor<string, IList<string>, CollectionOperation<string>> mergeAccessor) +/// : base(mergeAccessor) { } /// /// public void AddTag(string key, string tag) => Merge(key, CollectionOperation<string>.Add(tag)); /// public void RemoveTag(string key, string tag) => Merge(key, CollectionOperation<string>.Remove(tag)); /// } /// /// -public abstract class MergeableRocksDbStore +public abstract class MergeableRocksDbStore : RocksDbStoreBase { - private readonly IMergeAccessor _rocksDbAccessor; + private readonly IMergeAccessor _mergeAccessor; /// /// Initializes a new instance of the class. /// /// The RocksDB accessor to use for database operations. - protected MergeableRocksDbStore(IMergeAccessor rocksDbAccessor) + protected MergeableRocksDbStore(IMergeAccessor rocksDbAccessor) : base(rocksDbAccessor) { - _rocksDbAccessor = rocksDbAccessor; + _mergeAccessor = rocksDbAccessor; } /// @@ -64,89 +62,5 @@ protected MergeableRocksDbStore(IMergeAccessor rocksDbAc /// /// The key to merge the operand with. /// The operand to merge with the existing value. - public void Merge(TKey key, TOperand operand) => _rocksDbAccessor.Merge(key, operand); - - /// - /// Removes the specified key and its associated value from the store. - /// - /// The key to remove. - public void Remove(TKey key) => _rocksDbAccessor.Remove(key); - - /// - /// Adds or updates the specified key-value pair in the store. - /// - /// The key to add or update. - /// The value to add or update. - public void Put(TKey key, TValue value) => _rocksDbAccessor.Put(key, value); - - /// - /// Tries to get the value associated with the specified key in the store. - /// - /// The key of the value to get. - /// The value associated with the specified key, if the key is found; otherwise, the default value for the type of the value parameter. - /// true if the key is found; otherwise, false. - public bool TryGet(TKey key, [MaybeNullWhen(false)] out TValue value) => _rocksDbAccessor.TryGet(key, out value); - - /// - /// Puts the specified keys and values in the store. - /// - /// The keys to put in the store. - /// The values to put in the store. - public void PutRange(ReadOnlySpan keys, ReadOnlySpan values) => _rocksDbAccessor.PutRange(keys, values); - - /// - /// Puts the specified values in the store using the specified key selector function to generate keys. - /// - /// The values to put in the store. - /// The function to use to generate keys for the values. - public void PutRange(ReadOnlySpan values, Func keySelector) => _rocksDbAccessor.PutRange(values, keySelector); - - /// - /// Adds or updates a collection of key-value pairs in the store. - /// - /// The collection of key-value pairs to add or update. - public void PutRange(IReadOnlyList<(TKey key, TValue value)> items) => _rocksDbAccessor.PutRange(items); - - /// - /// Gets all the values in the store. - /// - /// An enumerable collection of all the values in the store. - public IEnumerable GetAllValues() => _rocksDbAccessor.GetAllValues(); - - /// - /// Determines whether the store contains a value for a specific key. - /// - /// The key to check in the store for an associated value. - /// true if the store contains an element with the specified key; otherwise, false. - public bool HasKey(TKey key) => _rocksDbAccessor.HasKey(key); - - /// - /// Resets the column family associated with the store. - /// This operation destroys the current column family and creates a new one, - /// effectively removing all stored key-value pairs. - /// - /// Note: This method is intended for scenarios where a complete reset of the column family - /// is required. The operation may involve internal reallocation and metadata changes, which - /// can impact performance during execution. Use with caution in high-frequency workflows. - /// - public void Clear() => _rocksDbAccessor.Clear(); - - /// - /// Gets the number of key-value pairs currently stored. - /// - /// - /// This method is not a constant-time operation. Internally, it iterates over all entries in the store - /// to compute the count. While the keys and values are not deserialized during iteration, this process may still - /// be expensive for large datasets. - /// - /// Use this method with caution in performance-critical paths, especially if the store contains a high number of entries. - /// - /// The total count of items in the store. - public int Count() => _rocksDbAccessor.Count(); - - /// - /// Gets all the keys in the store. - /// - /// An enumerable collection of all the keys in the store. - public IEnumerable GetAllKeys() => _rocksDbAccessor.GetAllKeys(); + public void Merge(TKey key, TOperand operand) => _mergeAccessor.Merge(key, operand); } diff --git a/src/RocksDb.Extensions/RocksDbStore.cs b/src/RocksDb.Extensions/RocksDbStore.cs index c39f6b9..3b573e6 100644 --- a/src/RocksDb.Extensions/RocksDbStore.cs +++ b/src/RocksDb.Extensions/RocksDbStore.cs @@ -1,5 +1,3 @@ -using System.Diagnostics.CodeAnalysis; - namespace RocksDb.Extensions; /// @@ -7,62 +5,15 @@ namespace RocksDb.Extensions; /// /// The type of the store's keys. /// The type of the store's values. -public abstract class RocksDbStore +public abstract class RocksDbStore : RocksDbStoreBase { - private readonly IRocksDbAccessor _rocksDbAccessor; - /// /// Initializes a new instance of the class with the specified RocksDB accessor. /// /// The RocksDB accessor to use for database operations. - protected RocksDbStore(IRocksDbAccessor rocksDbAccessor) => _rocksDbAccessor = rocksDbAccessor; - - /// - /// Removes the specified key and its associated value from the store. - /// - /// The key to remove. - public void Remove(TKey key) => _rocksDbAccessor.Remove(key); - - /// - /// Adds or updates the specified key-value pair in the store. - /// - /// The key to add or update. - /// The value to add or update. - public void Put(TKey key, TValue value) => _rocksDbAccessor.Put(key, value); - - /// - /// Tries to get the value associated with the specified key in the store. - /// - /// The key of the value to get. - /// The value associated with the specified key, if the key is found; otherwise, the default value for the type of the value parameter. - /// true if the key is found; otherwise, false. - public bool TryGet(TKey key, [MaybeNullWhen(false)] out TValue value) => _rocksDbAccessor.TryGet(key, out value); - - /// - /// Puts the specified keys and values in the store. - /// - /// The keys to put in the store. - /// The values to put in the store. - public void PutRange(ReadOnlySpan keys, ReadOnlySpan values) => _rocksDbAccessor.PutRange(keys, values); - - /// - /// Puts the specified values in the store using the specified key selector function to generate keys. - /// - /// The values to put in the store. - /// The function to use to generate keys for the values. - public void PutRange(ReadOnlySpan values, Func keySelector) => _rocksDbAccessor.PutRange(values, keySelector); - - /// - /// Adds or updates a collection of key-value pairs in the store. - /// - /// The collection of key-value pairs to add or update. - public void PutRange(IReadOnlyList<(TKey key, TValue value)> items) => _rocksDbAccessor.PutRange(items); - - /// - /// Gets all the values in the store. - /// - /// An enumerable collection of all the values in the store. - public IEnumerable GetAllValues() => _rocksDbAccessor.GetAllValues(); + protected RocksDbStore(IRocksDbAccessor rocksDbAccessor) : base(rocksDbAccessor) + { + } /// /// Gets all the values in the store. (Obsolete, use GetAllValues instead) @@ -70,41 +21,4 @@ public abstract class RocksDbStore /// An enumerable collection of all the values in the store. [Obsolete("Use GetAllValues() instead.")] public IEnumerable GetAll() => GetAllValues(); - - /// - /// Determines whether the store contains a value for a specific key. - /// - /// The key to check in the store for an associated value. - /// true if the store contains an element with the specified key; otherwise, false. - public bool HasKey(TKey key) => _rocksDbAccessor.HasKey(key); - - /// - /// Resets the column family associated with the store. - /// This operation destroys the current column family and creates a new one, - /// effectively removing all stored key-value pairs. - /// - /// Note: This method is intended for scenarios where a complete reset of the column family - /// is required. The operation may involve internal reallocation and metadata changes, which - /// can impact performance during execution. Use with caution in high-frequency workflows. - /// - public void Clear() => _rocksDbAccessor.Clear(); - - /// - /// Gets the number of key-value pairs currently stored. - /// - /// - /// This method is not a constant-time operation. Internally, it iterates over all entries in the store - /// to compute the count. While the keys and values are not deserialized during iteration, this process may still - /// be expensive for large datasets. - /// - /// Use this method with caution in performance-critical paths, especially if the store contains a high number of entries. - /// - /// The total count of items in the store. - public int Count() => _rocksDbAccessor.Count(); - - /// - /// Gets all the keys in the store. - /// - /// An enumerable collection of all the keys in the store. - public IEnumerable GetAllKeys() => _rocksDbAccessor.GetAllKeys(); } diff --git a/src/RocksDb.Extensions/RocksDbStoreBase.cs b/src/RocksDb.Extensions/RocksDbStoreBase.cs new file mode 100644 index 0000000..0d04dbe --- /dev/null +++ b/src/RocksDb.Extensions/RocksDbStoreBase.cs @@ -0,0 +1,111 @@ +using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; + +namespace RocksDb.Extensions; + +/// +/// Base class containing common operations for RocksDB stores. +/// This class is not intended for direct use by library consumers. +/// Use or instead. +/// +/// The type of the store's keys. +/// The type of the store's values. +[EditorBrowsable(EditorBrowsableState.Never)] +public abstract class RocksDbStoreBase +{ + private readonly IRocksDbAccessor _rocksDbAccessor; + + /// + /// Initializes a new instance of the class. + /// + /// The RocksDB accessor to use for database operations. + protected internal RocksDbStoreBase(IRocksDbAccessor rocksDbAccessor) + { + _rocksDbAccessor = rocksDbAccessor; + } + + /// + /// Removes the specified key and its associated value from the store. + /// + /// The key to remove. + public void Remove(TKey key) => _rocksDbAccessor.Remove(key); + + /// + /// Adds or updates the specified key-value pair in the store. + /// + /// The key to add or update. + /// The value to add or update. + public void Put(TKey key, TValue value) => _rocksDbAccessor.Put(key, value); + + /// + /// Tries to get the value associated with the specified key in the store. + /// + /// The key of the value to get. + /// The value associated with the specified key, if the key is found; otherwise, the default value for the type of the value parameter. + /// true if the key is found; otherwise, false. + public bool TryGet(TKey key, [MaybeNullWhen(false)] out TValue value) => _rocksDbAccessor.TryGet(key, out value); + + /// + /// Puts the specified keys and values in the store. + /// + /// The keys to put in the store. + /// The values to put in the store. + public void PutRange(ReadOnlySpan keys, ReadOnlySpan values) => _rocksDbAccessor.PutRange(keys, values); + + /// + /// Puts the specified values in the store using the specified key selector function to generate keys. + /// + /// The values to put in the store. + /// The function to use to generate keys for the values. + public void PutRange(ReadOnlySpan values, Func keySelector) => _rocksDbAccessor.PutRange(values, keySelector); + + /// + /// Adds or updates a collection of key-value pairs in the store. + /// + /// The collection of key-value pairs to add or update. + public void PutRange(IReadOnlyList<(TKey key, TValue value)> items) => _rocksDbAccessor.PutRange(items); + + /// + /// Gets all the values in the store. + /// + /// An enumerable collection of all the values in the store. + public IEnumerable GetAllValues() => _rocksDbAccessor.GetAllValues(); + + /// + /// Determines whether the store contains a value for a specific key. + /// + /// The key to check in the store for an associated value. + /// true if the store contains an element with the specified key; otherwise, false. + public bool HasKey(TKey key) => _rocksDbAccessor.HasKey(key); + + /// + /// Resets the column family associated with the store. + /// This operation destroys the current column family and creates a new one, + /// effectively removing all stored key-value pairs. + /// + /// Note: This method is intended for scenarios where a complete reset of the column family + /// is required. The operation may involve internal reallocation and metadata changes, which + /// can impact performance during execution. Use with caution in high-frequency workflows. + /// + public void Clear() => _rocksDbAccessor.Clear(); + + /// + /// Gets the number of key-value pairs currently stored. + /// + /// + /// This method is not a constant-time operation. Internally, it iterates over all entries in the store + /// to compute the count. While the keys and values are not deserialized during iteration, this process may still + /// be expensive for large datasets. + /// + /// Use this method with caution in performance-critical paths, especially if the store contains a high number of entries. + /// + /// The total count of items in the store. + public int Count() => _rocksDbAccessor.Count(); + + /// + /// Gets all the keys in the store. + /// + /// An enumerable collection of all the keys in the store. + public IEnumerable GetAllKeys() => _rocksDbAccessor.GetAllKeys(); +} + From 47f24e196c220676f5cb48d15329c9173cbfa5de Mon Sep 17 00:00:00 2001 From: Havret Date: Wed, 17 Dec 2025 17:53:57 +0100 Subject: [PATCH 18/19] wip --- .../ListOperationSerializer.cs | 83 +++++-------------- 1 file changed, 20 insertions(+), 63 deletions(-) diff --git a/src/RocksDb.Extensions/ListOperationSerializer.cs b/src/RocksDb.Extensions/ListOperationSerializer.cs index f8d0c60..fcbe3ad 100644 --- a/src/RocksDb.Extensions/ListOperationSerializer.cs +++ b/src/RocksDb.Extensions/ListOperationSerializer.cs @@ -9,66 +9,41 @@ namespace RocksDb.Extensions; /// /// The serialized format consists of: /// - 1 byte: Operation type (0 = Add, 1 = Remove) -/// - 4 bytes: Number of items -/// - For each item: -/// - 4 bytes: Size of the serialized item -/// - N bytes: Serialized item data +/// - Remaining bytes: Serialized list using VariableSizeListSerializer format /// internal class ListOperationSerializer : ISerializer> { - private readonly ISerializer _itemSerializer; + private readonly ISerializer> _listSerializer; public ListOperationSerializer(ISerializer itemSerializer) { - _itemSerializer = itemSerializer; + _listSerializer = new VariableSizeListSerializer(itemSerializer); } public bool TryCalculateSize(ref CollectionOperation value, out int size) { - // 1 byte for operation type + 4 bytes for count - size = sizeof(byte) + sizeof(int); - - for (int i = 0; i < value.Items.Count; i++) + // 1 byte for operation type + size of the list + size = sizeof(byte); + + var items = value.Items; + if (_listSerializer.TryCalculateSize(ref items, out var listSize)) { - var item = value.Items[i]; - if (_itemSerializer.TryCalculateSize(ref item, out var itemSize)) - { - size += sizeof(int); // size prefix for each item - size += itemSize; - } + size += listSize; + return true; } - return true; + return false; } public void WriteTo(ref CollectionOperation value, ref Span span) { - int offset = 0; - // Write operation type (1 byte) - span[offset] = (byte)value.Type; - offset += sizeof(byte); - - // Write count - var slice = span.Slice(offset, sizeof(int)); - BitConverter.TryWriteBytes(slice, value.Items.Count); - offset += sizeof(int); + span[0] = (byte)value.Type; - // Write each item with size prefix - for (int i = 0; i < value.Items.Count; i++) - { - var item = value.Items[i]; - if (_itemSerializer.TryCalculateSize(ref item, out var itemSize)) - { - slice = span.Slice(offset, sizeof(int)); - BitConverter.TryWriteBytes(slice, itemSize); - offset += sizeof(int); - - slice = span.Slice(offset, itemSize); - _itemSerializer.WriteTo(ref item, ref slice); - offset += itemSize; - } - } + // Write the list using the list serializer + var listSpan = span.Slice(sizeof(byte)); + var items = value.Items; + _listSerializer.WriteTo(ref items, ref listSpan); } public void WriteTo(ref CollectionOperation value, IBufferWriter buffer) @@ -78,30 +53,12 @@ public void WriteTo(ref CollectionOperation value, IBufferWriter buffer public CollectionOperation Deserialize(ReadOnlySpan buffer) { - int offset = 0; - // Read operation type - var operationType = (OperationType)buffer[offset]; - offset += sizeof(byte); - - // Read count - var slice = buffer.Slice(offset, sizeof(int)); - var count = BitConverter.ToInt32(slice); - offset += sizeof(int); + var operationType = (OperationType)buffer[0]; - // Read items - var items = new List(count); - for (int i = 0; i < count; i++) - { - slice = buffer.Slice(offset, sizeof(int)); - var itemSize = BitConverter.ToInt32(slice); - offset += sizeof(int); - - slice = buffer.Slice(offset, itemSize); - var item = _itemSerializer.Deserialize(slice); - items.Add(item); - offset += itemSize; - } + // Read the list using the list serializer + var listBuffer = buffer.Slice(sizeof(byte)); + var items = _listSerializer.Deserialize(listBuffer); return new CollectionOperation(operationType, items); } From 00fb7803d67184c83f5d114a5765b8e47c632091 Mon Sep 17 00:00:00 2001 From: Havret Date: Wed, 17 Dec 2025 18:03:16 +0100 Subject: [PATCH 19/19] wip --- .../ListOperationSerializer.cs | 23 +++++- .../MergeOperatorTests.cs | 75 +++++++++++++++++++ 2 files changed, 96 insertions(+), 2 deletions(-) diff --git a/src/RocksDb.Extensions/ListOperationSerializer.cs b/src/RocksDb.Extensions/ListOperationSerializer.cs index fcbe3ad..1a7953f 100644 --- a/src/RocksDb.Extensions/ListOperationSerializer.cs +++ b/src/RocksDb.Extensions/ListOperationSerializer.cs @@ -7,9 +7,24 @@ namespace RocksDb.Extensions; /// Serializes CollectionOperation<T> which contains an operation type (Add/Remove) and a list of items. /// /// +/// /// The serialized format consists of: /// - 1 byte: Operation type (0 = Add, 1 = Remove) -/// - Remaining bytes: Serialized list using VariableSizeListSerializer format +/// - Remaining bytes: Serialized list using FixedSizeListSerializer (for primitives) or VariableSizeListSerializer (for complex types) +/// +/// +/// Space efficiency optimization: +/// - For primitive types (int, long, bool, etc.), uses FixedSizeListSerializer which stores: +/// - 4 bytes: list count +/// - N * elementSize bytes: all elements (no per-element size prefix) +/// Example: List<int> with 3 elements = 4 + (3 * 4) = 16 bytes +/// +/// +/// - For non-primitive types (strings, objects, protobuf messages), uses VariableSizeListSerializer which stores: +/// - 4 bytes: list count +/// - For each element: 4 bytes size prefix + element data +/// Example: List<string> with ["ab", "cde"] = 4 + (4+2) + (4+3) = 17 bytes +/// /// internal class ListOperationSerializer : ISerializer> { @@ -17,7 +32,11 @@ internal class ListOperationSerializer : ISerializer> public ListOperationSerializer(ISerializer itemSerializer) { - _listSerializer = new VariableSizeListSerializer(itemSerializer); + // Use FixedSizeListSerializer for primitive types to avoid storing size for each element + // Use VariableSizeListSerializer for non-primitive types where elements may vary in size + _listSerializer = typeof(T).IsPrimitive + ? new FixedSizeListSerializer(itemSerializer) + : new VariableSizeListSerializer(itemSerializer); } public bool TryCalculateSize(ref CollectionOperation value, out int size) diff --git a/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs b/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs index 94cc8a5..3db2f3f 100644 --- a/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs +++ b/test/RocksDb.Extensions.Tests/MergeOperatorTests.cs @@ -1,6 +1,8 @@ using NUnit.Framework; using RocksDb.Extensions.MergeOperators; using RocksDb.Extensions.Tests.Utils; +using RocksDb.Extensions.Tests.Protos; +using RocksDb.Extensions.Protobuf; namespace RocksDb.Extensions.Tests; @@ -22,6 +24,24 @@ public void RemoveTags(string key, params string[] tags) } } +public class ScoresStore : MergeableRocksDbStore, CollectionOperation> +{ + public ScoresStore(IMergeAccessor, CollectionOperation> mergeAccessor) + : base(mergeAccessor) + { + } + + public void AddScores(string key, params int[] scores) + { + Merge(key, CollectionOperation.Add(scores)); + } + + public void RemoveScores(string key, params int[] scores) + { + Merge(key, CollectionOperation.Remove(scores)); + } +} + public class MergeOperatorTests { [Test] @@ -172,4 +192,59 @@ public void should_remove_only_first_occurrence_of_duplicate_items() Assert.That(tags[0], Is.EqualTo("tag")); Assert.That(tags[1], Is.EqualTo("tag")); } + + [Test] + public void should_add_primitive_types_to_list_using_list_merge_operator() + { + // Arrange + using var testFixture = TestFixture.Create(rockDb => + { + rockDb.AddMergeableStore, ScoresStore, CollectionOperation>( + "scores", + new ListMergeOperator()); + }); + + var store = testFixture.GetStore(); + var key = "player-1"; + + // Act + store.AddScores(key, 100, 200); + store.AddScores(key, 300); + + // Assert + Assert.That(store.TryGet(key, out var scores), Is.True); + Assert.That(scores, Is.Not.Null); + Assert.That(scores!.Count, Is.EqualTo(3)); + Assert.That(scores, Does.Contain(100)); + Assert.That(scores, Does.Contain(200)); + Assert.That(scores, Does.Contain(300)); + } + + [Test] + public void should_add_and_remove_primitive_types_using_list_merge_operator() + { + // Arrange + using var testFixture = TestFixture.Create(rockDb => + { + rockDb.AddMergeableStore, ScoresStore, CollectionOperation>( + "scores", + new ListMergeOperator()); + }); + + var store = testFixture.GetStore(); + var key = "player-1"; + + // Act - Add items, then remove some + store.AddScores(key, 100, 200, 300, 400); + store.RemoveScores(key, 200, 400); // Remove middle values + + // Assert + Assert.That(store.TryGet(key, out var scores), Is.True); + Assert.That(scores, Is.Not.Null); + Assert.That(scores!.Count, Is.EqualTo(2)); + Assert.That(scores, Does.Contain(100)); + Assert.That(scores, Does.Contain(300)); + Assert.That(scores, Does.Not.Contain(200)); + Assert.That(scores, Does.Not.Contain(400)); + } } \ No newline at end of file