From 5134ea034b9bcd6c566353bf8b6a36782d49287d Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Mon, 21 Jul 2025 16:56:05 +0300 Subject: [PATCH 01/27] updated editorconfig --- .editorconfig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.editorconfig b/.editorconfig index 802c1da..30024f0 100644 --- a/.editorconfig +++ b/.editorconfig @@ -83,6 +83,9 @@ dotnet_diagnostic.IDE0056.severity = none # simplify index operator dotnet_diagnostic.IDE0057.severity = none # use range operator dotnet_diagnostic.IDE0301.severity = none # simplify collection initialization dotnet_diagnostic.IDE0053.severity = none # expression body lambda +dotnet_diagnostic.IDE0046.severity = none # simplify if(s) - conditional operator +dotnet_diagnostic.IDE0305.severity = none # [, ...] instead of .ToArray() + # namespace declaration csharp_style_namespace_declarations = file_scoped:warning From 9dcaaf3649c0e2c5efe8d5739f09892560febf67 Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Mon, 21 Jul 2025 16:56:23 +0300 Subject: [PATCH 02/27] Changed tracking variables from int to long --- src/ArrowDbCore/ArrowDb.Serialization.cs | 2 +- src/ArrowDbCore/ArrowDb.cs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ArrowDbCore/ArrowDb.Serialization.cs b/src/ArrowDbCore/ArrowDb.Serialization.cs index 7ccb432..1eb4382 100644 --- a/src/ArrowDbCore/ArrowDb.Serialization.cs +++ b/src/ArrowDbCore/ArrowDb.Serialization.cs @@ -10,7 +10,7 @@ public partial class ArrowDb { /// If there are no pending updates, this method does nothing, otherwise it serializes the database and resets the pending updates counter /// public async Task SerializeAsync() { - if (_pendingChanges == 0) { + if (Interlocked.Read(ref _pendingChanges) == 0) { return; } try { diff --git a/src/ArrowDbCore/ArrowDb.cs b/src/ArrowDbCore/ArrowDb.cs index 6217234..5b7427c 100644 --- a/src/ArrowDbCore/ArrowDb.cs +++ b/src/ArrowDbCore/ArrowDb.cs @@ -10,12 +10,12 @@ public sealed partial class ArrowDb { /// /// Returns the number of active instances /// - public static int RunningInstances => s_runningInstances; + public static long RunningInstances => Interlocked.Read(ref s_runningInstances); /// /// Tracks the number of running instances /// - private static volatile int s_runningInstances; + private static long s_runningInstances; /// /// The backing dictionary @@ -53,12 +53,12 @@ private void OnChangeInternal(ArrowDbChangeEventArgs args) { /// /// Returns the number of pending changes (number of changes that have not been serialized) /// - public int PendingChanges => _pendingChanges; + public long PendingChanges => Interlocked.Read(ref _pendingChanges); /// /// Thread-safe pending changes tracker /// - private volatile int _pendingChanges; + private long _pendingChanges; /// /// Private Ctor From e2f8ccbeb8b41562f70798a6783c41ff2ee5539e Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Mon, 21 Jul 2025 16:56:32 +0300 Subject: [PATCH 03/27] Updated version and changelog --- CHANGELOG.md | 4 ++++ src/ArrowDbCore/ArrowDbCore.csproj | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70a9713..253c8e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog (Sorted by Date in Descending Order) +## 1.4.0.1 + +* Thread-safe counters types were changed from `int` to `long`, this includes `PendingChanges` and `RunningInstances`. + ## 1.4.0.0 * `GetOrAddAsync` and `Upsert` (which has the `updateCondition` argument) now both have overloads that accept a `TArg` parameter as well as a modified factory function that can use it, in order to avoid closures. diff --git a/src/ArrowDbCore/ArrowDbCore.csproj b/src/ArrowDbCore/ArrowDbCore.csproj index 384fff5..a211ab9 100644 --- a/src/ArrowDbCore/ArrowDbCore.csproj +++ b/src/ArrowDbCore/ArrowDbCore.csproj @@ -4,7 +4,7 @@ net9.0 enable enable - 1.4.0.0 + 1.4.0.1 true true From 9d2a70ff6310fb5ec759e489e5a5a0979e783a45 Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Tue, 22 Jul 2025 12:17:55 +0300 Subject: [PATCH 04/27] Updated ArrowDbTransactionScope --- CHANGELOG.md | 1 + src/ArrowDbCore/ArrowDb.cs | 5 +++++ src/ArrowDbCore/ArrowDbTransactionScope.cs | 12 ++++++++++-- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 253c8e0..63650f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## 1.4.0.1 * Thread-safe counters types were changed from `int` to `long`, this includes `PendingChanges` and `RunningInstances`. +* `ArrowDbTransactionScope` was updated to allow nested transaction, and prevent corruption that can be caused by multiple transactions running concurrently on the same `ArrowDb` instance. ## 1.4.0.0 diff --git a/src/ArrowDbCore/ArrowDb.cs b/src/ArrowDbCore/ArrowDb.cs index 5b7427c..5baee04 100644 --- a/src/ArrowDbCore/ArrowDb.cs +++ b/src/ArrowDbCore/ArrowDb.cs @@ -60,6 +60,11 @@ private void OnChangeInternal(ArrowDbChangeEventArgs args) { /// private long _pendingChanges; + /// + /// Thread-safe transaction depth tracker + /// + internal long TransactionDepth = 0; + /// /// Private Ctor /// diff --git a/src/ArrowDbCore/ArrowDbTransactionScope.cs b/src/ArrowDbCore/ArrowDbTransactionScope.cs index b854eeb..3966836 100644 --- a/src/ArrowDbCore/ArrowDbTransactionScope.cs +++ b/src/ArrowDbCore/ArrowDbTransactionScope.cs @@ -6,6 +6,7 @@ namespace ArrowDbCore; /// internal sealed class ArrowDbTransactionScope : IAsyncDisposable { private readonly ArrowDb _database; + private bool _disposed; /// /// Initializes a new instance of the class. @@ -13,12 +14,19 @@ internal sealed class ArrowDbTransactionScope : IAsyncDisposable { /// The database instance public ArrowDbTransactionScope(ArrowDb database) { _database = database; + Interlocked.Increment(ref _database.TransactionDepth); } /// /// Disposes the scope and calls /// - public async ValueTask DisposeAsync() { - await _database.SerializeAsync(); + public async ValueTask DisposeAsync() { + if (_disposed) { + return; + } + if (Interlocked.Decrement(ref _database.TransactionDepth) == 0) { + await _database.SerializeAsync().ConfigureAwait(false); + } + _disposed = true; } } From ea427ddb31aaf8e6a5ab4f7dc0396e704a7aaed9 Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Tue, 22 Jul 2025 14:32:04 +0300 Subject: [PATCH 05/27] Removed sourcelink and updated version --- src/ArrowDbCore/ArrowDbCore.csproj | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/ArrowDbCore/ArrowDbCore.csproj b/src/ArrowDbCore/ArrowDbCore.csproj index a211ab9..3d15eb7 100644 --- a/src/ArrowDbCore/ArrowDbCore.csproj +++ b/src/ArrowDbCore/ArrowDbCore.csproj @@ -4,7 +4,7 @@ net9.0 enable enable - 1.4.0.1 + 1.5.0.0 true true @@ -32,10 +32,6 @@ - - - - <_Parameter1>ArrowDbCore.Tests.Unit From 23fd4638fbec8208b70491b06b7b6491107abf2f Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Tue, 22 Jul 2025 14:32:19 +0300 Subject: [PATCH 06/27] Updated changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63650f7..847b193 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,9 @@ # Changelog (Sorted by Date in Descending Order) -## 1.4.0.1 +## 1.5.0.0 +* File based serializers `FileSerializer` and `AesFileSerializer` now use a new base class implementation and have gained the ability to `journal` (maintain durability through crashes and other `IOException`, and ensure successful atomic write or complete rejection of changes), and cross-process isolation, preventing race condition that could be caused when multiple processes try to access the same `ArrowDb` file. + * If you had a class implementing `FileSerializer` this change may or may not break functionality and you should run tests to ensure everything still works as expected (With that said, my tests were not broken and did not require any adjusting). * Thread-safe counters types were changed from `int` to `long`, this includes `PendingChanges` and `RunningInstances`. * `ArrowDbTransactionScope` was updated to allow nested transaction, and prevent corruption that can be caused by multiple transactions running concurrently on the same `ArrowDb` instance. From 4e8762b15f9711f3e5ab1b799fc23dd246270e14 Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Tue, 22 Jul 2025 14:32:32 +0300 Subject: [PATCH 07/27] Added SHA256 conversion extension --- src/ArrowDbCore/Extensions.cs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 src/ArrowDbCore/Extensions.cs diff --git a/src/ArrowDbCore/Extensions.cs b/src/ArrowDbCore/Extensions.cs new file mode 100644 index 0000000..c17339b --- /dev/null +++ b/src/ArrowDbCore/Extensions.cs @@ -0,0 +1,24 @@ +using System.Buffers; +using System.Security.Cryptography; +using System.Text; + +namespace ArrowDbCore; + +/// +/// Extension methods +/// +internal static class Extensions { + /// + /// Converts an input into a Hex Hash in an efficient manner + /// + /// + internal static string ToSHA256Hash(string input) { + var inputLength = Encoding.UTF8.GetMaxByteCount(input.Length); + using var memOwner = MemoryPool.Shared.Rent(inputLength); + Span span = memOwner.Memory.Span; + int written = Encoding.UTF8.GetBytes(input, span); + Span hashBuffer = stackalloc byte[32]; + SHA256.HashData(span.Slice(0, written), hashBuffer); + return Convert.ToHexString(hashBuffer); + } +} \ No newline at end of file From 8fd8538c8909566e2d7a7e8c75ad24f91da3f126 Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Tue, 22 Jul 2025 14:32:56 +0300 Subject: [PATCH 08/27] Refactored serializers to add journaling and cross-process isolation --- .../Serializers/AesFileSerializer.cs | 62 ++++++---------- .../Serializers/BaseFileSerializer.cs | 73 +++++++++++++++++++ src/ArrowDbCore/Serializers/FileSerializer.cs | 37 +++------- 3 files changed, 106 insertions(+), 66 deletions(-) create mode 100644 src/ArrowDbCore/Serializers/BaseFileSerializer.cs diff --git a/src/ArrowDbCore/Serializers/AesFileSerializer.cs b/src/ArrowDbCore/Serializers/AesFileSerializer.cs index 2d68024..f75b7fd 100644 --- a/src/ArrowDbCore/Serializers/AesFileSerializer.cs +++ b/src/ArrowDbCore/Serializers/AesFileSerializer.cs @@ -6,54 +6,36 @@ namespace ArrowDbCore.Serializers; /// -/// An managed file/disk backed serializer +/// An managed file/disk backed serializer. /// -public sealed class AesFileSerializer : IDbSerializer { - /// - /// The path to the file - /// - private readonly string _path; - - /// - /// The Aes instance - /// +public sealed class AesFileSerializer : BaseFileSerializer { private readonly Aes _aes; - - /// - /// The json type info for the dictionary - /// private readonly JsonTypeInfo> _jsonTypeInfo; - /// + /// /// Initializes a new instance of the class. /// - /// The path to the file - /// The instance to use - /// The json type info for the dictionary - public AesFileSerializer(string path, Aes aes, JsonTypeInfo> jsonTypeInfo) { - _path = path; - _aes = aes; - _jsonTypeInfo = jsonTypeInfo; + /// The path to the file. + /// The instance to use. + /// The json type info for the dictionary. + public AesFileSerializer(string path, Aes aes, JsonTypeInfo> jsonTypeInfo) + : base(path) { + _aes = aes; + _jsonTypeInfo = jsonTypeInfo; } - /// - public ValueTask> DeserializeAsync() { - if (!File.Exists(_path) || new FileInfo(_path).Length == 0) { - return ValueTask.FromResult(new ConcurrentDictionary()); - } - using var fileStream = File.OpenRead(_path); - using var decryptor = _aes.CreateDecryptor(); - using var cryptoStream = new CryptoStream(fileStream, decryptor, CryptoStreamMode.Read); - var res = JsonSerializer.Deserialize(cryptoStream, _jsonTypeInfo); - return ValueTask.FromResult(res ?? new ConcurrentDictionary()); - } - - /// - public ValueTask SerializeAsync(ConcurrentDictionary data) { - using var fileStream = File.Create(_path); + /// + protected override void SerializeData(Stream stream, ConcurrentDictionary data) { using var encryptor = _aes.CreateEncryptor(); - using var cryptoStream = new CryptoStream(fileStream, encryptor, CryptoStreamMode.Write); + using var cryptoStream = new CryptoStream(stream, encryptor, CryptoStreamMode.Write); JsonSerializer.Serialize(cryptoStream, data, _jsonTypeInfo); - return ValueTask.CompletedTask; } -} \ No newline at end of file + + /// + protected override ValueTask> DeserializeData(Stream stream) { + using var decryptor = _aes.CreateDecryptor(); + using var cryptoStream = new CryptoStream(stream, decryptor, CryptoStreamMode.Read); + var res = JsonSerializer.Deserialize(cryptoStream, _jsonTypeInfo); + return ValueTask.FromResult(res ?? new ConcurrentDictionary()); + } +} diff --git a/src/ArrowDbCore/Serializers/BaseFileSerializer.cs b/src/ArrowDbCore/Serializers/BaseFileSerializer.cs new file mode 100644 index 0000000..31fe801 --- /dev/null +++ b/src/ArrowDbCore/Serializers/BaseFileSerializer.cs @@ -0,0 +1,73 @@ +using System.Collections.Concurrent; + +namespace ArrowDbCore.Serializers; + +/// +/// Provides a base implementation for file-based serializers that ensures atomic and multi-process safe writes. +/// +public abstract class BaseFileSerializer : IDbSerializer { + private readonly string _dbFilePath; + private readonly string _tempFilePath; + private readonly Mutex _mutex; + + /// + /// Initializes a new instance of the class. + /// + /// The path to the database file. + protected BaseFileSerializer(string path) { + _dbFilePath = Path.GetFullPath(path); + _tempFilePath = $"{_dbFilePath}.tmp"; + string mutexName = $"Global\\ArrowDb-{Extensions.ToSHA256Hash(_dbFilePath)}"; + _mutex = new Mutex(false, mutexName); + } + + /// + /// Finalizer to ensure the system-wide mutex is released when the serializer is garbage collected. + /// + ~BaseFileSerializer() { + _mutex.Dispose(); + } + + /// + public ValueTask> DeserializeAsync() { + if (!File.Exists(_dbFilePath) || new FileInfo(_dbFilePath).Length == 0) { + return ValueTask.FromResult(new ConcurrentDictionary()); + } + + _mutex.WaitOne(); + try { + using var fileStream = File.OpenRead(_dbFilePath); + return DeserializeData(fileStream); + } finally { + _mutex.ReleaseMutex(); + } + } + + /// + public ValueTask SerializeAsync(ConcurrentDictionary data) { + _mutex.WaitOne(); + try { + using var fileStream = File.Create(_tempFilePath); + SerializeData(fileStream, data); + File.Move(_tempFilePath, _dbFilePath, true); + } finally { + _mutex.ReleaseMutex(); + } + + return ValueTask.CompletedTask; + } + + /// + /// When overridden in a derived class, serializes the data to the provided stream. + /// + /// The stream to write the data to. + /// The data to serialize. + protected abstract void SerializeData(Stream stream, ConcurrentDictionary data); + + /// + /// When overridden in a derived class, deserializes the data from the provided stream. + /// + /// The stream to read the data from. + /// The deserialized dictionary. + protected abstract ValueTask> DeserializeData(Stream stream); +} \ No newline at end of file diff --git a/src/ArrowDbCore/Serializers/FileSerializer.cs b/src/ArrowDbCore/Serializers/FileSerializer.cs index 16cf141..390239a 100644 --- a/src/ArrowDbCore/Serializers/FileSerializer.cs +++ b/src/ArrowDbCore/Serializers/FileSerializer.cs @@ -2,47 +2,32 @@ using System.Text.Json; using System.Text.Json.Serialization.Metadata; - namespace ArrowDbCore.Serializers; /// -/// A file/disk backed serializer +/// A file/disk backed serializer using JSON. /// -public class FileSerializer : IDbSerializer { - /// - /// The path to the file - /// - private readonly string _path; - - /// - /// The json type info for the dictionary - /// +public class FileSerializer : BaseFileSerializer { private readonly JsonTypeInfo> _jsonTypeInfo; /// /// Initializes a new instance of the class. /// - /// The path to the file - /// The json type info for the dictionary - public FileSerializer(string path, JsonTypeInfo> jsonTypeInfo) { - _path = path; + /// The path to the file. + /// The json type info for the dictionary. + public FileSerializer(string path, JsonTypeInfo> jsonTypeInfo) + : base(path) { _jsonTypeInfo = jsonTypeInfo; } /// - public ValueTask> DeserializeAsync() { - if (!File.Exists(_path) || new FileInfo(_path).Length == 0) { - return ValueTask.FromResult(new ConcurrentDictionary()); - } - using var file = File.OpenRead(_path); - var result = JsonSerializer.Deserialize(file, _jsonTypeInfo) ?? new(); - return ValueTask.FromResult(result); + protected override void SerializeData(Stream stream, ConcurrentDictionary data) { + JsonSerializer.Serialize(stream, data, _jsonTypeInfo); } /// - public ValueTask SerializeAsync(ConcurrentDictionary data) { - using var file = File.Create(_path); - JsonSerializer.Serialize(file, data, _jsonTypeInfo); - return ValueTask.CompletedTask; + protected override ValueTask> DeserializeData(Stream stream) { + var result = JsonSerializer.Deserialize(stream, _jsonTypeInfo) ?? new(); + return ValueTask.FromResult(result); } } From 686d269a8fcc830e39fe6e76be4d16ffa90edc68 Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Wed, 23 Jul 2025 09:37:45 +0300 Subject: [PATCH 09/27] Updated dependencies --- .../ArrowDbCore.Benchmarks/ArrowDbCore.Benchmarks.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/ArrowDbCore.Benchmarks/ArrowDbCore.Benchmarks.csproj b/benchmarks/ArrowDbCore.Benchmarks/ArrowDbCore.Benchmarks.csproj index 7f8f33a..7fc0562 100644 --- a/benchmarks/ArrowDbCore.Benchmarks/ArrowDbCore.Benchmarks.csproj +++ b/benchmarks/ArrowDbCore.Benchmarks/ArrowDbCore.Benchmarks.csproj @@ -12,8 +12,8 @@ - - + + From d9a48bdf8f42e419e97db4c90021d96d429ae68f Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Wed, 23 Jul 2025 09:38:05 +0300 Subject: [PATCH 10/27] Added benchmark project to compare versions --- ArrowDbCore.slnx | 1 + ...DbCore.Benchmarks.VersionComparison.csproj | 19 +++++++ .../JContext.cs | 7 +++ .../Person.cs | 21 ++++++++ .../Program.cs | 11 ++++ .../RandomOperationsBenchmark.cs | 51 +++++++++++++++++++ .../SerializationToFileBenchmark.cs | 46 +++++++++++++++++ .../VersionComparisonConfig.cs | 26 ++++++++++ src/ArrowDbCore/ArrowDbCore.csproj | 2 +- 9 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 benchmarks/ArrowDbCore.Benchmarks.VersionComparison/ArrowDbCore.Benchmarks.VersionComparison.csproj create mode 100644 benchmarks/ArrowDbCore.Benchmarks.VersionComparison/JContext.cs create mode 100644 benchmarks/ArrowDbCore.Benchmarks.VersionComparison/Person.cs create mode 100644 benchmarks/ArrowDbCore.Benchmarks.VersionComparison/Program.cs create mode 100644 benchmarks/ArrowDbCore.Benchmarks.VersionComparison/RandomOperationsBenchmark.cs create mode 100644 benchmarks/ArrowDbCore.Benchmarks.VersionComparison/SerializationToFileBenchmark.cs create mode 100644 benchmarks/ArrowDbCore.Benchmarks.VersionComparison/VersionComparisonConfig.cs diff --git a/ArrowDbCore.slnx b/ArrowDbCore.slnx index afa3d27..a54098c 100644 --- a/ArrowDbCore.slnx +++ b/ArrowDbCore.slnx @@ -1,5 +1,6 @@ + diff --git a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/ArrowDbCore.Benchmarks.VersionComparison.csproj b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/ArrowDbCore.Benchmarks.VersionComparison.csproj new file mode 100644 index 0000000..53e4e02 --- /dev/null +++ b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/ArrowDbCore.Benchmarks.VersionComparison.csproj @@ -0,0 +1,19 @@ + + + + Exe + net9.0 + enable + enable + + + + + + + + + + + + \ No newline at end of file diff --git a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/JContext.cs b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/JContext.cs new file mode 100644 index 0000000..775344b --- /dev/null +++ b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/JContext.cs @@ -0,0 +1,7 @@ +using System.Text.Json.Serialization; + +namespace ArrowDbCore.Benchmarks.VersionComparison; + +[JsonSourceGenerationOptions(WriteIndented = false, NumberHandling = JsonNumberHandling.AllowReadingFromString, UseStringEnumConverter = true)] +[JsonSerializable(typeof(Person))] +public partial class JContext : JsonSerializerContext {} \ No newline at end of file diff --git a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/Person.cs b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/Person.cs new file mode 100644 index 0000000..4bf6178 --- /dev/null +++ b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/Person.cs @@ -0,0 +1,21 @@ +using Bogus; + +namespace ArrowDbCore.Benchmarks.VersionComparison; + +public sealed class Person { + public int Id { get; set; } + public string Name { get; set; } = string.Empty; + public string Surname { get; set; } = string.Empty; + public int Age { get; set; } + + public static IEnumerable GeneratePeople(int count, Faker faker) { + for (var i = 0; i < count; i++) { + yield return new Person { + Id = i, + Name = faker.Name.FirstName(), + Surname = faker.Name.LastName(), + Age = faker.Random.Int(0, 100) + }; + } + } +} \ No newline at end of file diff --git a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/Program.cs b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/Program.cs new file mode 100644 index 0000000..8d6408b --- /dev/null +++ b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/Program.cs @@ -0,0 +1,11 @@ +using BenchmarkDotNet.Running; + +namespace ArrowDbCore.Benchmarks.VersionComparison; + +public class Program +{ + public static void Main(string[] args) + { + BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, new VersionComparisonConfig()); + } +} \ No newline at end of file diff --git a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/RandomOperationsBenchmark.cs b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/RandomOperationsBenchmark.cs new file mode 100644 index 0000000..87e1e30 --- /dev/null +++ b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/RandomOperationsBenchmark.cs @@ -0,0 +1,51 @@ +using System.Diagnostics; +using BenchmarkDotNet.Attributes; +using Bogus; + +namespace ArrowDbCore.Benchmarks.VersionComparison; + +[MemoryDiagnoser(false)] +[RankColumn] +[MediumRunJob] +public class RandomOperationsBenchmarks { + private Person[] _items = []; + private ArrowDb _db = default!; + + [Params(100, 10_000, 1_000_000)] + public int Count { get; set; } + + [IterationSetup] + public void Setup() { + var faker = new Faker { + Random = new Randomizer(1337) + }; + + _items = Person.GeneratePeople(Count, faker).ToArray(); + + Trace.Assert(_items.Length == Count); + + _db = ArrowDb.CreateInMemory().GetAwaiter().GetResult(); + } + + [Benchmark] + public void RandomOperations() { + Parallel.For(0, Count, i => { + // Pick a random operation: 0 = add/update, 1 = remove + int operationType = Random.Shared.Next(0, 2); + + var item = _items[i]; + + var key = item.Name; + var jsonTypeInfo = JContext.Default.Person; + + switch (operationType) { + case 0: // Add/Update + _db.Upsert(key, item, jsonTypeInfo); + break; + case 1: // Remove + _db.TryRemove(key); + break; + } + }); + } +} diff --git a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/SerializationToFileBenchmark.cs b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/SerializationToFileBenchmark.cs new file mode 100644 index 0000000..d81250c --- /dev/null +++ b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/SerializationToFileBenchmark.cs @@ -0,0 +1,46 @@ +using System.Diagnostics; +using BenchmarkDotNet.Attributes; +using Bogus; + +namespace ArrowDbCore.Benchmarks.VersionComparison; + +[MemoryDiagnoser(false)] +[RankColumn] +[MediumRunJob] +public class SerializationToFileBenchmarks { + private ArrowDb _db = default!; + + [Params(100, 10_000, 1_000_000)] + public int Size { get; set; } + + [IterationSetup] + public void Setup() { + var faker = new Faker { + Random = new Randomizer(1337) + }; + + _db = ArrowDb.CreateFromFile("test.db").GetAwaiter().GetResult(); + + Span buffer = stackalloc char[64]; + + foreach (var person in Person.GeneratePeople(Size, faker)) { + _ = person.Id.TryFormat(buffer, out var written); + var id = buffer.Slice(0, written); + _db.Upsert(id, person, JContext.Default.Person); + } + + Trace.Assert(_db.Count == Size); + } + + [IterationCleanup] + public void Cleanup() { + if (File.Exists("test.db")) { + File.Delete("test.db"); + } + } + + [Benchmark] + public async Task SerializeAsync() { + await _db.SerializeAsync(); + } +} diff --git a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/VersionComparisonConfig.cs b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/VersionComparisonConfig.cs new file mode 100644 index 0000000..36bdb6c --- /dev/null +++ b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/VersionComparisonConfig.cs @@ -0,0 +1,26 @@ +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Toolchains.CsProj; + +namespace ArrowDbCore.Benchmarks.VersionComparison; + +public class VersionComparisonConfig : ManualConfig +{ + public VersionComparisonConfig() + { + SummaryStyle = SummaryStyle.Default.WithRatioStyle(RatioStyle.Trend); + + AddJob(Job.Default + .WithToolchain( + CsProjCoreToolchain.FromProject( + "../../src/ArrowDbCore/ArrowDbCore.csproj")) + .WithId("Current")); + + AddJob(Job.Default + .WithNuGet("ArrowDb", "1.4.0") + .WithBaseline(true) + .WithId("Stable")); + } +} \ No newline at end of file diff --git a/src/ArrowDbCore/ArrowDbCore.csproj b/src/ArrowDbCore/ArrowDbCore.csproj index 3d15eb7..8f78988 100644 --- a/src/ArrowDbCore/ArrowDbCore.csproj +++ b/src/ArrowDbCore/ArrowDbCore.csproj @@ -4,7 +4,7 @@ net9.0 enable enable - 1.5.0.0 + 1.5.0-rc.1 true true From 851ca3926c9b0953cfb13f6dd656c594184bb009 Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Wed, 23 Jul 2025 10:20:12 +0300 Subject: [PATCH 11/27] Removed unused config --- benchmarks/ArrowDbCore.Benchmarks/TrentRatioConfig.cs | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 benchmarks/ArrowDbCore.Benchmarks/TrentRatioConfig.cs diff --git a/benchmarks/ArrowDbCore.Benchmarks/TrentRatioConfig.cs b/benchmarks/ArrowDbCore.Benchmarks/TrentRatioConfig.cs deleted file mode 100644 index c445356..0000000 --- a/benchmarks/ArrowDbCore.Benchmarks/TrentRatioConfig.cs +++ /dev/null @@ -1,10 +0,0 @@ -using BenchmarkDotNet.Columns; -using BenchmarkDotNet.Configs; - -namespace ArrowDbCore.Benchmarks; - -public class TrentRatioConfig : ManualConfig { - public TrentRatioConfig() { - SummaryStyle = BenchmarkDotNet.Reports.SummaryStyle.Default.WithRatioStyle(RatioStyle.Trend); - } -} \ No newline at end of file From 1e51d6aa4bf2aea7d9c292b5e93a76e8d6d9f28c Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Wed, 23 Jul 2025 10:20:35 +0300 Subject: [PATCH 12/27] Added commons project and refactored benchmarks --- ArrowDbCore.slnx | 1 + .../ArrowDbCore.Benchmarks.Common.csproj | 13 ++++++++++++ .../JContext.cs | 2 +- .../Person.cs | 2 +- ...DbCore.Benchmarks.VersionComparison.csproj | 1 + .../JContext.cs | 7 ------- .../Person.cs | 21 ------------------- .../Program.cs | 18 +++++++++------- .../RandomOperationsBenchmark.cs | 4 +++- .../SerializationToFileBenchmark.cs | 2 ++ .../VersionComparisonConfig.cs | 6 ++---- .../ArrowDbCore.Benchmarks.csproj | 1 + ...andomOperationsBenchmarks-report-github.md | 17 +++++++++++++++ .../RandomOperationsBenchmark.cs | 2 ++ .../SerializationToFileBenchmark.cs | 2 ++ 15 files changed, 56 insertions(+), 43 deletions(-) create mode 100644 benchmarks/ArrowDbCore.Benchmarks.Common/ArrowDbCore.Benchmarks.Common.csproj rename benchmarks/{ArrowDbCore.Benchmarks => ArrowDbCore.Benchmarks.Common}/JContext.cs (87%) rename benchmarks/{ArrowDbCore.Benchmarks => ArrowDbCore.Benchmarks.Common}/Person.cs (92%) delete mode 100644 benchmarks/ArrowDbCore.Benchmarks.VersionComparison/JContext.cs delete mode 100644 benchmarks/ArrowDbCore.Benchmarks.VersionComparison/Person.cs create mode 100644 benchmarks/ArrowDbCore.Benchmarks/BenchmarkDotNet.Artifacts/results/ArrowDbCore.Benchmarks.RandomOperationsBenchmarks-report-github.md diff --git a/ArrowDbCore.slnx b/ArrowDbCore.slnx index a54098c..87413d9 100644 --- a/ArrowDbCore.slnx +++ b/ArrowDbCore.slnx @@ -1,5 +1,6 @@ + diff --git a/benchmarks/ArrowDbCore.Benchmarks.Common/ArrowDbCore.Benchmarks.Common.csproj b/benchmarks/ArrowDbCore.Benchmarks.Common/ArrowDbCore.Benchmarks.Common.csproj new file mode 100644 index 0000000..d9ff41a --- /dev/null +++ b/benchmarks/ArrowDbCore.Benchmarks.Common/ArrowDbCore.Benchmarks.Common.csproj @@ -0,0 +1,13 @@ + + + + net9.0 + enable + enable + + + + + + + diff --git a/benchmarks/ArrowDbCore.Benchmarks/JContext.cs b/benchmarks/ArrowDbCore.Benchmarks.Common/JContext.cs similarity index 87% rename from benchmarks/ArrowDbCore.Benchmarks/JContext.cs rename to benchmarks/ArrowDbCore.Benchmarks.Common/JContext.cs index 697addd..34ac477 100644 --- a/benchmarks/ArrowDbCore.Benchmarks/JContext.cs +++ b/benchmarks/ArrowDbCore.Benchmarks.Common/JContext.cs @@ -1,6 +1,6 @@ using System.Text.Json.Serialization; -namespace ArrowDbCore.Benchmarks; +namespace ArrowDbCore.Benchmarks.Common; [JsonSourceGenerationOptions(WriteIndented = false, NumberHandling = JsonNumberHandling.AllowReadingFromString, UseStringEnumConverter = true)] [JsonSerializable(typeof(Person))] diff --git a/benchmarks/ArrowDbCore.Benchmarks/Person.cs b/benchmarks/ArrowDbCore.Benchmarks.Common/Person.cs similarity index 92% rename from benchmarks/ArrowDbCore.Benchmarks/Person.cs rename to benchmarks/ArrowDbCore.Benchmarks.Common/Person.cs index 09d9c68..2e4f963 100644 --- a/benchmarks/ArrowDbCore.Benchmarks/Person.cs +++ b/benchmarks/ArrowDbCore.Benchmarks.Common/Person.cs @@ -1,6 +1,6 @@ using Bogus; -namespace ArrowDbCore.Benchmarks; +namespace ArrowDbCore.Benchmarks.Common; public sealed class Person { public int Id { get; set; } diff --git a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/ArrowDbCore.Benchmarks.VersionComparison.csproj b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/ArrowDbCore.Benchmarks.VersionComparison.csproj index 53e4e02..972e31a 100644 --- a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/ArrowDbCore.Benchmarks.VersionComparison.csproj +++ b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/ArrowDbCore.Benchmarks.VersionComparison.csproj @@ -14,6 +14,7 @@ + \ No newline at end of file diff --git a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/JContext.cs b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/JContext.cs deleted file mode 100644 index 775344b..0000000 --- a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/JContext.cs +++ /dev/null @@ -1,7 +0,0 @@ -using System.Text.Json.Serialization; - -namespace ArrowDbCore.Benchmarks.VersionComparison; - -[JsonSourceGenerationOptions(WriteIndented = false, NumberHandling = JsonNumberHandling.AllowReadingFromString, UseStringEnumConverter = true)] -[JsonSerializable(typeof(Person))] -public partial class JContext : JsonSerializerContext {} \ No newline at end of file diff --git a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/Person.cs b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/Person.cs deleted file mode 100644 index 4bf6178..0000000 --- a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/Person.cs +++ /dev/null @@ -1,21 +0,0 @@ -using Bogus; - -namespace ArrowDbCore.Benchmarks.VersionComparison; - -public sealed class Person { - public int Id { get; set; } - public string Name { get; set; } = string.Empty; - public string Surname { get; set; } = string.Empty; - public int Age { get; set; } - - public static IEnumerable GeneratePeople(int count, Faker faker) { - for (var i = 0; i < count; i++) { - yield return new Person { - Id = i, - Name = faker.Name.FirstName(), - Surname = faker.Name.LastName(), - Age = faker.Random.Int(0, 100) - }; - } - } -} \ No newline at end of file diff --git a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/Program.cs b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/Program.cs index 8d6408b..e4edc63 100644 --- a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/Program.cs +++ b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/Program.cs @@ -1,11 +1,13 @@ +using ArrowDbCore.Benchmarks.VersionComparison; + using BenchmarkDotNet.Running; -namespace ArrowDbCore.Benchmarks.VersionComparison; +BenchmarkRunner.Run(); -public class Program -{ - public static void Main(string[] args) - { - BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, new VersionComparisonConfig()); - } -} \ No newline at end of file +// public class Program +// { +// public static void Main(string[] args) +// { +// BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, new VersionComparisonConfig()); +// } +// } \ No newline at end of file diff --git a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/RandomOperationsBenchmark.cs b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/RandomOperationsBenchmark.cs index 87e1e30..0b2a423 100644 --- a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/RandomOperationsBenchmark.cs +++ b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/RandomOperationsBenchmark.cs @@ -1,12 +1,14 @@ using System.Diagnostics; using BenchmarkDotNet.Attributes; using Bogus; +using ArrowDbCore.Benchmarks.Common; +using Person = ArrowDbCore.Benchmarks.Common.Person; namespace ArrowDbCore.Benchmarks.VersionComparison; [MemoryDiagnoser(false)] [RankColumn] -[MediumRunJob] +[Config(typeof(VersionComparisonConfig))] public class RandomOperationsBenchmarks { private Person[] _items = []; private ArrowDb _db = default!; diff --git a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/SerializationToFileBenchmark.cs b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/SerializationToFileBenchmark.cs index d81250c..1a79316 100644 --- a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/SerializationToFileBenchmark.cs +++ b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/SerializationToFileBenchmark.cs @@ -1,6 +1,8 @@ using System.Diagnostics; using BenchmarkDotNet.Attributes; using Bogus; +using ArrowDbCore.Benchmarks.Common; +using Person = ArrowDbCore.Benchmarks.Common.Person; namespace ArrowDbCore.Benchmarks.VersionComparison; diff --git a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/VersionComparisonConfig.cs b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/VersionComparisonConfig.cs index 36bdb6c..60435d6 100644 --- a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/VersionComparisonConfig.cs +++ b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/VersionComparisonConfig.cs @@ -13,9 +13,7 @@ public VersionComparisonConfig() SummaryStyle = SummaryStyle.Default.WithRatioStyle(RatioStyle.Trend); AddJob(Job.Default - .WithToolchain( - CsProjCoreToolchain.FromProject( - "../../src/ArrowDbCore/ArrowDbCore.csproj")) + .WithToolchain(CsProjCoreToolchain.NetCoreApp90) .WithId("Current")); AddJob(Job.Default @@ -23,4 +21,4 @@ public VersionComparisonConfig() .WithBaseline(true) .WithId("Stable")); } -} \ No newline at end of file +} diff --git a/benchmarks/ArrowDbCore.Benchmarks/ArrowDbCore.Benchmarks.csproj b/benchmarks/ArrowDbCore.Benchmarks/ArrowDbCore.Benchmarks.csproj index 7fc0562..f3944d8 100644 --- a/benchmarks/ArrowDbCore.Benchmarks/ArrowDbCore.Benchmarks.csproj +++ b/benchmarks/ArrowDbCore.Benchmarks/ArrowDbCore.Benchmarks.csproj @@ -9,6 +9,7 @@ + diff --git a/benchmarks/ArrowDbCore.Benchmarks/BenchmarkDotNet.Artifacts/results/ArrowDbCore.Benchmarks.RandomOperationsBenchmarks-report-github.md b/benchmarks/ArrowDbCore.Benchmarks/BenchmarkDotNet.Artifacts/results/ArrowDbCore.Benchmarks.RandomOperationsBenchmarks-report-github.md new file mode 100644 index 0000000..464b29e --- /dev/null +++ b/benchmarks/ArrowDbCore.Benchmarks/BenchmarkDotNet.Artifacts/results/ArrowDbCore.Benchmarks.RandomOperationsBenchmarks-report-github.md @@ -0,0 +1,17 @@ +``` + +BenchmarkDotNet v0.15.2, macOS Sequoia 15.5 (24F74) [Darwin 24.5.0] +Apple M2 Pro, 1 CPU, 10 logical and 10 physical cores +.NET SDK 9.0.303 + [Host] : .NET 9.0.7 (9.0.725.31616), Arm64 RyuJIT AdvSIMD + MediumRun : .NET 9.0.7 (9.0.725.31616), Arm64 RyuJIT AdvSIMD + +Job=MediumRun InvocationCount=1 IterationCount=15 +LaunchCount=2 UnrollFactor=1 WarmupCount=10 + +``` +| Method | Count | Mean | Error | StdDev | Rank | Allocated | +|----------------- |-------- |--------------:|-------------:|-------------:|-----:|------------:| +| **RandomOperations** | **100** | **41.75 μs** | **5.523 μs** | **8.096 μs** | **1** | **15.11 KB** | +| **RandomOperations** | **10000** | **1,779.51 μs** | **484.766 μs** | **679.575 μs** | **2** | **722.35 KB** | +| **RandomOperations** | **1000000** | **101,016.96 μs** | **1,921.740 μs** | **2,876.370 μs** | **3** | **53529.98 KB** | diff --git a/benchmarks/ArrowDbCore.Benchmarks/RandomOperationsBenchmark.cs b/benchmarks/ArrowDbCore.Benchmarks/RandomOperationsBenchmark.cs index 0a01ffd..51837ec 100644 --- a/benchmarks/ArrowDbCore.Benchmarks/RandomOperationsBenchmark.cs +++ b/benchmarks/ArrowDbCore.Benchmarks/RandomOperationsBenchmark.cs @@ -1,6 +1,8 @@ using System.Diagnostics; using BenchmarkDotNet.Attributes; using Bogus; +using ArrowDbCore.Benchmarks.Common; +using Person = ArrowDbCore.Benchmarks.Common.Person; namespace ArrowDbCore.Benchmarks; diff --git a/benchmarks/ArrowDbCore.Benchmarks/SerializationToFileBenchmark.cs b/benchmarks/ArrowDbCore.Benchmarks/SerializationToFileBenchmark.cs index 274b640..91872cf 100644 --- a/benchmarks/ArrowDbCore.Benchmarks/SerializationToFileBenchmark.cs +++ b/benchmarks/ArrowDbCore.Benchmarks/SerializationToFileBenchmark.cs @@ -1,6 +1,8 @@ using System.Diagnostics; using BenchmarkDotNet.Attributes; using Bogus; +using ArrowDbCore.Benchmarks.Common; +using Person = ArrowDbCore.Benchmarks.Common.Person; namespace ArrowDbCore.Benchmarks; From e37ca3593900346abe6ffc5435aa3cda74f8f098 Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Wed, 23 Jul 2025 11:12:09 +0300 Subject: [PATCH 13/27] updated gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index e04ec1c..07a1f13 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,7 @@ bld/ # Visual Studio 2017 auto generated files Generated\ Files/ +benchmarks/*/*.Artifacts.Results/* # MSTest test Results [Tt]est[Rr]esult*/ @@ -482,3 +483,4 @@ $RECYCLE.BIN/ # Vim temporary swap files *.swp +benchmarks/ArrowDbCore.Benchmarks.VersionComparison/BenchmarkDotNet.Artifacts/* From 22846f6cf89a80160cdd1ee675470f0205ccf70f Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Wed, 23 Jul 2025 11:12:20 +0300 Subject: [PATCH 14/27] Updated version to stable --- src/ArrowDbCore/ArrowDbCore.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ArrowDbCore/ArrowDbCore.csproj b/src/ArrowDbCore/ArrowDbCore.csproj index 8f78988..b31e880 100644 --- a/src/ArrowDbCore/ArrowDbCore.csproj +++ b/src/ArrowDbCore/ArrowDbCore.csproj @@ -4,7 +4,7 @@ net9.0 enable enable - 1.5.0-rc.1 + 1.5.0 true true From 43eca1bf3350d4f4b525873fd0a5b3aca97305fe Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Wed, 23 Jul 2025 11:12:53 +0300 Subject: [PATCH 15/27] Fixed benchmarks to correctly compare nuget versions dynamically --- ...DbCore.Benchmarks.VersionComparison.csproj | 7 +- .../Program.cs | 10 +-- .../SerializationToFileBenchmark.cs | 2 +- .../VersionComparisonConfig.cs | 67 +++++++++++++++---- 4 files changed, 63 insertions(+), 23 deletions(-) diff --git a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/ArrowDbCore.Benchmarks.VersionComparison.csproj b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/ArrowDbCore.Benchmarks.VersionComparison.csproj index 972e31a..067aa97 100644 --- a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/ArrowDbCore.Benchmarks.VersionComparison.csproj +++ b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/ArrowDbCore.Benchmarks.VersionComparison.csproj @@ -10,10 +10,15 @@ + + + + + + - diff --git a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/Program.cs b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/Program.cs index e4edc63..485961f 100644 --- a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/Program.cs +++ b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/Program.cs @@ -2,12 +2,4 @@ using BenchmarkDotNet.Running; -BenchmarkRunner.Run(); - -// public class Program -// { -// public static void Main(string[] args) -// { -// BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, new VersionComparisonConfig()); -// } -// } \ No newline at end of file +BenchmarkRunner.Run(); \ No newline at end of file diff --git a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/SerializationToFileBenchmark.cs b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/SerializationToFileBenchmark.cs index 1a79316..da9b149 100644 --- a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/SerializationToFileBenchmark.cs +++ b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/SerializationToFileBenchmark.cs @@ -8,7 +8,7 @@ namespace ArrowDbCore.Benchmarks.VersionComparison; [MemoryDiagnoser(false)] [RankColumn] -[MediumRunJob] +[Config(typeof(VersionComparisonConfig))] public class SerializationToFileBenchmarks { private ArrowDb _db = default!; diff --git a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/VersionComparisonConfig.cs b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/VersionComparisonConfig.cs index 60435d6..c13e532 100644 --- a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/VersionComparisonConfig.cs +++ b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/VersionComparisonConfig.cs @@ -2,23 +2,66 @@ using BenchmarkDotNet.Configs; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Reports; -using BenchmarkDotNet.Toolchains.CsProj; + +using NuGet.Common; + +using NuGet.Protocol; + +using NuGet.Protocol.Core.Types; + +using NuGet.Versioning; namespace ArrowDbCore.Benchmarks.VersionComparison; -public class VersionComparisonConfig : ManualConfig -{ - public VersionComparisonConfig() - { - SummaryStyle = SummaryStyle.Default.WithRatioStyle(RatioStyle.Trend); +public class VersionComparisonConfig : ManualConfig { + public const string PackageId = "ArrowDb"; + + public VersionComparisonConfig() { + var (stable, latest) = GetLatestVersionsAsync(PackageId) + .GetAwaiter() + .GetResult(); - AddJob(Job.Default - .WithToolchain(CsProjCoreToolchain.NetCoreApp90) - .WithId("Current")); + SummaryStyle = SummaryStyle.Default.WithRatioStyle(RatioStyle.Trend); - AddJob(Job.Default - .WithNuGet("ArrowDb", "1.4.0") + AddJob(Job.MediumRun .WithBaseline(true) - .WithId("Stable")); + .WithNuGet(PackageId, stable.ToNormalizedString()) + .WithId($"Stable-{stable.ToNormalizedString()}")); + + AddJob(Job.MediumRun + .WithNuGet(PackageId, latest.ToNormalizedString()) + .WithId($"Latest-{latest.ToNormalizedString()}")); + } + + private static async Task<(NuGetVersion stable, NuGetVersion latest)> GetLatestVersionsAsync(string packageId) + { + // Point at the official NuGet v3 API + var source = Repository.Factory.GetCoreV3("https://api.nuget.org/v3/index.json"); + var metaResource = await source.GetResourceAsync(); + + // Fetch all versions (incl. prerelease) and filter out unlisted packages + var allMetadata = await metaResource.GetMetadataAsync( + packageId, + includePrerelease: true, + includeUnlisted: false, + sourceCacheContext: new SourceCacheContext(), + log: NullLogger.Instance, + token: CancellationToken.None); + + // Extract distinct versions + var versions = allMetadata + .Select(meta => meta.Identity.Version) + .Distinct() + .OrderBy(v => v) // ascending + .ToList(); + + // Highest overall version (could be prerelease) + var latest = versions.Last(); + + // Highest *stable* (no prerelease); if none, fall back to latest + var stableVersions = versions.Where(v => !v.IsPrerelease).ToList(); + var stable = stableVersions.Any() ? stableVersions.Last() : latest; + + return (stable, latest); } } From 1f2c6489a76e90a088f4f7972266ac29aa4f7217 Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Wed, 23 Jul 2025 11:13:00 +0300 Subject: [PATCH 16/27] Updated changelog --- CHANGELOG.md | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 847b193..d523ce6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,31 +2,36 @@ ## 1.5.0.0 -* File based serializers `FileSerializer` and `AesFileSerializer` now use a new base class implementation and have gained the ability to `journal` (maintain durability through crashes and other `IOException`, and ensure successful atomic write or complete rejection of changes), and cross-process isolation, preventing race condition that could be caused when multiple processes try to access the same `ArrowDb` file. - * If you had a class implementing `FileSerializer` this change may or may not break functionality and you should run tests to ensure everything still works as expected (With that said, my tests were not broken and did not require any adjusting). -* Thread-safe counters types were changed from `int` to `long`, this includes `PendingChanges` and `RunningInstances`. -* `ArrowDbTransactionScope` was updated to allow nested transaction, and prevent corruption that can be caused by multiple transactions running concurrently on the same `ArrowDb` instance. +- File based serializers `FileSerializer` and `AesFileSerializer` now use a new base class implementation and have gained the ability to `journal` (maintain durability through crashes and other `IOException`, and ensure successful atomic write or complete rejection of changes), and cross-process isolation, preventing race condition that could be caused when multiple processes try to access the same `ArrowDb` file. + - If you had a class implementing `FileSerializer` this change may or may not break functionality and you should run tests to ensure everything still works as expected (With that said, my tests were not broken and did not require any adjusting). +- Thread-safe counters types were changed from `int` to `long`, this includes `PendingChanges` and `RunningInstances`. +- `ArrowDbTransactionScope` was updated to allow nested transaction, and prevent corruption that can be caused by multiple transactions running concurrently on the same `ArrowDb` instance. + +### Perf Improvements + +- Random queued (Non serialized) operations are up to 20% faster due to improvement in cross-threaded state management. +- Serialization allocates up to 250% less memory 🔥 across all benchmarks. ## 1.4.0.0 -* `GetOrAddAsync` and `Upsert` (which has the `updateCondition` argument) now both have overloads that accept a `TArg` parameter as well as a modified factory function that can use it, in order to avoid closures. +- `GetOrAddAsync` and `Upsert` (which has the `updateCondition` argument) now both have overloads that accept a `TArg` parameter as well as a modified factory function that can use it, in order to avoid closures. ## 1.3.0.0 -* Added overloads of `Upsert` that accept a `string` key, while the `ReadOnlySpan` overloads are amazing in specific cases where its use can prevent a string allocation for the lookup, in other places where the input was originally a `string` that was implicitly converted to a `ReadOnlySpan` for the parameter, this would've caused a copy to be allocated for the key when the key did not exist. The same scenario will now use the `string` overload and use it for the key directly, avoiding the intermediate copy. -* Added a `ValueTask` based `GetOrAddAsync` method, commonly used in caching scenarios. +- Added overloads of `Upsert` that accept a `string` key, while the `ReadOnlySpan` overloads are amazing in specific cases where its use can prevent a string allocation for the lookup, in other places where the input was originally a `string` that was implicitly converted to a `ReadOnlySpan` for the parameter, this would've caused a copy to be allocated for the key when the key did not exist. The same scenario will now use the `string` overload and use it for the key directly, avoiding the intermediate copy. +- Added a `ValueTask` based `GetOrAddAsync` method, commonly used in caching scenarios. ## 1.2.0.0 -* An overload to `Upsert` without `updateCondition` was added and would now act as default path in case `updateCondition` wasn't specified, this should further optimize such cases by removing condition checks and another reference from the stack during runtime. -* Internal methods which are rather small and frequently invoked will now be prioritized for inlining by JIT, this should slightly improve perf, especially in NativeAot. -* Added a new factory initializer `CreateFromFileWithAes` that received an `Aes` instance as parameter. It will then use it to encrypt and decrypt the output and input during serialization and deserialization respectively. +- An overload to `Upsert` without `updateCondition` was added and would now act as default path in case `updateCondition` wasn't specified, this should further optimize such cases by removing condition checks and another reference from the stack during runtime. +- Internal methods which are rather small and frequently invoked will now be prioritized for inlining by JIT, this should slightly improve perf, especially in NativeAot. +- Added a new factory initializer `CreateFromFileWithAes` that received an `Aes` instance as parameter. It will then use it to encrypt and decrypt the output and input during serialization and deserialization respectively. ## 1.1.0.0 -* Fixed issue with `FileSerializer` where serialization would write over existing file data which could create invalid tokens, causing deserialization to fail. -* Added static `ArrowDb.GenerateTypedKey` method that accepts the type of the value, specific key (identifier) and a buffer, it returns a `ReadOnlySpan` key that prefixes the type to the specific key. +- Fixed issue with `FileSerializer` where serialization would write over existing file data which could create invalid tokens, causing deserialization to fail. +- Added static `ArrowDb.GenerateTypedKey` method that accepts the type of the value, specific key (identifier) and a buffer, it returns a `ReadOnlySpan` key that prefixes the type to the specific key. ## 1.0.0.0 -* Initial Release +- Initial Release From ebd49faf54144bd1a9c82a39fcf220542990fa7a Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Wed, 23 Jul 2025 11:23:47 +0300 Subject: [PATCH 17/27] Updated readme --- README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 2bb2106..ea3c8d5 100644 --- a/README.md +++ b/README.md @@ -83,10 +83,9 @@ await db.SerializeAsync(); For tracking some ArrowDb internals the following properties are exposed: ```csharp -int ArrowDb.RunningInstances; // Number of active ArrowDb instances (static) -int db.InstanceId; // The id of this ArrowDb instance +long ArrowDb.RunningInstances; // Number of active ArrowDb instances (static) +long db.PendingChanges; // The number of pending changes (number of changes that have not been serialized) int db.Count; // The number of entities in the ArrowDb -int db.PendingChanges; // The number of pending changes (number of changes that have not been serialized) ``` For reading the data we have the following methods: @@ -278,7 +277,7 @@ public interface IDbSerializer { } ``` -The `DeserializeAsync` method is invoked to load the db, and the `SerializeAsync` method is invoked to persist the db. +The `DeserializeAsync` method is invoked to load the db, and the `SerializeAsync` method is invoked to persist the db. For custom file-based serializers, it is recommended to inherit from `BaseFileSerializer` to get atomic and multi-process safe writes out of the box. Being that they return a `ValueTask`, the implementations can be async. This means that you can even implement serializers to persist the db to a remote server, or cloud, or whatever else you want. @@ -323,7 +322,7 @@ void SomeMethod() { } // the function scope ends here, and implicitly closes the scope of the transaction ``` -Using a transaction scope ensures that `SerializeAsync` is always called, even if an `Exception` is thrown. +Using a transaction scope ensures that `SerializeAsync` is always called, even if an `Exception` is thrown. These scopes can be nested, and serialization will only occur when the outermost scope is disposed. ## Subscribing to Changes From 7d3933657aefd5c0de592d14f099087ff7288596 Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Wed, 23 Jul 2025 11:28:17 +0300 Subject: [PATCH 18/27] updated workflow to include trimming analyzer --- .github/workflows/unit-tests-matrix.yaml | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests-matrix.yaml b/.github/workflows/unit-tests-matrix.yaml index 46b89ff..37b7cc6 100644 --- a/.github/workflows/unit-tests-matrix.yaml +++ b/.github/workflows/unit-tests-matrix.yaml @@ -22,4 +22,25 @@ jobs: with: platform: ubuntu-latest dotnet-version: 9.0.x - test-project-path: tests/ArrowDbCore.Tests.Integrity/ArrowDbCore.Tests.Integrity.csproj \ No newline at end of file + test-project-path: tests/ArrowDbCore.Tests.Integrity/ArrowDbCore.Tests.Integrity.csproj + + aot-trimming: + env: + PROJECT: tests/ArrowDbCore.Tests.Analyzers/ArrowDbCore.Tests.Analyzers.csproj + + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup .NET + uses: actions/setup-dotnet@v3 + with: + dotnet-version: 9.0.x + + - name: Install dependencies + run: dotnet restore ${{ env.PROJECT }} + + - name: Build As Release + run: dotnet build ${{ env.PROJECT }} --configuration Release \ No newline at end of file From 95b70c63976a73aaaed19aea2aeede86702e3db2 Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Wed, 23 Jul 2025 11:44:52 +0300 Subject: [PATCH 19/27] Increased test coverage for new features --- tests/ArrowDbCore.Tests.Unit/Concurrency.cs | 42 ++++++++++ tests/ArrowDbCore.Tests.Unit/Transactions.cs | 80 ++++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 tests/ArrowDbCore.Tests.Unit/Concurrency.cs create mode 100644 tests/ArrowDbCore.Tests.Unit/Transactions.cs diff --git a/tests/ArrowDbCore.Tests.Unit/Concurrency.cs b/tests/ArrowDbCore.Tests.Unit/Concurrency.cs new file mode 100644 index 0000000..d8b8a35 --- /dev/null +++ b/tests/ArrowDbCore.Tests.Unit/Concurrency.cs @@ -0,0 +1,42 @@ +using System.Security.Cryptography; + +using ArrowDbCore.Tests.Common; + +namespace ArrowDbCore.Tests.Unit; + +public class Concurrency { + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task Concurrent_Writes_ShouldBe_ThreadSafe(bool useAes) { + // Arrange + var path = Path.GetTempFileName(); + using var aes = Aes.Create(); + var db = await CreateDb(path, useAes, aes); + var person = new Person { Name = "John", Age = 42, BirthDate = DateTime.UtcNow, IsMarried = false }; + var taskCount = 100; + var tasks = new Task[taskCount]; + + // Act + for (var i = 0; i < taskCount; i++) { + var key = $"key{i}"; + tasks[i] = Task.Run(() => db.Upsert(key, person, JContext.Default.Person)); + } + + await Task.WhenAll(tasks); + await db.SerializeAsync(); + + // Assert + var db2 = await CreateDb(path, useAes, aes); + Assert.Equal(taskCount, db2.Count); + File.Delete(path); + } + + private async Task CreateDb(string path, bool useAes, Aes? aes = null) { + if (useAes) { + return await ArrowDb.CreateFromFileWithAes(path, aes!); + } + + return await ArrowDb.CreateFromFile(path); + } +} diff --git a/tests/ArrowDbCore.Tests.Unit/Transactions.cs b/tests/ArrowDbCore.Tests.Unit/Transactions.cs new file mode 100644 index 0000000..ffe27a3 --- /dev/null +++ b/tests/ArrowDbCore.Tests.Unit/Transactions.cs @@ -0,0 +1,80 @@ +using System.Security.Cryptography; + +using ArrowDbCore.Tests.Common; + +namespace ArrowDbCore.Tests.Unit; + +public class Transactions { + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task NestedTransactionScope_SerializesOnce(bool useAes) { + // Arrange + var path = Path.GetTempFileName(); + using var aes = Aes.Create(); + var db = await CreateDb(path, useAes, aes); + var person = new Person { Name = "John", Age = 42, BirthDate = DateTime.UtcNow, IsMarried = false }; + + // Act + await using (var scope1 = db.BeginTransaction()) { + db.Upsert("key1", person, JContext.Default.Person); + Assert.Equal(1, db.PendingChanges); + + await using (var scope2 = db.BeginTransaction()) { + db.Upsert("key2", person, JContext.Default.Person); + Assert.Equal(2, db.PendingChanges); + + // Still shouldn't serialize + await scope2.DisposeAsync(); + var db2 = await CreateDb(path, useAes, aes); + Assert.Equal(0, db2.Count); + } + + Assert.Equal(2, db.PendingChanges); + } + + // Assert + var db3 = await CreateDb(path, useAes, aes); + Assert.Equal(2, db3.Count); + Assert.Equal(0, db3.PendingChanges); + File.Delete(path); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task RollbackAsync_RevertsChanges(bool useAes) { + // Arrange + var path = Path.GetTempFileName(); + using var aes = Aes.Create(); + var db = await CreateDb(path, useAes, aes); + var person = new Person { Name = "John", Age = 42, BirthDate = DateTime.UtcNow, IsMarried = false }; + + db.Upsert("key1", person, JContext.Default.Person); + await db.SerializeAsync(); + Assert.Equal(1, db.Count); + Assert.Equal(0, db.PendingChanges); + + // Act + db.Upsert("key2", person, JContext.Default.Person); + Assert.Equal(2, db.Count); + Assert.Equal(1, db.PendingChanges); + + await db.RollbackAsync(); + + // Assert + Assert.Equal(1, db.Count); + Assert.Equal(0, db.PendingChanges); + Assert.True(db.ContainsKey("key1")); + Assert.False(db.ContainsKey("key2")); + File.Delete(path); + } + + private async Task CreateDb(string path, bool useAes, Aes? aes = null) { + if (useAes) { + return await ArrowDb.CreateFromFileWithAes(path, aes!); + } + + return await ArrowDb.CreateFromFile(path); + } +} From c82d00b6c00737dc2bb1e752f365812a735c00c4 Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Wed, 23 Jul 2025 12:26:06 +0300 Subject: [PATCH 20/27] Updated changelog again --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d523ce6..d2821ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - If you had a class implementing `FileSerializer` this change may or may not break functionality and you should run tests to ensure everything still works as expected (With that said, my tests were not broken and did not require any adjusting). - Thread-safe counters types were changed from `int` to `long`, this includes `PendingChanges` and `RunningInstances`. - `ArrowDbTransactionScope` was updated to allow nested transaction, and prevent corruption that can be caused by multiple transactions running concurrently on the same `ArrowDb` instance. +- `Upsert` and all its overloads will now reject (return `false`) whenever the value to be upserted is a `null` reference type. This is to enforce a no `null` policy that will simplify development by eliminating `null` checks on retrieved values. ### Perf Improvements From 7c1fe5ed53cb3406e42b5bb5eee73ca91b58ee44 Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Wed, 23 Jul 2025 12:26:33 +0300 Subject: [PATCH 21/27] Enforced rejection of nulls in upsert --- src/ArrowDbCore/ArrowDb.Upsert.cs | 53 ++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/src/ArrowDbCore/ArrowDb.Upsert.cs b/src/ArrowDbCore/ArrowDb.Upsert.cs index 80cbbbb..89a2e9c 100644 --- a/src/ArrowDbCore/ArrowDb.Upsert.cs +++ b/src/ArrowDbCore/ArrowDb.Upsert.cs @@ -6,27 +6,27 @@ namespace ArrowDbCore; public partial class ArrowDb { /// - /// Upsert the specified key with the specified value into the database + /// Upsert the specified key with the specified value into the database. /// /// The type of the value to upsert /// The key at which to upsert the value - /// The value to upsert + /// The value to upsert. This cannot be null. /// The json type info for the value type - /// True + /// True if the value was upserted, false if the provided value was null. public bool Upsert(string key, TValue value, JsonTypeInfo jsonTypeInfo) { return UpsertCore(key, value, jsonTypeInfo, default); } /// - /// Upsert the specified key with the specified value into the database + /// Upsert the specified key with the specified value into the database. /// /// The type of the value to upsert /// The key at which to upsert the value - /// The value to upsert + /// The value to upsert. This cannot be null. /// The json type info for the value type - /// True + /// True if the value was upserted, false if the provided value was null. /// - /// This method overload which uses ReadOnlySpan{char} will not allocate a new string for the key if it already exists, instead it will directly replace the value + /// This method overload which uses ReadOnlySpan{char} will not allocate a new string for the key if it already exists, instead it will directly replace the value. /// public bool Upsert(ReadOnlySpan key, TValue value, JsonTypeInfo jsonTypeInfo) { return UpsertCore, TValue, ReadOnlySpanAccessor>(key, value, jsonTypeInfo, default); @@ -36,6 +36,9 @@ public bool Upsert(ReadOnlySpan key, TValue value, JsonTypeInfo(TKey key, TValue value, JsonTypeInfo jsonTypeInfo, TAccessor accessor) where TKey : allows ref struct where TAccessor : IDictionaryAccessor, allows ref struct { + if (value is null) { + return false; + } WaitIfSerializing(); // Block if serializing byte[] utf8Value = JsonSerializer.SerializeToUtf8Bytes(value, jsonTypeInfo); accessor.Upsert(this, key, utf8Value); @@ -44,16 +47,19 @@ private bool UpsertCore(TKey key, TValue value, JsonTyp } /// - /// Tries to upsert the specified key with the specified value into the database + /// Tries to upsert the specified key with the specified value into the database. /// /// The type of the value to upsert /// The key at which to upsert the value - /// The value to upsert + /// The value to upsert. This cannot be null. /// The json type info for the value type /// A conditional check that determines whether this update should be performed - /// True if the value was upserted, false otherwise + /// True if the value was upserted, false otherwise. /// /// + /// The operation will be rejected if the provided is null. + /// + /// /// can be used to resolve write conflicts, the update will be rejected only if both conditions are met: /// /// 1. A value for the specified key exists and successfully deserialized to @@ -68,18 +74,21 @@ public bool Upsert(string key, TValue value, JsonTypeInfo jsonTy } /// - /// Tries to upsert the specified key with the specified value into the database + /// Tries to upsert the specified key with the specified value into the database. /// /// The type of the value to upsert /// The type of the argument for the updateCondition function /// The key at which to upsert the value - /// The value to upsert + /// The value to upsert. This cannot be null. /// The json type info for the value type /// A conditional check that determines whether this update should be performed /// An argument that could be provided to the updateCondition function to avoid a closure - /// True if the value was upserted, false otherwise + /// True if the value was upserted, false otherwise. /// /// + /// The operation will be rejected if the provided is null. + /// + /// /// can be used to resolve write conflicts, the update will be rejected only if both conditions are met: /// /// 1. A value for the specified key exists and successfully deserialized to @@ -94,16 +103,19 @@ public bool Upsert(string key, TValue value, JsonTypeInfo } /// - /// Tries to upsert the specified key with the specified value into the database + /// Tries to upsert the specified key with the specified value into the database. /// /// The type of the value to upsert /// The key at which to upsert the value - /// The value to upsert + /// The value to upsert. This cannot be null. /// The json type info for the value type /// A conditional check that determines whether this update should be performed - /// True if the value was upserted, false otherwise + /// True if the value was upserted, false otherwise. /// /// + /// The operation will be rejected if the provided is null. + /// + /// /// can be used to resolve write conflicts, the update will be rejected only if both conditions are met: /// /// 1. A value for the specified key exists and successfully deserialized to @@ -121,18 +133,21 @@ public bool Upsert(ReadOnlySpan key, TValue value, JsonTypeInfo - /// Tries to upsert the specified key with the specified value into the database + /// Tries to upsert the specified key with the specified value into the database. /// /// The type of the value to upsert /// The type of the argument for the updateCondition function /// The key at which to upsert the value - /// The value to upsert + /// The value to upsert. This cannot be null. /// The json type info for the value type /// A conditional check that determines whether this update should be performed /// An argument that could be provided to the updateCondition function to avoid a closure - /// True if the value was upserted, false otherwise + /// True if the value was upserted, false otherwise. /// /// + /// The operation will be rejected if the provided is null. + /// + /// /// can be used to resolve write conflicts, the update will be rejected only if both conditions are met: /// /// 1. A value for the specified key exists and successfully deserialized to From 0029d99f1931feed961327154906b536f7d3d8dc Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Wed, 23 Jul 2025 12:28:06 +0300 Subject: [PATCH 22/27] Increased test coverage --- tests/ArrowDbCore.Tests.Unit/GetOrAddAsync.cs | 18 +++++++++++++++++- tests/ArrowDbCore.Tests.Unit/Removes.cs | 14 ++++++++++++++ tests/ArrowDbCore.Tests.Unit/Upserts.cs | 15 +++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/tests/ArrowDbCore.Tests.Unit/GetOrAddAsync.cs b/tests/ArrowDbCore.Tests.Unit/GetOrAddAsync.cs index ff6ddcf..55ce15e 100644 --- a/tests/ArrowDbCore.Tests.Unit/GetOrAddAsync.cs +++ b/tests/ArrowDbCore.Tests.Unit/GetOrAddAsync.cs @@ -4,7 +4,7 @@ namespace ArrowDbCore.Tests.Unit; public class GetOrAddAsync { #pragma warning disable xUnit1031 // Do not use blocking task operations in test method -// this is required here for testing purposes + // this is required here for testing purposes [Fact] public async Task GetOrAddAsync_ReturnsSynchronously_WhenExists() { var db = await ArrowDb.CreateInMemory(); @@ -63,4 +63,20 @@ public async Task GetOrAddAsync_WithTArg_ReturnsAsynchronously_WhenNotExists() { Assert.False(task.IsCompletedSuccessfully); Assert.Equal(1, await task); } + + [Fact] + public async Task GetOrAddAsync_FailingFactory_DoesNotAddItem() { + // Arrange + var db = await ArrowDb.CreateInMemory(); + + // Act & Assert + await Assert.ThrowsAsync(() => + db.GetOrAddAsync("key", JContext.Default.Int32, _ => + ValueTask.FromException(new InvalidOperationException("Factory failed")) + ).AsTask() + ); + + Assert.Equal(0, db.Count); + Assert.False(db.ContainsKey("key")); + } } \ No newline at end of file diff --git a/tests/ArrowDbCore.Tests.Unit/Removes.cs b/tests/ArrowDbCore.Tests.Unit/Removes.cs index be302b3..3bd1e29 100644 --- a/tests/ArrowDbCore.Tests.Unit/Removes.cs +++ b/tests/ArrowDbCore.Tests.Unit/Removes.cs @@ -10,6 +10,20 @@ public async Task TryRemove_When_Not_Found_Returns_False() { Assert.False(db.TryRemove("1")); } + [Fact] + public async Task TryRemove_NotFound_DoesNotIncrementPendingChanges() { + // Arrange + var db = await ArrowDb.CreateInMemory(); + Assert.Equal(0, db.PendingChanges); + + // Act + var result = db.TryRemove("non_existent_key"); + + // Assert + Assert.False(result); + Assert.Equal(0, db.PendingChanges); + } + [Fact] public async Task TryRemove_When_Found_Returns_True() { var db = await ArrowDb.CreateInMemory(); diff --git a/tests/ArrowDbCore.Tests.Unit/Upserts.cs b/tests/ArrowDbCore.Tests.Unit/Upserts.cs index bf00e89..86d1f9a 100644 --- a/tests/ArrowDbCore.Tests.Unit/Upserts.cs +++ b/tests/ArrowDbCore.Tests.Unit/Upserts.cs @@ -24,6 +24,21 @@ public async Task Upsert_When_Found_Overwrites() { Assert.Equal(2, value); } + [Fact] + public async Task Upsert_NullValue_IsDisallowed() + { + // Arrange + var db = await ArrowDb.CreateInMemory(); + + // Act + var result = db.Upsert("key", null, JContext.Default.Person); + + // Assert + Assert.False(result); + Assert.Equal(0, db.Count); + Assert.False(db.ContainsKey("key")); + } + [Fact] public async Task Conditional_Update_When_Not_Found_Inserts() { var db = await ArrowDb.CreateInMemory(); From 17fe8ddc726951bf1e593504f296b0173ace8c07 Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Wed, 23 Jul 2025 12:28:20 +0300 Subject: [PATCH 23/27] Updated readme to reflect "no null" policy --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index ea3c8d5..53bab48 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,12 @@ ArrowDb is a fast, lightweight, and type-safe key-value database designed for .N * Cross-Platform and Fully AOT-compatible * Super-Easy API near mirroring of `Dictionary` +### A Note on `null` Values + +ArrowDb enforces a "no nulls" policy by design. Attempting to `Upsert` a `null` value will be rejected and return `false`. This simplifies the developer experience by guaranteeing that if a key exists, its value is never `null`. This eliminates the need for null-checking after retrieval, leading to cleaner and more predictable application code. + +This policy does not affect value types (`structs`); their `default` values (e.g., `0` for an `int`) are considered valid and are stored correctly. + ## Getting Started Installation is done via NuGet: `dotnet add package ArrowDbCore` From 922f3168cd4801b963eb6f1c61bd0c6d27d38f5c Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Wed, 23 Jul 2025 12:30:51 +0300 Subject: [PATCH 24/27] Bridged gap between published readme files --- README.md | 2 +- src/ArrowDbCore/Readme.Nuget.md | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 53bab48..167a474 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ ArrowDb is a fast, lightweight, and type-safe key-value database designed for .N ArrowDb enforces a "no nulls" policy by design. Attempting to `Upsert` a `null` value will be rejected and return `false`. This simplifies the developer experience by guaranteeing that if a key exists, its value is never `null`. This eliminates the need for null-checking after retrieval, leading to cleaner and more predictable application code. -This policy does not affect value types (`structs`); their `default` values (e.g., `0` for an `int`) are considered valid and are stored correctly. +This policy does not affect value types (`structs`); their `default` values (e.g., `0` for an `int`) are considered valid. ## Getting Started diff --git a/src/ArrowDbCore/Readme.Nuget.md b/src/ArrowDbCore/Readme.Nuget.md index aba95d2..67b4a7d 100644 --- a/src/ArrowDbCore/Readme.Nuget.md +++ b/src/ArrowDbCore/Readme.Nuget.md @@ -11,4 +11,10 @@ A fast, lightweight, and type-safe key-value database designed for .NET. * Cross-Platform and Fully AOT-compatible * Super-Easy API near mirroring of `Dictionary` +## A Note on `null` Values + +`ArrowDb` enforces a "no `null`s" policy by design. Attempting to `Upsert` a `null` value will be rejected and return `false`. This simplifies the developer experience by guaranteeing that if a key exists, its value is never `null`. This eliminates the need for null-checking after retrieval, leading to cleaner and more predictable application code. + +This policy does not affect value types (`structs`); their `default` values (e.g., `0` for an `int`) are considered valid. + Information on usage can be found in the [README](https://github.com/dusrdev/ArrowDb/blob/stable/README.md). From bbb4f620bddd74663aaa18e00722fe7fdbb9238b Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Wed, 23 Jul 2025 12:38:48 +0300 Subject: [PATCH 25/27] Address issue with windows not disposing stream before file move --- src/ArrowDbCore/Serializers/BaseFileSerializer.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ArrowDbCore/Serializers/BaseFileSerializer.cs b/src/ArrowDbCore/Serializers/BaseFileSerializer.cs index 31fe801..cd30f52 100644 --- a/src/ArrowDbCore/Serializers/BaseFileSerializer.cs +++ b/src/ArrowDbCore/Serializers/BaseFileSerializer.cs @@ -47,8 +47,10 @@ public ValueTask> DeserializeAsync() { public ValueTask SerializeAsync(ConcurrentDictionary data) { _mutex.WaitOne(); try { - using var fileStream = File.Create(_tempFilePath); - SerializeData(fileStream, data); + using (var fileStream = File.Create(_tempFilePath)) + { + SerializeData(fileStream, data); + } File.Move(_tempFilePath, _dbFilePath, true); } finally { _mutex.ReleaseMutex(); From 58cd792945e4fc0281b2a83d5244321f04a70634 Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Wed, 23 Jul 2025 12:40:42 +0300 Subject: [PATCH 26/27] Ensure trimming warning fails the build to ensure AOT compliance --- .../ArrowDbCore.Tests.Analyzers.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ArrowDbCore.Tests.Analyzers/ArrowDbCore.Tests.Analyzers.csproj b/tests/ArrowDbCore.Tests.Analyzers/ArrowDbCore.Tests.Analyzers.csproj index 965e2df..0b9b75f 100644 --- a/tests/ArrowDbCore.Tests.Analyzers/ArrowDbCore.Tests.Analyzers.csproj +++ b/tests/ArrowDbCore.Tests.Analyzers/ArrowDbCore.Tests.Analyzers.csproj @@ -7,6 +7,7 @@ enable true true + true From b1f68fc7d1cc82b50a522b2ec85a5d6b610bf842 Mon Sep 17 00:00:00 2001 From: David Shnayder Date: Wed, 23 Jul 2025 13:13:08 +0300 Subject: [PATCH 27/27] Added IDisposable implementation to Transaction scope and updated docs and tests --- CHANGELOG.md | 2 +- README.md | 2 ++ src/ArrowDbCore/ArrowDb.cs | 7 +++++-- src/ArrowDbCore/ArrowDbTransactionScope.cs | 17 ++++++++++++++--- tests/ArrowDbCore.Tests.Unit/Serialization.cs | 17 ++++++++++++++++- tests/ArrowDbCore.Tests.Unit/Transactions.cs | 5 ++--- 6 files changed, 40 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2821ca..65885cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ - File based serializers `FileSerializer` and `AesFileSerializer` now use a new base class implementation and have gained the ability to `journal` (maintain durability through crashes and other `IOException`, and ensure successful atomic write or complete rejection of changes), and cross-process isolation, preventing race condition that could be caused when multiple processes try to access the same `ArrowDb` file. - If you had a class implementing `FileSerializer` this change may or may not break functionality and you should run tests to ensure everything still works as expected (With that said, my tests were not broken and did not require any adjusting). - Thread-safe counters types were changed from `int` to `long`, this includes `PendingChanges` and `RunningInstances`. -- `ArrowDbTransactionScope` was updated to allow nested transaction, and prevent corruption that can be caused by multiple transactions running concurrently on the same `ArrowDb` instance. +- `ArrowDbTransactionScope` was updated to allow nested transactions, and prevent corruption that can be caused by multiple transactions running concurrently on the same `ArrowDb` instance. In addition, it was made `public` and also implements the regular `IDisposable` interface to allow usage in synchronous context. However, it is still your responsibility to ensure the contexts match. - `Upsert` and all its overloads will now reject (return `false`) whenever the value to be upserted is a `null` reference type. This is to enforce a no `null` policy that will simplify development by eliminating `null` checks on retrieved values. ### Perf Improvements diff --git a/README.md b/README.md index 167a474..ca83e3d 100644 --- a/README.md +++ b/README.md @@ -330,6 +330,8 @@ void SomeMethod() { Using a transaction scope ensures that `SerializeAsync` is always called, even if an `Exception` is thrown. These scopes can be nested, and serialization will only occur when the outermost scope is disposed. +`ArrowDbTransactionScope` also implements the regular `IDisposable` interface, meaning it can be used in a non-`async` method. However it internally calls the `DisposeAsync` method in a blocking manner, with the built in file-based serializers (`FileSerializer` and `AesFileSerializer`) it is completely safe as they naturally operate synchronously. However if you implemented a remote serializer or an `async` one, you should use the `Async Disposable` pattern accordingly. + ## Subscribing to Changes `ArrowDb` exposes an `OnChange` event that is raised whenever an operation that changes the database state, i.e, adding, updating, or removing a key, or clearing the database, is performed. The event is raised with a `ArrowDbChangeEventArgs` argument that contains the type of change that occurred. diff --git a/src/ArrowDbCore/ArrowDb.cs b/src/ArrowDbCore/ArrowDb.cs index 5baee04..9d09fb3 100644 --- a/src/ArrowDbCore/ArrowDb.cs +++ b/src/ArrowDbCore/ArrowDb.cs @@ -89,6 +89,9 @@ private ArrowDb(ConcurrentDictionary source, IDbSerializer seria /// /// Returns a transaction scope that implicitly calls when disposed /// - /// IAsyncDisposable - public IAsyncDisposable BeginTransaction() => new ArrowDbTransactionScope(this); + /// + /// The implements both and , allowing it to be used in both synchronous and asynchronous contexts. + /// + /// A new instance. + public ArrowDbTransactionScope BeginTransaction() => new(this); } diff --git a/src/ArrowDbCore/ArrowDbTransactionScope.cs b/src/ArrowDbCore/ArrowDbTransactionScope.cs index 3966836..d135440 100644 --- a/src/ArrowDbCore/ArrowDbTransactionScope.cs +++ b/src/ArrowDbCore/ArrowDbTransactionScope.cs @@ -4,7 +4,7 @@ namespace ArrowDbCore; /// /// Provides a scope that can be used to defer serialization until the scope is disposed /// -internal sealed class ArrowDbTransactionScope : IAsyncDisposable { +public sealed class ArrowDbTransactionScope : IAsyncDisposable, IDisposable { private readonly ArrowDb _database; private bool _disposed; @@ -12,7 +12,7 @@ internal sealed class ArrowDbTransactionScope : IAsyncDisposable { /// Initializes a new instance of the class. /// /// The database instance - public ArrowDbTransactionScope(ArrowDb database) { + internal ArrowDbTransactionScope(ArrowDb database) { _database = database; Interlocked.Increment(ref _database.TransactionDepth); } @@ -28,5 +28,16 @@ public async ValueTask DisposeAsync() { await _database.SerializeAsync().ConfigureAwait(false); } _disposed = true; - } + } + + /// + /// Disposes the scope and calls in a blocking operation + /// + public void Dispose() { + var task = DisposeAsync(); + if (task.IsCompleted) { + return; + } + task.GetAwaiter().GetResult(); + } } diff --git a/tests/ArrowDbCore.Tests.Unit/Serialization.cs b/tests/ArrowDbCore.Tests.Unit/Serialization.cs index adaed9b..b5064bf 100644 --- a/tests/ArrowDbCore.Tests.Unit/Serialization.cs +++ b/tests/ArrowDbCore.Tests.Unit/Serialization.cs @@ -46,7 +46,7 @@ public async Task Serialize_Using_Event_Resets_Changes() { } [Fact] - public async Task DeferredSerializationScope_Serialize_After_Dispose() { + public async Task DeferredSerializationScope_SerializeAsync_After_DisposeAsync() { var db = await ArrowDb.CreateInMemory(); Assert.Equal(0, db.Count); await using (_ = db.BeginTransaction()) { @@ -60,6 +60,21 @@ public async Task DeferredSerializationScope_Serialize_After_Dispose() { Assert.Equal(0, db.PendingChanges); } + [Fact] + public async Task DeferredSerializationScope_Serialize_After_Dispose() { + var db = await ArrowDb.CreateInMemory(); + Assert.Equal(0, db.Count); + using (_ = db.BeginTransaction()) { + db.Upsert("1", 1, JContext.Default.Int32); + Assert.True(db.ContainsKey("1")); + Assert.Equal(1, db.Count); + Assert.Equal(1, db.PendingChanges); + } + Assert.True(db.ContainsKey("1")); + Assert.Equal(1, db.Count); + Assert.Equal(0, db.PendingChanges); + } + private static async Task File_Serializes_And_Deserializes_As_Expected(string path, Func> factory) { try { var db = await factory(); diff --git a/tests/ArrowDbCore.Tests.Unit/Transactions.cs b/tests/ArrowDbCore.Tests.Unit/Transactions.cs index ffe27a3..e0bbd72 100644 --- a/tests/ArrowDbCore.Tests.Unit/Transactions.cs +++ b/tests/ArrowDbCore.Tests.Unit/Transactions.cs @@ -25,10 +25,9 @@ public async Task NestedTransactionScope_SerializesOnce(bool useAes) { Assert.Equal(2, db.PendingChanges); // Still shouldn't serialize - await scope2.DisposeAsync(); - var db2 = await CreateDb(path, useAes, aes); - Assert.Equal(0, db2.Count); } + var db2 = await CreateDb(path, useAes, aes); + Assert.Equal(0, db2.Count); Assert.Equal(2, db.PendingChanges); }