Version 1.5.0.0

.editorconfig

@@ -83,6 +83,9 @@ dotnet_diagnostic.IDE0056.severity = none # simplify index operator
 dotnet_diagnostic.IDE0057.severity = none # use range operator
 dotnet_diagnostic.IDE0301.severity = none # simplify collection initialization
 dotnet_diagnostic.IDE0053.severity = none # expression body lambda
+dotnet_diagnostic.IDE0046.severity = none # simplify if(s) - conditional operator
+dotnet_diagnostic.IDE0305.severity = none # [, ...] instead of .ToArray()
+
 
 # namespace declaration
 csharp_style_namespace_declarations = file_scoped:warning

.github/workflows/unit-tests-matrix.yaml

@@ -22,4 +22,25 @@ jobs:
     with:
       platform: ubuntu-latest
       dotnet-version: 9.0.x
-      test-project-path: tests/ArrowDbCore.Tests.Integrity/ArrowDbCore.Tests.Integrity.csproj
+      test-project-path: tests/ArrowDbCore.Tests.Integrity/ArrowDbCore.Tests.Integrity.csproj
+
+  aot-trimming:
+    env:
+      PROJECT: tests/ArrowDbCore.Tests.Analyzers/ArrowDbCore.Tests.Analyzers.csproj
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: 9.0.x
+
+      - name: Install dependencies
+        run: dotnet restore ${{ env.PROJECT }}
+
+      - name: Build As Release
+        run: dotnet build ${{ env.PROJECT }} --configuration Release

.gitignore

@@ -42,6 +42,7 @@ bld/
 
 # Visual Studio 2017 auto generated files
 Generated\ Files/
+benchmarks/*/*.Artifacts.Results/*
 
 # MSTest test Results
 [Tt]est[Rr]esult*/
@@ -482,3 +483,4 @@ $RECYCLE.BIN/
 
 # Vim temporary swap files
 *.swp
+benchmarks/ArrowDbCore.Benchmarks.VersionComparison/BenchmarkDotNet.Artifacts/*

ArrowDbCore.slnx

@@ -1,5 +1,7 @@
 <Solution>
   <Folder Name="/benchmarks/">
+    <Project Path="benchmarks/ArrowDbCore.Benchmarks.Common/ArrowDbCore.Benchmarks.Common.csproj" />
+    <Project Path="benchmarks/ArrowDbCore.Benchmarks.VersionComparison/ArrowDbCore.Benchmarks.VersionComparison.csproj" />
     <Project Path="benchmarks/ArrowDbCore.Benchmarks/ArrowDbCore.Benchmarks.csproj" />
   </Folder>
   <Folder Name="/src/">

CHANGELOG.md

@@ -1,25 +1,38 @@
 # Changelog (Sorted by Date in Descending Order)
 
+## 1.5.0.0
+
+- File based serializers `FileSerializer` and `AesFileSerializer` now use a new base class implementation and have gained the ability to `journal` (maintain durability through crashes and other `IOException`, and ensure successful atomic write or complete rejection of changes), and cross-process isolation, preventing race condition that could be caused when multiple processes try to access the same `ArrowDb` file.
+  - If you had a class implementing `FileSerializer` this change may or may not break functionality and you should run tests to ensure everything still works as expected (With that said, my tests were not broken and did not require any adjusting).
+- Thread-safe counters types were changed from `int` to `long`, this includes `PendingChanges` and `RunningInstances`.
+- `ArrowDbTransactionScope` was updated to allow nested transactions, and prevent corruption that can be caused by multiple transactions running concurrently on the same `ArrowDb` instance. In addition, it was made `public` and also implements the regular `IDisposable` interface to allow usage in synchronous context. However, it is still your responsibility to ensure the contexts match.
+- `Upsert` and all its overloads will now reject (return `false`) whenever the value to be upserted is a `null` reference type. This is to enforce a no `null` policy that will simplify development by eliminating `null` checks on retrieved values.
+
+### Perf Improvements
+
+- Random queued (Non serialized) operations are up to 20% faster due to improvement in cross-threaded state management.
+- Serialization allocates up to 250% less memory 🔥 across all benchmarks.
+
 ## 1.4.0.0
 
-* `GetOrAddAsync` and `Upsert` (which has the `updateCondition` argument) now both have overloads that accept a `TArg` parameter as well as a modified factory function that can use it, in order to avoid closures.
+- `GetOrAddAsync` and `Upsert` (which has the `updateCondition` argument) now both have overloads that accept a `TArg` parameter as well as a modified factory function that can use it, in order to avoid closures.
 
 ## 1.3.0.0
 
-* Added overloads of `Upsert` that accept a `string` key, while the `ReadOnlySpan<char>` overloads are amazing in specific cases where its use can prevent a string allocation for the lookup, in other places where the input was originally a `string` that was implicitly converted to a `ReadOnlySpan<char>` for the parameter, this would've caused a copy to be allocated for the key when the key did not exist. The same scenario will now use the `string` overload and use it for the key directly, avoiding the intermediate copy.
-* Added a `ValueTask` based `GetOrAddAsync` method, commonly used in caching scenarios.
+- Added overloads of `Upsert` that accept a `string` key, while the `ReadOnlySpan<char>` overloads are amazing in specific cases where its use can prevent a string allocation for the lookup, in other places where the input was originally a `string` that was implicitly converted to a `ReadOnlySpan<char>` for the parameter, this would've caused a copy to be allocated for the key when the key did not exist. The same scenario will now use the `string` overload and use it for the key directly, avoiding the intermediate copy.
+- Added a `ValueTask` based `GetOrAddAsync` method, commonly used in caching scenarios.
 
 ## 1.2.0.0
 
-* An overload to `Upsert` without `updateCondition` was added and would now act as default path in case `updateCondition` wasn't specified, this should further optimize such cases by removing condition checks and another reference from the stack during runtime.
-* Internal methods which are rather small and frequently invoked will now be prioritized for inlining by JIT, this should slightly improve perf, especially in NativeAot.
-* Added a new factory initializer `CreateFromFileWithAes` that received an `Aes` instance as parameter. It will then use it to encrypt and decrypt the output and input during serialization and deserialization respectively.
+- An overload to `Upsert` without `updateCondition` was added and would now act as default path in case `updateCondition` wasn't specified, this should further optimize such cases by removing condition checks and another reference from the stack during runtime.
+- Internal methods which are rather small and frequently invoked will now be prioritized for inlining by JIT, this should slightly improve perf, especially in NativeAot.
+- Added a new factory initializer `CreateFromFileWithAes` that received an `Aes` instance as parameter. It will then use it to encrypt and decrypt the output and input during serialization and deserialization respectively.
 
 ## 1.1.0.0
 
-* Fixed issue with `FileSerializer` where serialization would write over existing file data which could create invalid tokens, causing deserialization to fail.
-* Added static `ArrowDb.GenerateTypedKey<T>` method that accepts the type of the value, specific key (identifier) and a buffer, it returns a `ReadOnlySpan<char>` key that prefixes the type to the specific key.
+- Fixed issue with `FileSerializer` where serialization would write over existing file data which could create invalid tokens, causing deserialization to fail.
+- Added static `ArrowDb.GenerateTypedKey<T>` method that accepts the type of the value, specific key (identifier) and a buffer, it returns a `ReadOnlySpan<char>` key that prefixes the type to the specific key.
 
 ## 1.0.0.0
 
-* Initial Release
+- Initial Release

README.md

@@ -21,6 +21,12 @@ ArrowDb is a fast, lightweight, and type-safe key-value database designed for .N
 * Cross-Platform and Fully AOT-compatible
 * Super-Easy API near mirroring of `Dictionary<TKey, TValue>`
 
+### A Note on `null` Values
+
+ArrowDb enforces a "no nulls" policy by design. Attempting to `Upsert` a `null` value will be rejected and return `false`. This simplifies the developer experience by guaranteeing that if a key exists, its value is never `null`. This eliminates the need for null-checking after retrieval, leading to cleaner and more predictable application code.
+
+This policy does not affect value types (`structs`); their `default` values (e.g., `0` for an `int`) are considered valid.
+
 ## Getting Started
 
 Installation is done via NuGet: `dotnet add package ArrowDbCore`
@@ -83,10 +89,9 @@ await db.SerializeAsync();
 For tracking some ArrowDb internals the following properties are exposed:
 
 ```csharp
-int ArrowDb.RunningInstances;  // Number of active ArrowDb instances (static)
-int db.InstanceId;              // The id of this ArrowDb instance
+long ArrowDb.RunningInstances;  // Number of active ArrowDb instances (static)
+long db.PendingChanges;          // The number of pending changes (number of changes that have not been serialized)
 int db.Count;                   // The number of entities in the ArrowDb
-int db.PendingChanges;          // The number of pending changes (number of changes that have not been serialized)
 ```
 
 For reading the data we have the following methods:
@@ -278,7 +283,7 @@ public interface IDbSerializer {
 }
 ```
 
-The `DeserializeAsync` method is invoked to load the db, and the `SerializeAsync` method is invoked to persist the db.
+The `DeserializeAsync` method is invoked to load the db, and the `SerializeAsync` method is invoked to persist the db. For custom file-based serializers, it is recommended to inherit from `BaseFileSerializer` to get atomic and multi-process safe writes out of the box.
 
 Being that they return a `ValueTask`, the implementations can be async. This means that you can even implement serializers to persist the db to a remote server, or cloud, or whatever else you want.
 
@@ -323,7 +328,9 @@ void SomeMethod() {
 } // the function scope ends here, and implicitly closes the scope of the transaction
 ```
 
-Using a transaction scope ensures that `SerializeAsync` is always called, even if an `Exception` is thrown.
+Using a transaction scope ensures that `SerializeAsync` is always called, even if an `Exception` is thrown. These scopes can be nested, and serialization will only occur when the outermost scope is disposed.
+
+`ArrowDbTransactionScope` also implements the regular `IDisposable` interface, meaning it can be used in a non-`async` method. However it internally calls the `DisposeAsync` method in a blocking manner, with the built in file-based serializers (`FileSerializer` and `AesFileSerializer`) it is completely safe as they naturally operate synchronously. However if you implemented a remote serializer or an `async` one, you should use the `Async Disposable` pattern accordingly.
 
 ## Subscribing to Changes
 

benchmarks/ArrowDbCore.Benchmarks.Common/ArrowDbCore.Benchmarks.Common.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net9.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Bogus" Version="35.6.3" />
+  </ItemGroup>
+
+</Project>

benchmarks/ArrowDbCore.Benchmarks/JContext.cs → benchmarks/ArrowDbCore.Benchmarks.Common/JContext.cs

@@ -1,6 +1,6 @@
 using System.Text.Json.Serialization;
 
-namespace ArrowDbCore.Benchmarks;
+namespace ArrowDbCore.Benchmarks.Common;
 
 [JsonSourceGenerationOptions(WriteIndented = false, NumberHandling = JsonNumberHandling.AllowReadingFromString, UseStringEnumConverter = true)]
 [JsonSerializable(typeof(Person))]

benchmarks/ArrowDbCore.Benchmarks/Person.cs → benchmarks/ArrowDbCore.Benchmarks.Common/Person.cs

@@ -1,6 +1,6 @@
 using Bogus;
 
-namespace ArrowDbCore.Benchmarks;
+namespace ArrowDbCore.Benchmarks.Common;
 
 public sealed class Person {
     public int Id { get; set; }

benchmarks/ArrowDbCore.Benchmarks.VersionComparison/ArrowDbCore.Benchmarks.VersionComparison.csproj

@@ -0,0 +1,25 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net9.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="BenchmarkDotNet" Version="0.15.2" />
+    <PackageReference Include="Bogus" Version="35.6.3" />
+
+    <PackageReference Include="NuGet.Protocol" Version="6.14.0" />
+    <PackageReference Include="NuGet.Versioning" Version="6.14.0" />
+    <PackageReference Include="NuGet.Common" Version="6.14.0" />
+
+    <PackageReference Include="ArrowDb" Version="1.4.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\ArrowDbCore.Benchmarks.Common\ArrowDbCore.Benchmarks.Common.csproj" />
+  </ItemGroup>
+
+</Project>

benchmarks/ArrowDbCore.Benchmarks.VersionComparison/Program.cs

@@ -0,0 +1,5 @@
+using ArrowDbCore.Benchmarks.VersionComparison;
+
+using BenchmarkDotNet.Running;
+
+BenchmarkRunner.Run<RandomOperationsBenchmarks>();

benchmarks/ArrowDbCore.Benchmarks.VersionComparison/RandomOperationsBenchmark.cs

@@ -0,0 +1,53 @@
+using System.Diagnostics;
+using BenchmarkDotNet.Attributes;
+using Bogus;
+using ArrowDbCore.Benchmarks.Common;
+using Person = ArrowDbCore.Benchmarks.Common.Person;
+
+namespace ArrowDbCore.Benchmarks.VersionComparison;
+
+[MemoryDiagnoser(false)]
+[RankColumn]
+[Config(typeof(VersionComparisonConfig))]
+public class RandomOperationsBenchmarks {
+    private Person[] _items = [];
+    private ArrowDb _db = default!;
+
+    [Params(100, 10_000, 1_000_000)]
+    public int Count { get; set; }
+
+    [IterationSetup]
+    public void Setup() {
+        var faker = new Faker {
+            Random = new Randomizer(1337)
+        };
+
+		_items = Person.GeneratePeople(Count, faker).ToArray();
+
+        Trace.Assert(_items.Length == Count);
+
+        _db = ArrowDb.CreateInMemory().GetAwaiter().GetResult();
+    }
+
+    [Benchmark]
+    public void RandomOperations() {
+        Parallel.For(0, Count, i => {
+            // Pick a random operation: 0 = add/update, 1 = remove
+            int operationType = Random.Shared.Next(0, 2);
+
+            var item = _items[i];
+
+            var key = item.Name;
+            var jsonTypeInfo = JContext.Default.Person;
+
+            switch (operationType) {
+                case 0: // Add/Update
+                    _db.Upsert(key, item, jsonTypeInfo);
+                    break;
+                case 1: // Remove
+                    _db.TryRemove(key);
+                    break;
+            }
+        });
+    }
+}

benchmarks/ArrowDbCore.Benchmarks.VersionComparison/SerializationToFileBenchmark.cs

@@ -0,0 +1,48 @@
+using System.Diagnostics;
+using BenchmarkDotNet.Attributes;
+using Bogus;
+using ArrowDbCore.Benchmarks.Common;
+using Person = ArrowDbCore.Benchmarks.Common.Person;
+
+namespace ArrowDbCore.Benchmarks.VersionComparison;
+
+[MemoryDiagnoser(false)]
+[RankColumn]
+[Config(typeof(VersionComparisonConfig))]
+public class SerializationToFileBenchmarks {
+    private ArrowDb _db = default!;
+
+    [Params(100, 10_000, 1_000_000)]
+    public int Size { get; set; }
+
+    [IterationSetup]
+    public void Setup() {
+        var faker = new Faker {
+            Random = new Randomizer(1337)
+        };
+
+        _db = ArrowDb.CreateFromFile("test.db").GetAwaiter().GetResult();
+
+        Span<char> buffer = stackalloc char[64];
+
+		foreach (var person in Person.GeneratePeople(Size, faker)) {
+            _ = person.Id.TryFormat(buffer, out var written);
+            var id = buffer.Slice(0, written);
+            _db.Upsert(id, person, JContext.Default.Person);
+        }
+
+        Trace.Assert(_db.Count == Size);
+    }
+
+    [IterationCleanup]
+    public void Cleanup() {
+        if (File.Exists("test.db")) {
+            File.Delete("test.db");
+        }
+    }
+
+    [Benchmark]
+    public async Task SerializeAsync() {
+        await _db.SerializeAsync();
+    }
+}

benchmarks/ArrowDbCore.Benchmarks.VersionComparison/VersionComparisonConfig.cs

@@ -0,0 +1,67 @@
+using BenchmarkDotNet.Columns;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Reports;
+
+using NuGet.Common;
+
+using NuGet.Protocol;
+
+using NuGet.Protocol.Core.Types;
+
+using NuGet.Versioning;
+
+namespace ArrowDbCore.Benchmarks.VersionComparison;
+
+public class VersionComparisonConfig : ManualConfig {
+    public const string PackageId = "ArrowDb";
+
+    public VersionComparisonConfig() {
+        var (stable, latest) = GetLatestVersionsAsync(PackageId)
+            .GetAwaiter()
+            .GetResult();
+
+        SummaryStyle = SummaryStyle.Default.WithRatioStyle(RatioStyle.Trend);
+
+        AddJob(Job.MediumRun
+            .WithBaseline(true)
+            .WithNuGet(PackageId, stable.ToNormalizedString())
+            .WithId($"Stable-{stable.ToNormalizedString()}"));
+
+        AddJob(Job.MediumRun
+            .WithNuGet(PackageId, latest.ToNormalizedString())
+            .WithId($"Latest-{latest.ToNormalizedString()}"));
+    }
+
+    private static async Task<(NuGetVersion stable, NuGetVersion latest)> GetLatestVersionsAsync(string packageId)
+    {
+        // Point at the official NuGet v3 API
+        var source = Repository.Factory.GetCoreV3("https://api.nuget.org/v3/index.json");
+        var metaResource = await source.GetResourceAsync<PackageMetadataResource>();
+
+        // Fetch all versions (incl. prerelease) and filter out unlisted packages
+        var allMetadata = await metaResource.GetMetadataAsync(
+            packageId,
+            includePrerelease: true,
+            includeUnlisted: false,
+            sourceCacheContext: new SourceCacheContext(),
+            log: NullLogger.Instance,
+            token: CancellationToken.None);
+
+        // Extract distinct versions
+        var versions = allMetadata
+            .Select(meta => meta.Identity.Version)
+            .Distinct()
+            .OrderBy(v => v)         // ascending
+            .ToList();
+
+        // Highest overall version (could be prerelease)
+        var latest = versions.Last();
+
+        // Highest *stable* (no prerelease); if none, fall back to latest
+        var stableVersions = versions.Where(v => !v.IsPrerelease).ToList();
+        var stable = stableVersions.Any() ? stableVersions.Last() : latest;
+
+        return (stable, latest);
+    }
+}