diff --git a/.github/workflows/unit-tests-matrix.yaml b/.github/workflows/unit-tests-matrix.yaml index 37b7cc6..5d8b2df 100644 --- a/.github/workflows/unit-tests-matrix.yaml +++ b/.github/workflows/unit-tests-matrix.yaml @@ -37,7 +37,7 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v3 with: - dotnet-version: 9.0.x + dotnet-version: 10.0.x - name: Install dependencies run: dotnet restore ${{ env.PROJECT }} diff --git a/.github/workflows/unit-tests-ubuntu.yaml b/.github/workflows/unit-tests-ubuntu.yaml index ed9bc45..48b1113 100644 --- a/.github/workflows/unit-tests-ubuntu.yaml +++ b/.github/workflows/unit-tests-ubuntu.yaml @@ -12,5 +12,5 @@ jobs: uses: dusrdev/actions/.github/workflows/reusable-dotnet-test-mtp.yaml@main with: platform: ubuntu-latest - dotnet-version: 9.0.x + dotnet-version: 10.0.x test-project-path: ${{ matrix.project }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 07a1f13..b651e7a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,12 @@ ## ## Get latest from `dotnet new gitignore` +# Benchmark results +**/BenchmarkDotNet.Artifacts/results/* +**/BenchmarkDotNet.Artifacts/*.log +**/BenchmarkDotNet.Artifacts/*.csv +**/BenchmarkDotNet.Artifacts/*.html + # dotenv files .env diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..431ccfd --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,92 @@ +# AGENTS.md + +This file contains repo-specific instructions for AI coding agents working in this repository. + +## Critical points before answering any question or performing any task + +- Never assume based on read data from earlier points in the conversation or logical guesses - Always read the latest version of each file that is references in the tasks or conversation, unless a different version is explicitly asked for. +- Never change the source project/code to make something in the unit tests easier when it costs perf or otherwise. The source code is much more important than testing convenience. If you have suggestion on how to refactor the source to allow easier testing - always prompt the user asking if he would like them implemented, never assume that without asking. +- When answering about the capabilities, source code and otherwise of 3rd party libraries ensure to always give the most correct and up-to-date answer. If you are not sure, search the web to find what happens in the exact version used. +- When discussing APIs, make sure all of your logic aligns with whether the APIs are publicly used / public but only used internally / or internal. +- If you require source that you don't have access to - ask the user if he may provide them. +- Always start answering questions about the codebase, or tasks by reading AGENTS.md, and make sure to keep it up-to-date. +- If public APIs or usage semantics change, prompt the user and asks if he would like to update the README.md / Changelog files (if they exist). + +## Repo overview (ArrowDbCore) + +- Product: ArrowDb (NuGet package id: `ArrowDb`) - fast, lightweight, type-safe key-value database for .NET. +- Runtime target: `net9.0` (repo relies on .NET 9 APIs such as `ConcurrentDictionary<,>.AlternateLookup>`). +- Goals: tiny footprint, minimal allocations, thread-safe concurrency, AOT + trimming compatibility, System.Text.Json source-gen (no reflection). + +## How ArrowDb works (implementation notes) + +- Data model: an in-memory `ConcurrentDictionary` (`ArrowDb.Source`). + - Keys are `string` in storage; many APIs accept `ReadOnlySpan` to avoid allocations when looking up/removing keys derived from slices. + - Values are UTF-8 JSON bytes produced by `System.Text.Json` using caller-provided `JsonTypeInfo` (source-generated metadata). +- Type safety: `TryGetValue`/`Upsert` require a `JsonTypeInfo`; passing the wrong `JsonTypeInfo` for stored bytes can throw `JsonException`. +- Change tracking: + - Any successful mutation calls `OnChangeInternal(...)`, which increments `_pendingChanges` and then invokes the `OnChange` event. + - `PendingChanges` is a `long` counter; `SerializeAsync()` resets it to `0` only if no new changes happened during the serialization window (conditional reset to avoid losing the “needs another serialize” signal). +- Concurrency model: + - Normal reads/writes are lock-free at the dictionary level (`ConcurrentDictionary`). + - A per-instance `SemaphoreSlim` guards `SerializeAsync()`/`RollbackAsync()`. Writers do not take the semaphore, but they do call `WaitIfSerializing()` to avoid mutating while a serialize is actively in progress. + - `RollbackAsync()` increments a monotonic in-memory epoch (`StateEpoch`). Mutating operations (`Upsert`, `TryRemove`, `TryClear`) detect an epoch change during the operation and return `false` to signal the mutation was not reliable relative to the rollback. + - Multi-process safety for file serializers is implemented via a system-wide named `Mutex` in `BaseFileSerializer` (per DB path). +- Transactions: + - `BeginTransaction()` returns `ArrowDbTransactionScope`. + - `ArrowDbTransactionScope` increments `ArrowDb.TransactionDepth` on creation and decrements on dispose. + - Only when the outermost scope is disposed (`TransactionDepth` reaches `0`) does it call `SerializeAsync()`; nested scopes do not serialize. + +## Source code map (what lives where) + +- `src/ArrowDbCore/ArrowDb.cs`: core state (`Source`, `Lookup`, `Serializer`, `Semaphore`), counters (`RunningInstances`, `PendingChanges`), `OnChange` event, and `BeginTransaction()`. +- `src/ArrowDbCore/ArrowDb.Factory.cs`: factory initializers (`CreateFromFile`, `CreateFromFileWithAes`, `CreateInMemory`, `CreateCustom`) + `GenerateTypedKey(...)`. +- `src/ArrowDbCore/ArrowDbJsonContext.cs`: internal `JsonSerializerContext` used by file serializers to (de)serialize `ConcurrentDictionary` without reflection. +- `src/ArrowDbCore/ArrowDb.Read.cs`: read-only API (`Count`, `Keys`, `ContainsKey`, `TryGetValue`). + - Note: `TryGetValue` returns `true` for value types even when the value is `default(T)`. For reference/nullable types it returns `false` when the deserialized value is `null`, preserving the “no null-check after `TryGetValue == true`” guarantee. +- `src/ArrowDbCore/ArrowDb.Upsert.cs`: `Upsert` overloads + optimistic concurrency via `updateCondition`. + - Span-vs-string keys: `Upsert(ReadOnlySpan ...)` uses `Lookup[...]`; this avoids allocating a new string when updating an existing key, but inserting a non-existing key may still allocate a new string key internally. Prefer the `string` overload when the key is already a `string`. + - Null policy: `UpsertCore` returns `false` for `null` reference values (no-`null` design). +- `src/ArrowDbCore/ArrowDb.GetOrAdd.cs`: `GetOrAddAsync` helpers (string keys only); note the check-then-upsert is not atomic across threads (duplicate factory calls are possible under races). +- `src/ArrowDbCore/ArrowDb.Remove.cs`: `TryRemove(ReadOnlySpan)`, `TryClear()`, and `Clear()` (obsolete; use `TryClear()`). +- `src/ArrowDbCore/ArrowDb.Serialization.cs`: `SerializeAsync()` and `RollbackAsync()` + the `WaitIfSerializing()` gate, conditional `PendingChanges` reset, and rollback epoch bump (`StateEpoch`). +- `src/ArrowDbCore/ArrowDbTransactionScope.cs`: transaction scope that defers serialization until disposed (supports both `IDisposable` and `IAsyncDisposable`). +- `src/ArrowDbCore/ArrowDb.IDictionaryAccessor.cs`: internal indirection used by `UpsertCore` to write via either `Source` (string keys) or `Lookup` (span keys). +- `src/ArrowDbCore/IDbSerializer.cs`: public serializer abstraction for persisting/loading the dictionary. +- `src/ArrowDbCore/Serializers/BaseFileSerializer.cs`: shared file serializer base (atomic write via `*.tmp` + `File.Move`, cross-process lock via named mutex). +- `src/ArrowDbCore/Serializers/FileSerializer.cs`: JSON file serializer (writes plain JSON). +- `src/ArrowDbCore/Serializers/AesFileSerializer.cs`: AES-encrypted JSON file serializer (wraps stream with `CryptoStream`). +- `src/ArrowDbCore/Serializers/InMemorySerializer.cs`: no-op serializer for purely in-memory databases. +- `src/ArrowDbCore/ChangeEventArgs.cs`: `ArrowDbChangeEventArgs` + `ArrowDbChangeType` used by `OnChange`. +- `src/ArrowDbCore/Extensions.cs`: internal helpers (currently used for SHA-256 hashing to derive mutex names). + +## Repository layout + +- `src/ArrowDbCore/`: main library (public API lives here). +- `tests/`: + - `ArrowDbCore.Tests.Unit/`: unit tests (Microsoft Testing Platform + xUnit v3). + - `ArrowDbCore.Tests.Unit.Isolated/`: unit tests intended to be runnable in isolation (Microsoft Testing Platform + xUnit v3). + - `ArrowDbCore.Tests.Integrity/`: integrity tests (Microsoft Testing Platform + xUnit v3; may do heavier scenarios). + - `ArrowDbCore.Tests.Analyzers/`: builds the library with trimming/AOT settings to catch issues early (not a test runner). + - `ArrowDbCore.Tests.Common/`: shared test utilities. +- `benchmarks/`: + - `ArrowDbCore.Benchmarks/`: main benchmarks (BenchmarkDotNet). + - `ArrowDbCore.Benchmarks.VersionComparison/`: compares current code vs a referenced released package. + +## Common commands (local + CI parity) + +- Build: `dotnet build ArrowDbCore.slnx -c Release` +- Unit tests (CI matrix): `dotnet test tests/ArrowDbCore.Tests.Unit/ArrowDbCore.Tests.Unit.csproj -c Release` and `dotnet test tests/ArrowDbCore.Tests.Unit.Isolated/ArrowDbCore.Tests.Unit.Isolated.csproj -c Release` +- Integrity tests (CI): `dotnet test tests/ArrowDbCore.Tests.Integrity/ArrowDbCore.Tests.Integrity.csproj -c Release` +- AOT/trimming sanity build (CI): `dotnet build tests/ArrowDbCore.Tests.Analyzers/ArrowDbCore.Tests.Analyzers.csproj -c Release` +- Benchmarks: `dotnet run -c Release --project benchmarks/ArrowDbCore.Benchmarks/ArrowDbCore.Benchmarks.csproj` + +## Code conventions and constraints + +- Follow `.editorconfig` (notably: file-scoped namespaces; explicit types over `var`; and the repo prefers CRLF line endings). +- Avoid adding new NuGet dependencies to `src/ArrowDbCore/ArrowDbCore.csproj` unless explicitly requested (the library is intentionally dependency-free). +- Performance-first: avoid avoidable allocations; prefer `ReadOnlySpan` APIs and the `Lookup` alternate lookup path; avoid introducing LINQ or other allocation-heavy patterns into hot paths. +- Preserve documented semantics (README): + - Reference-type `null` values are rejected on `Upsert` (no-`null` policy). + - Type safety is enforced via `JsonTypeInfo`/`JsonSerializerContext`; do not introduce reflection-based serialization. +- If a change affects public API/behavior/versioning, confirm intent and then update `README.md`, `CHANGELOG.md`, and `src/ArrowDbCore/Readme.Nuget.md` as appropriate. diff --git a/CHANGELOG.md b/CHANGELOG.md index 65885cb..4ce24b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog (Sorted by Date in Descending Order) +## 1.6.0.0 + +- Improve correctness of internal change counting to ensure that changes that happened during serialization are still tracked. +- `TryGetValue` will now return true for `value types` that have a default value since it is a valid value for them. +- `Upsert` can return `false` if a `RollbackAsync` occurred concurrently, indicating the write was not reliable relative to the rollback (retry after rollback completes if needed). +- `TryRemove` and `TryClear` can return `false` if a `RollbackAsync` occurred concurrently, indicating the operation was not reliable relative to the rollback. +- `Clear` is now obsolete; use `TryClear` to detect rollback races. + ## 1.5.0.0 - File based serializers `FileSerializer` and `AesFileSerializer` now use a new base class implementation and have gained the ability to `journal` (maintain durability through crashes and other `IOException`, and ensure successful atomic write or complete rejection of changes), and cross-process isolation, preventing race condition that could be caused when multiple processes try to access the same `ArrowDb` file. diff --git a/README.md b/README.md index ca83e3d..4dc230a 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,8 @@ ArrowDb is a fast, lightweight, and type-safe key-value database designed for .NET. * Super-Lightweight (dll size is ~19KB - approximately 9X smaller than [UltraLiteDb](https://github.com/rejemy/UltraLiteDB)) -* Ultra-Fast (1,000,000 random operations / ~100ms on M2 MacBook Pro) -* Minimal-Allocation (~2KB for serialization of 1,000,000 items) +* Ultra-Fast (1,000,000 random operations / ~98ms on M2 MacBook Pro) +* Minimal-Allocation (constant ~520 bytes for serialization any db size) * Thread-Safe and Concurrent * ACID compliant on transaction level * Type-Safe (no reflection - compile-time enforced via source-generated `JsonSerializerContext`) @@ -59,7 +59,7 @@ public class Person { public partial class MyJsonContext : JsonSerializerContext {} ``` -Now we can upsert (insert or update) a `Person` into the db: +Now we can upsert (insert or update, similar to "put") a `Person` into the db: ```csharp var john = new Person { Id = 1, Name = "John", Surname = "Doe", Age = 42 }; @@ -103,7 +103,7 @@ bool db.TryGetValue(ReadOnlySpan key, JsonTypeInfo jsonTyp Notice that all APIs accept keys as `ReadOnlySpan` to avoid unnecessary allocations. This means that if you check for a key by some slice of a string, there is no need to allocate a string just for the lookup. -Upserting (adding or updating) is done via 6 overloads: +Upserting (adding or updating, similar to "put") is done via 6 overloads: ```csharp bool db.Upsert(string key, TValue value, JsonTypeInfo jsonTypeInfo); @@ -126,7 +126,8 @@ And removal: ```csharp bool db.TryRemove(ReadOnlySpan key); // removes the entry with the specified key -void Clear(); // removes all entries from the ArrowDb instance +bool db.TryClear(); // clears all entries; returns false if a concurrent RollbackAsync occurred +void db.Clear(); // obsolete: use TryClear() ``` ## Optimistic Concurrency Control @@ -247,6 +248,10 @@ async ValueTask GetOrAddAsync(string key, JsonTypeInfo GeneratePeople(int count, Faker faker) { - for (var i = 0; i < count; i++) { - yield return new Person { - Id = i, - Name = faker.Name.FirstName(), - Surname = faker.Name.LastName(), - Age = faker.Random.Int(0, 100) - }; - } - } + public static IEnumerable GeneratePeople(int count, Faker faker) { + for (var i = 0; i < count; i++) { + yield return new Person { + Id = i, + Name = faker.Name.FirstName(), + Surname = faker.Name.LastName(), + Age = faker.Random.Int(0, 100) + }; + } + } } \ No newline at end of file diff --git a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/ArrowDbCore.Benchmarks.VersionComparison.csproj b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/ArrowDbCore.Benchmarks.VersionComparison.csproj index 067aa97..091528b 100644 --- a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/ArrowDbCore.Benchmarks.VersionComparison.csproj +++ b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/ArrowDbCore.Benchmarks.VersionComparison.csproj @@ -2,20 +2,20 @@ Exe - net9.0 + net10.0 enable enable - - + + - - - + + + - + diff --git a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/RandomOperationsBenchmark.cs b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/RandomOperationsBenchmark.cs index 0b2a423..2e3b76f 100644 --- a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/RandomOperationsBenchmark.cs +++ b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/RandomOperationsBenchmark.cs @@ -1,7 +1,11 @@ using System.Diagnostics; + +using ArrowDbCore.Benchmarks.Common; + using BenchmarkDotNet.Attributes; + using Bogus; -using ArrowDbCore.Benchmarks.Common; + using Person = ArrowDbCore.Benchmarks.Common.Person; namespace ArrowDbCore.Benchmarks.VersionComparison; @@ -22,7 +26,7 @@ public void Setup() { Random = new Randomizer(1337) }; - _items = Person.GeneratePeople(Count, faker).ToArray(); + _items = Person.GeneratePeople(Count, faker).ToArray(); Trace.Assert(_items.Length == Count); @@ -50,4 +54,4 @@ public void RandomOperations() { } }); } -} +} \ No newline at end of file diff --git a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/SerializationToFileBenchmark.cs b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/SerializationToFileBenchmark.cs index da9b149..36e0504 100644 --- a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/SerializationToFileBenchmark.cs +++ b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/SerializationToFileBenchmark.cs @@ -1,7 +1,11 @@ using System.Diagnostics; + +using ArrowDbCore.Benchmarks.Common; + using BenchmarkDotNet.Attributes; + using Bogus; -using ArrowDbCore.Benchmarks.Common; + using Person = ArrowDbCore.Benchmarks.Common.Person; namespace ArrowDbCore.Benchmarks.VersionComparison; @@ -25,7 +29,7 @@ public void Setup() { Span buffer = stackalloc char[64]; - foreach (var person in Person.GeneratePeople(Size, faker)) { + foreach (var person in Person.GeneratePeople(Size, faker)) { _ = person.Id.TryFormat(buffer, out var written); var id = buffer.Slice(0, written); _db.Upsert(id, person, JContext.Default.Person); @@ -45,4 +49,4 @@ public void Cleanup() { public async Task SerializeAsync() { await _db.SerializeAsync(); } -} +} \ No newline at end of file diff --git a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/VersionComparisonConfig.cs b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/VersionComparisonConfig.cs index c13e532..5a3dd84 100644 --- a/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/VersionComparisonConfig.cs +++ b/benchmarks/ArrowDbCore.Benchmarks.VersionComparison/VersionComparisonConfig.cs @@ -33,8 +33,7 @@ public VersionComparisonConfig() { .WithId($"Latest-{latest.ToNormalizedString()}")); } - private static async Task<(NuGetVersion stable, NuGetVersion latest)> GetLatestVersionsAsync(string packageId) - { + private static async Task<(NuGetVersion stable, NuGetVersion latest)> GetLatestVersionsAsync(string packageId) { // Point at the official NuGet v3 API var source = Repository.Factory.GetCoreV3("https://api.nuget.org/v3/index.json"); var metaResource = await source.GetResourceAsync(); @@ -64,4 +63,4 @@ public VersionComparisonConfig() { return (stable, latest); } -} +} \ No newline at end of file diff --git a/benchmarks/ArrowDbCore.Benchmarks/ArrowDbCore.Benchmarks.csproj b/benchmarks/ArrowDbCore.Benchmarks/ArrowDbCore.Benchmarks.csproj index f3944d8..cdb6863 100644 --- a/benchmarks/ArrowDbCore.Benchmarks/ArrowDbCore.Benchmarks.csproj +++ b/benchmarks/ArrowDbCore.Benchmarks/ArrowDbCore.Benchmarks.csproj @@ -2,7 +2,7 @@ Exe - net9.0 + net10.0 enable enable @@ -13,8 +13,8 @@ - - + + diff --git a/benchmarks/ArrowDbCore.Benchmarks/BenchmarkDotNet.Artifacts/ArrowDbCore.Benchmarks.RandomOperationsBenchmarks-report-github.md b/benchmarks/ArrowDbCore.Benchmarks/BenchmarkDotNet.Artifacts/ArrowDbCore.Benchmarks.RandomOperationsBenchmarks-report-github.md deleted file mode 100644 index 454669c..0000000 --- a/benchmarks/ArrowDbCore.Benchmarks/BenchmarkDotNet.Artifacts/ArrowDbCore.Benchmarks.RandomOperationsBenchmarks-report-github.md +++ /dev/null @@ -1,18 +0,0 @@ -# ArrowDb.Benchmarks.RandomOperationsBenchmarks - -```log -BenchmarkDotNet v0.14.0, macOS Sequoia 15.2 (24C101) [Darwin 24.2.0] -Apple M2 Pro, 1 CPU, 10 logical and 10 physical cores -.NET SDK 9.0.100 - [Host] : .NET 9.0.0 (9.0.24.52809), Arm64 RyuJIT AdvSIMD - MediumRun : .NET 9.0.0 (9.0.24.52809), Arm64 RyuJIT AdvSIMD - -Job=MediumRun InvocationCount=1 IterationCount=15 -LaunchCount=2 UnrollFactor=1 WarmupCount=10 -``` - -| Method | Count | Mean | Error | StdDev | Rank | Allocated | -|----------------- |-------- |--------------:|-------------:|-------------:|-----:|------------:| -| **RandomOperations** | **100** | **39.41 μs** | **2.790 μs** | **4.090 μs** | **1** | **14.56 KB** | -| **RandomOperations** | **10000** | **1,724.96 μs** | **487.760 μs** | **699.531 μs** | **2** | **1008.37 KB** | -| **RandomOperations** | **1000000** | **104,545.08 μs** | **1,799.174 μs** | **2,637.206 μs** | **3** | **62710.32 KB** | diff --git a/benchmarks/ArrowDbCore.Benchmarks/BenchmarkDotNet.Artifacts/ArrowDbCore.Benchmarks.RandomOperationsBenchmarks.md b/benchmarks/ArrowDbCore.Benchmarks/BenchmarkDotNet.Artifacts/ArrowDbCore.Benchmarks.RandomOperationsBenchmarks.md new file mode 100644 index 0000000..109ba76 --- /dev/null +++ b/benchmarks/ArrowDbCore.Benchmarks/BenchmarkDotNet.Artifacts/ArrowDbCore.Benchmarks.RandomOperationsBenchmarks.md @@ -0,0 +1,17 @@ +``` + +BenchmarkDotNet v0.15.8, macOS Tahoe 26.1 (25B78) [Darwin 25.1.0] +Apple M2 Pro, 1 CPU, 10 logical and 10 physical cores +.NET SDK 10.0.101 + [Host] : .NET 10.0.1 (10.0.1, 10.0.125.57005), Arm64 RyuJIT armv8.0-a + MediumRun : .NET 10.0.1 (10.0.1, 10.0.125.57005), Arm64 RyuJIT armv8.0-a + +Job=MediumRun InvocationCount=1 IterationCount=15 +LaunchCount=2 UnrollFactor=1 WarmupCount=10 + +``` +| Method | Count | Mean | Error | StdDev | Rank | Allocated | +|----------------- |-------- |-------------:|-------------:|-------------:|-----:|------------:| +| **RandomOperations** | **100** | **41.73 μs** | **3.662 μs** | **5.252 μs** | **1** | **15.84 KB** | +| **RandomOperations** | **10000** | **1,349.40 μs** | **65.665 μs** | **89.883 μs** | **2** | **701.72 KB** | +| **RandomOperations** | **1000000** | **98,975.55 μs** | **1,918.205 μs** | **2,811.681 μs** | **3** | **53612.05 KB** | diff --git a/benchmarks/ArrowDbCore.Benchmarks/BenchmarkDotNet.Artifacts/ArrowDbCore.Benchmarks.SerializationToFileBenchmarks.md b/benchmarks/ArrowDbCore.Benchmarks/BenchmarkDotNet.Artifacts/ArrowDbCore.Benchmarks.SerializationToFileBenchmarks.md index c03f1c2..9f7cd2a 100644 --- a/benchmarks/ArrowDbCore.Benchmarks/BenchmarkDotNet.Artifacts/ArrowDbCore.Benchmarks.SerializationToFileBenchmarks.md +++ b/benchmarks/ArrowDbCore.Benchmarks/BenchmarkDotNet.Artifacts/ArrowDbCore.Benchmarks.SerializationToFileBenchmarks.md @@ -1,18 +1,17 @@ -# ArrowDb.Benchmarks.SerializationToFileBenchmarks +``` -```log -BenchmarkDotNet v0.14.0, macOS Sequoia 15.2 (24C101) [Darwin 24.2.0] +BenchmarkDotNet v0.15.8, macOS Tahoe 26.1 (25B78) [Darwin 25.1.0] Apple M2 Pro, 1 CPU, 10 logical and 10 physical cores -.NET SDK 9.0.100 - [Host] : .NET 9.0.0 (9.0.24.52809), Arm64 RyuJIT AdvSIMD - MediumRun : .NET 9.0.0 (9.0.24.52809), Arm64 RyuJIT AdvSIMD +.NET SDK 10.0.101 + [Host] : .NET 10.0.1 (10.0.1, 10.0.125.57005), Arm64 RyuJIT armv8.0-a + MediumRun : .NET 10.0.1 (10.0.1, 10.0.125.57005), Arm64 RyuJIT armv8.0-a -Job=MediumRun InvocationCount=1 IterationCount=15 -LaunchCount=2 UnrollFactor=1 WarmupCount=10 -``` +Job=MediumRun InvocationCount=1 IterationCount=15 +LaunchCount=2 UnrollFactor=1 WarmupCount=10 +``` | Method | Size | Mean | Error | StdDev | Rank | Allocated | |--------------- |-------- |-------------:|------------:|------------:|-----:|----------:| -| **SerializeAsync** | **100** | **117.5 μs** | **9.29 μs** | **13.33 μs** | **1** | **2.45 KB** | -| **SerializeAsync** | **10000** | **1,671.1 μs** | **151.09 μs** | **221.46 μs** | **2** | **2.63 KB** | -| **SerializeAsync** | **1000000** | **158,980.0 μs** | **1,682.80 μs** | **2,413.43 μs** | **3** | **2.02 KB** | +| **SerializeAsync** | **100** | **207.4 μs** | **17.56 μs** | **25.73 μs** | **1** | **520 B** | +| **SerializeAsync** | **10000** | **2,409.7 μs** | **333.98 μs** | **499.89 μs** | **2** | **520 B** | +| **SerializeAsync** | **1000000** | **144,343.7 μs** | **1,514.16 μs** | **2,219.44 μs** | **3** | **520 B** | diff --git a/benchmarks/ArrowDbCore.Benchmarks/BenchmarkDotNet.Artifacts/results/ArrowDbCore.Benchmarks.RandomOperationsBenchmarks-report-github.md b/benchmarks/ArrowDbCore.Benchmarks/BenchmarkDotNet.Artifacts/results/ArrowDbCore.Benchmarks.RandomOperationsBenchmarks-report-github.md deleted file mode 100644 index 464b29e..0000000 --- a/benchmarks/ArrowDbCore.Benchmarks/BenchmarkDotNet.Artifacts/results/ArrowDbCore.Benchmarks.RandomOperationsBenchmarks-report-github.md +++ /dev/null @@ -1,17 +0,0 @@ -``` - -BenchmarkDotNet v0.15.2, macOS Sequoia 15.5 (24F74) [Darwin 24.5.0] -Apple M2 Pro, 1 CPU, 10 logical and 10 physical cores -.NET SDK 9.0.303 - [Host] : .NET 9.0.7 (9.0.725.31616), Arm64 RyuJIT AdvSIMD - MediumRun : .NET 9.0.7 (9.0.725.31616), Arm64 RyuJIT AdvSIMD - -Job=MediumRun InvocationCount=1 IterationCount=15 -LaunchCount=2 UnrollFactor=1 WarmupCount=10 - -``` -| Method | Count | Mean | Error | StdDev | Rank | Allocated | -|----------------- |-------- |--------------:|-------------:|-------------:|-----:|------------:| -| **RandomOperations** | **100** | **41.75 μs** | **5.523 μs** | **8.096 μs** | **1** | **15.11 KB** | -| **RandomOperations** | **10000** | **1,779.51 μs** | **484.766 μs** | **679.575 μs** | **2** | **722.35 KB** | -| **RandomOperations** | **1000000** | **101,016.96 μs** | **1,921.740 μs** | **2,876.370 μs** | **3** | **53529.98 KB** | diff --git a/benchmarks/ArrowDbCore.Benchmarks/RandomOperationsBenchmark.cs b/benchmarks/ArrowDbCore.Benchmarks/RandomOperationsBenchmark.cs index 51837ec..c3e0bd4 100644 --- a/benchmarks/ArrowDbCore.Benchmarks/RandomOperationsBenchmark.cs +++ b/benchmarks/ArrowDbCore.Benchmarks/RandomOperationsBenchmark.cs @@ -1,7 +1,11 @@ using System.Diagnostics; + +using ArrowDbCore.Benchmarks.Common; + using BenchmarkDotNet.Attributes; + using Bogus; -using ArrowDbCore.Benchmarks.Common; + using Person = ArrowDbCore.Benchmarks.Common.Person; namespace ArrowDbCore.Benchmarks; @@ -22,7 +26,7 @@ public void Setup() { Random = new Randomizer(1337) }; - _items = Person.GeneratePeople(Count, faker).ToArray(); + _items = Person.GeneratePeople(Count, faker).ToArray(); Trace.Assert(_items.Length == Count); diff --git a/benchmarks/ArrowDbCore.Benchmarks/SerializationToFileBenchmark.cs b/benchmarks/ArrowDbCore.Benchmarks/SerializationToFileBenchmark.cs index 91872cf..6c13422 100644 --- a/benchmarks/ArrowDbCore.Benchmarks/SerializationToFileBenchmark.cs +++ b/benchmarks/ArrowDbCore.Benchmarks/SerializationToFileBenchmark.cs @@ -1,7 +1,11 @@ using System.Diagnostics; + +using ArrowDbCore.Benchmarks.Common; + using BenchmarkDotNet.Attributes; + using Bogus; -using ArrowDbCore.Benchmarks.Common; + using Person = ArrowDbCore.Benchmarks.Common.Person; namespace ArrowDbCore.Benchmarks; @@ -25,7 +29,7 @@ public void Setup() { Span buffer = stackalloc char[64]; - foreach (var person in Person.GeneratePeople(Size, faker)) { + foreach (var person in Person.GeneratePeople(Size, faker)) { _ = person.Id.TryFormat(buffer, out var written); var id = buffer.Slice(0, written); _db.Upsert(id, person, JContext.Default.Person); diff --git a/src/ArrowDbCore/ArrowDb.Factory.cs b/src/ArrowDbCore/ArrowDb.Factory.cs index ae48ccf..179e1cd 100644 --- a/src/ArrowDbCore/ArrowDb.Factory.cs +++ b/src/ArrowDbCore/ArrowDb.Factory.cs @@ -5,16 +5,16 @@ namespace ArrowDbCore; public partial class ArrowDb { - /// - /// Initializes a file/disk backed database at the specified path - /// - /// The path that the file that backs the database - /// A database instance - public static async ValueTask CreateFromFile(string path) { - var serializer = new FileSerializer(path, ArrowDbJsonContext.Default.ConcurrentDictionaryStringByteArray); - var data = await serializer.DeserializeAsync(); - return new ArrowDb(data, serializer); - } + /// + /// Initializes a file/disk backed database at the specified path + /// + /// The path that the file that backs the database + /// A database instance + public static async ValueTask CreateFromFile(string path) { + var serializer = new FileSerializer(path, ArrowDbJsonContext.Default.ConcurrentDictionaryStringByteArray); + var data = await serializer.DeserializeAsync(); + return new ArrowDb(data, serializer); + } /// /// Initializes an managed file/disk backed database at the specified path @@ -23,55 +23,55 @@ public static async ValueTask CreateFromFile(string path) { /// The instance to use /// A database instance public static async ValueTask CreateFromFileWithAes(string path, Aes aes) { - var serializer = new AesFileSerializer(path, aes, ArrowDbJsonContext.Default.ConcurrentDictionaryStringByteArray); - var data = await serializer.DeserializeAsync(); - return new ArrowDb(data, serializer); - } + var serializer = new AesFileSerializer(path, aes, ArrowDbJsonContext.Default.ConcurrentDictionaryStringByteArray); + var data = await serializer.DeserializeAsync(); + return new ArrowDb(data, serializer); + } - /// - /// Initializes an in-memory database - /// - /// A database instance - public static async ValueTask CreateInMemory() { - var serializer = new InMemorySerializer(); - var data = await serializer.DeserializeAsync(); - return new ArrowDb(data, serializer); - } + /// + /// Initializes an in-memory database + /// + /// A database instance + public static async ValueTask CreateInMemory() { + var serializer = new InMemorySerializer(); + var data = await serializer.DeserializeAsync(); + return new ArrowDb(data, serializer); + } - /// - /// Initializes a database with a custom implementation - /// - /// A custom implementation - /// A database instance - public static async ValueTask CreateCustom(IDbSerializer serializer) { - var data = await serializer.DeserializeAsync(); - return new ArrowDb(data, serializer); - } + /// + /// Initializes a database with a custom implementation + /// + /// A custom implementation + /// A database instance + public static async ValueTask CreateCustom(IDbSerializer serializer) { + var data = await serializer.DeserializeAsync(); + return new ArrowDb(data, serializer); + } - /// - /// Generates a typed key for the specified specific key in a very efficient manner - /// - /// The type of the value - /// The key that is specific to the value - /// The buffer to use for the generation - /// - /// A key that is formatted as ":" - /// - public static ReadOnlySpan GenerateTypedKey(ReadOnlySpan specificKey, Span buffer) { - var typeName = TypeNameCache.TypeName; - var length = typeName.Length + 1 + specificKey.Length; // type:specificKey - ArgumentOutOfRangeException.ThrowIfGreaterThan(length, buffer.Length); + /// + /// Generates a typed key for the specified specific key in a very efficient manner + /// + /// The type of the value + /// The key that is specific to the value + /// The buffer to use for the generation + /// + /// A key that is formatted as ":" + /// + public static ReadOnlySpan GenerateTypedKey(ReadOnlySpan specificKey, Span buffer) { + var typeName = TypeNameCache.TypeName; + var length = typeName.Length + 1 + specificKey.Length; // type:specificKey + ArgumentOutOfRangeException.ThrowIfGreaterThan(length, buffer.Length); typeName.CopyTo(buffer); buffer[typeName.Length] = ':'; specificKey.CopyTo(buffer.Slice(typeName.Length + 1)); return buffer.Slice(0, length); - } + } - // A static class that caches type names during runtime - private static class TypeNameCache { - /// - /// The name of the type of T - /// - public static readonly string TypeName = typeof(T).Name; - } -} + // A static class that caches type names during runtime + private static class TypeNameCache { + /// + /// The name of the type of T + /// + public static readonly string TypeName = typeof(T).Name; + } +} \ No newline at end of file diff --git a/src/ArrowDbCore/ArrowDb.GetOrAdd.cs b/src/ArrowDbCore/ArrowDb.GetOrAdd.cs index 138178a..4182bdd 100644 --- a/src/ArrowDbCore/ArrowDb.GetOrAdd.cs +++ b/src/ArrowDbCore/ArrowDb.GetOrAdd.cs @@ -4,43 +4,59 @@ namespace ArrowDbCore; public partial class ArrowDb { - /// - /// Tries to retrieve a value stored in the database under , if it doesn't exist, it uses the factory to create and add it, then returns it. - /// - /// The type of the value to get or add - /// The key at which to find or add the value - /// The json type info for the value type - /// The function used to generate a value for the key - /// The value after finding or adding it - /// - /// - public async ValueTask GetOrAddAsync(string key, JsonTypeInfo jsonTypeInfo, Func> valueFactory) { - if (Lookup.TryGetValue(key, out var source)) { - return JsonSerializer.Deserialize(new ReadOnlySpan(source), jsonTypeInfo)!; - } - var val = await valueFactory(key); - Upsert(key, val, jsonTypeInfo); - return val; - } + /// + /// Tries to retrieve a value stored in the database under , if it doesn't exist, it uses the factory to create and add it, then returns it. + /// + /// The type of the value to get or add + /// The key at which to find or add the value + /// The json type info for the value type + /// The function used to generate a value for the key + /// The value after finding or adding it + /// + /// + /// This method is intentionally not atomic: under concurrent callers, may be invoked multiple times for the same . + /// The returned value is always the value produced by this invocation of . + /// The final stored value is last-writer-wins, since it is persisted via . + /// + /// + /// If you need single-invocation semantics for (e.g. the factory has side-effects or is expensive), guard the call site with a keyed lock. + /// + /// + public async ValueTask GetOrAddAsync(string key, JsonTypeInfo jsonTypeInfo, Func> valueFactory) { + if (Lookup.TryGetValue(key, out var source)) { + return JsonSerializer.Deserialize(new ReadOnlySpan(source), jsonTypeInfo)!; + } + var val = await valueFactory(key); + Upsert(key, val, jsonTypeInfo); + return val; + } - /// - /// Tries to retrieve a value stored in the database under , if it doesn't exist, it uses the factory to create and add it, then returns it. - /// - /// The type of the value to get or add - /// The type of the argument for the updateCondition function - /// The key at which to find or add the value - /// The json type info for the value type - /// The function used to generate a value for the key - /// An argument that could be provided to the valueFactory function to avoid a closure - /// The value after finding or adding it - /// - /// - public async ValueTask GetOrAddAsync(string key, JsonTypeInfo jsonTypeInfo, Func> valueFactory, TArg factoryArgument) { - if (Lookup.TryGetValue(key, out var source)) { - return JsonSerializer.Deserialize(new ReadOnlySpan(source), jsonTypeInfo)!; - } - var val = await valueFactory(key, factoryArgument); - Upsert(key, val, jsonTypeInfo); - return val; - } -} + /// + /// Tries to retrieve a value stored in the database under , if it doesn't exist, it uses the factory to create and add it, then returns it. + /// + /// The type of the value to get or add + /// The type of the argument for the updateCondition function + /// The key at which to find or add the value + /// The json type info for the value type + /// The function used to generate a value for the key + /// An argument that could be provided to the valueFactory function to avoid a closure + /// The value after finding or adding it + /// + /// + /// This method is intentionally not atomic: under concurrent callers, may be invoked multiple times for the same . + /// The returned value is always the value produced by this invocation of . + /// The final stored value is last-writer-wins, since it is persisted via . + /// + /// + /// If you need single-invocation semantics for (e.g. the factory has side-effects or is expensive), guard the call site with a keyed lock. + /// + /// + public async ValueTask GetOrAddAsync(string key, JsonTypeInfo jsonTypeInfo, Func> valueFactory, TArg factoryArgument) { + if (Lookup.TryGetValue(key, out var source)) { + return JsonSerializer.Deserialize(new ReadOnlySpan(source), jsonTypeInfo)!; + } + var val = await valueFactory(key, factoryArgument); + Upsert(key, val, jsonTypeInfo); + return val; + } +} \ No newline at end of file diff --git a/src/ArrowDbCore/ArrowDb.IDictionaryAccessor.cs b/src/ArrowDbCore/ArrowDb.IDictionaryAccessor.cs index b17b369..9a9e776 100644 --- a/src/ArrowDbCore/ArrowDb.IDictionaryAccessor.cs +++ b/src/ArrowDbCore/ArrowDb.IDictionaryAccessor.cs @@ -1,37 +1,37 @@ namespace ArrowDbCore; public partial class ArrowDb { - /// - /// Provides an interface that unifies methods of upserting values to ArrowDb - /// - /// - private interface IDictionaryAccessor where TKey : allows ref struct { - /// - /// Assigns the to the in - /// - /// The ArrowDb instance - /// The key to use - /// The value to add/update - void Upsert(ArrowDb instance, TKey key, byte[] value); - } + /// + /// Provides an interface that unifies methods of upserting values to ArrowDb + /// + /// + private interface IDictionaryAccessor where TKey : allows ref struct { + /// + /// Assigns the to the in + /// + /// The ArrowDb instance + /// The key to use + /// The value to add/update + void Upsert(ArrowDb instance, TKey key, byte[] value); + } - /// - /// Implements by using the source dictionary directly - /// - private readonly ref struct StringAccessor : IDictionaryAccessor { - /// - public void Upsert(ArrowDb instance, string key, byte[] value) { - instance.Source[key] = value; - } - } + /// + /// Implements by using the source dictionary directly + /// + private readonly ref struct StringAccessor : IDictionaryAccessor { + /// + public void Upsert(ArrowDb instance, string key, byte[] value) { + instance.Source[key] = value; + } + } - /// - /// Implements by using the lookup - /// + /// + /// Implements by using the lookup + /// private readonly ref struct ReadOnlySpanAccessor : IDictionaryAccessor> { - /// + /// public void Upsert(ArrowDb instance, ReadOnlySpan key, byte[] value) { - instance.Lookup[key] = value; - } + instance.Lookup[key] = value; + } } -} +} \ No newline at end of file diff --git a/src/ArrowDbCore/ArrowDb.Read.cs b/src/ArrowDbCore/ArrowDb.Read.cs index 9d01550..61b0d5b 100644 --- a/src/ArrowDbCore/ArrowDb.Read.cs +++ b/src/ArrowDbCore/ArrowDb.Read.cs @@ -4,37 +4,37 @@ namespace ArrowDbCore; public partial class ArrowDb { - /// - /// Returns the number of entries in the database - /// - public int Count => Source.Count; + /// + /// Returns the number of entries in the database + /// + public int Count => Source.Count; - /// - /// Checks if the database contains the specified key - /// - /// The key to search for - /// True if the key exists, false otherwise - public bool ContainsKey(ReadOnlySpan key) => Lookup.ContainsKey(key); + /// + /// Checks if the database contains the specified key + /// + /// The key to search for + /// True if the key exists, false otherwise + public bool ContainsKey(ReadOnlySpan key) => Lookup.ContainsKey(key); - /// - /// Tries to read and parse a value of the database - /// - /// The type of the value to read - /// The key to search for - /// The json type info for the value type - /// The resulting value - /// True if the value exists and was parsed successfully, false otherwise - public bool TryGetValue(ReadOnlySpan key, JsonTypeInfo jsonTypeInfo, out TValue value) { - if (!Lookup.TryGetValue(key, out byte[]? existingReference)) { - value = default!; - return false; - } - value = JsonSerializer.Deserialize(new ReadOnlySpan(existingReference), jsonTypeInfo)!; - return !EqualityComparer.Default.Equals(value, default); - } + /// + /// Tries to read and parse a value of the database + /// + /// The type of the value to read + /// The key to search for + /// The json type info for the value type + /// The resulting value + /// True if the value exists and was parsed successfully, false otherwise + public bool TryGetValue(ReadOnlySpan key, JsonTypeInfo jsonTypeInfo, out TValue value) { + if (!Lookup.TryGetValue(key, out byte[]? existingReference)) { + value = default!; + return false; + } + value = JsonSerializer.Deserialize(new ReadOnlySpan(existingReference), jsonTypeInfo)!; + return typeof(TValue).IsValueType || value is not null; + } - /// - /// Returns a read-only collection of the database keys - /// - public ICollection Keys => Source.Keys; -} + /// + /// Returns a read-only collection of the database keys + /// + public ICollection Keys => Source.Keys; +} \ No newline at end of file diff --git a/src/ArrowDbCore/ArrowDb.Remove.cs b/src/ArrowDbCore/ArrowDb.Remove.cs index ae84c40..6b2cc62 100644 --- a/src/ArrowDbCore/ArrowDb.Remove.cs +++ b/src/ArrowDbCore/ArrowDb.Remove.cs @@ -1,29 +1,41 @@ namespace ArrowDbCore; public partial class ArrowDb { - /// - /// Tries to remove the specified key from the database - /// - /// The key to remove - /// True if the key was removed, false otherwise - public bool TryRemove(ReadOnlySpan key) { - WaitIfSerializing(); // block if the database is currently serializing - var removed = Lookup.TryRemove(key, out byte[]? _); - if (removed) { - OnChangeInternal(ArrowDbChangeEventArgs.Remove); // trigger change event - } - return removed; - } + /// + /// Tries to remove the specified key from the database + /// + /// The key to remove + /// True if the key was removed, false otherwise + public bool TryRemove(ReadOnlySpan key) { + var observedEpoch = Volatile.Read(ref StateEpoch); + WaitIfSerializing(); // block if the database is currently serializing + var removed = Lookup.TryRemove(key, out byte[]? _); + if (removed) { + OnChangeInternal(ArrowDbChangeEventArgs.Remove); // trigger change event + } + return removed && Volatile.Read(ref StateEpoch) == observedEpoch; + } - /// - /// Clears the database - /// - public void Clear() { - if (Source.IsEmpty) { - return; - } - WaitIfSerializing(); // block if the database is currently serializing - Source.Clear(); - OnChangeInternal(ArrowDbChangeEventArgs.Clear); // trigger change event - } -} + /// + /// Tries to clear the database + /// + /// True if the clear was completed without a concurrent rollback, false otherwise + public bool TryClear() { + if (Source.IsEmpty) { + return true; + } + var observedEpoch = Volatile.Read(ref StateEpoch); + WaitIfSerializing(); // block if the database is currently serializing + Source.Clear(); + OnChangeInternal(ArrowDbChangeEventArgs.Clear); // trigger change event + return Volatile.Read(ref StateEpoch) == observedEpoch; + } + + /// + /// Clears the database + /// + [Obsolete("Use TryClear() instead. Clear() ignores rollback races and cannot signal if a concurrent RollbackAsync occurred.")] + public void Clear() { + _ = TryClear(); + } +} \ No newline at end of file diff --git a/src/ArrowDbCore/ArrowDb.Serialization.cs b/src/ArrowDbCore/ArrowDb.Serialization.cs index 1eb4382..75a0202 100644 --- a/src/ArrowDbCore/ArrowDb.Serialization.cs +++ b/src/ArrowDbCore/ArrowDb.Serialization.cs @@ -3,49 +3,51 @@ namespace ArrowDbCore; public partial class ArrowDb { - /// - /// Serializes the database - /// - /// - /// If there are no pending updates, this method does nothing, otherwise it serializes the database and resets the pending updates counter - /// - public async Task SerializeAsync() { - if (Interlocked.Read(ref _pendingChanges) == 0) { - return; - } - try { - await Semaphore.WaitAsync(); - await Serializer.SerializeAsync(Source); - Interlocked.Exchange(ref _pendingChanges, 0); // reset pending changes - } finally { - Semaphore.Release(); - } - } + /// + /// Serializes the database + /// + /// + /// If there are no pending updates, this method does nothing, otherwise it serializes the database and resets the pending updates counter + /// + public async Task SerializeAsync() { + if (Interlocked.Read(ref _pendingChanges) == 0) { + return; + } + try { + await Semaphore.WaitAsync(); + var observedPendingChanges = Interlocked.Read(ref _pendingChanges); + await Serializer.SerializeAsync(Source); + Interlocked.CompareExchange(ref _pendingChanges, 0, observedPendingChanges); // reset pending changes only if unchanged + } finally { + Semaphore.Release(); + } + } - /// - /// Waits for the semaphore if the database is currently serializing - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void WaitIfSerializing() { - if (Semaphore.CurrentCount == 0) { - Semaphore.Wait(); - Semaphore.Release(); - } - } + /// + /// Waits for the semaphore if the database is currently serializing + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void WaitIfSerializing() { + if (Semaphore.CurrentCount == 0) { + Semaphore.Wait(); + Semaphore.Release(); + } + } - /// - /// Rolls the entire database to the last persisted state - /// - public async Task RollbackAsync() { - try { - await Semaphore.WaitAsync(); - var prevState = await Serializer.DeserializeAsync(); - Source.Clear(); - Interlocked.Exchange(ref Source, prevState); - Lookup = Source.GetAlternateLookup>(); - Interlocked.Exchange(ref _pendingChanges, 0); - } finally { - Semaphore.Release(); - } - } -} + /// + /// Rolls the entire database to the last persisted state + /// + public async Task RollbackAsync() { + try { + await Semaphore.WaitAsync(); + Interlocked.Increment(ref StateEpoch); + var prevState = await Serializer.DeserializeAsync(); + Source.Clear(); + Interlocked.Exchange(ref Source, prevState); + Lookup = Source.GetAlternateLookup>(); + Interlocked.Exchange(ref _pendingChanges, 0); + } finally { + Semaphore.Release(); + } + } +} \ No newline at end of file diff --git a/src/ArrowDbCore/ArrowDb.Upsert.cs b/src/ArrowDbCore/ArrowDb.Upsert.cs index 89a2e9c..83cbf78 100644 --- a/src/ArrowDbCore/ArrowDb.Upsert.cs +++ b/src/ArrowDbCore/ArrowDb.Upsert.cs @@ -5,73 +5,74 @@ namespace ArrowDbCore; public partial class ArrowDb { - /// - /// Upsert the specified key with the specified value into the database. - /// - /// The type of the value to upsert - /// The key at which to upsert the value - /// The value to upsert. This cannot be null. - /// The json type info for the value type - /// True if the value was upserted, false if the provided value was null. - public bool Upsert(string key, TValue value, JsonTypeInfo jsonTypeInfo) { - return UpsertCore(key, value, jsonTypeInfo, default); - } + /// + /// Upsert the specified key with the specified value into the database. + /// + /// The type of the value to upsert + /// The key at which to upsert the value + /// The value to upsert. This cannot be null. + /// The json type info for the value type + /// True if the value was upserted, false if the provided value was null. + public bool Upsert(string key, TValue value, JsonTypeInfo jsonTypeInfo) { + return UpsertCore(key, value, jsonTypeInfo, default); + } - /// - /// Upsert the specified key with the specified value into the database. - /// - /// The type of the value to upsert - /// The key at which to upsert the value - /// The value to upsert. This cannot be null. - /// The json type info for the value type - /// True if the value was upserted, false if the provided value was null. - /// - /// This method overload which uses ReadOnlySpan{char} will not allocate a new string for the key if it already exists, instead it will directly replace the value. - /// - public bool Upsert(ReadOnlySpan key, TValue value, JsonTypeInfo jsonTypeInfo) { - return UpsertCore, TValue, ReadOnlySpanAccessor>(key, value, jsonTypeInfo, default); - } + /// + /// Upsert the specified key with the specified value into the database. + /// + /// The type of the value to upsert + /// The key at which to upsert the value + /// The value to upsert. This cannot be null. + /// The json type info for the value type + /// True if the value was upserted, false if the provided value was null. + /// + /// This method overload which uses ReadOnlySpan{char} will not allocate a new string for the key if it already exists, instead it will directly replace the value. + /// + public bool Upsert(ReadOnlySpan key, TValue value, JsonTypeInfo jsonTypeInfo) { + return UpsertCore, TValue, ReadOnlySpanAccessor>(key, value, jsonTypeInfo, default); + } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool UpsertCore(TKey key, TValue value, JsonTypeInfo jsonTypeInfo, TAccessor accessor) - where TKey : allows ref struct - where TAccessor : IDictionaryAccessor, allows ref struct { - if (value is null) { - return false; - } - WaitIfSerializing(); // Block if serializing - byte[] utf8Value = JsonSerializer.SerializeToUtf8Bytes(value, jsonTypeInfo); - accessor.Upsert(this, key, utf8Value); - OnChangeInternal(ArrowDbChangeEventArgs.Upsert); // Trigger change event - return true; - } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool UpsertCore(TKey key, TValue value, JsonTypeInfo jsonTypeInfo, TAccessor accessor) + where TKey : allows ref struct + where TAccessor : IDictionaryAccessor, allows ref struct { + if (value is null) { + return false; + } + var observedEpoch = Volatile.Read(ref StateEpoch); + WaitIfSerializing(); // Block if serializing + byte[] utf8Value = JsonSerializer.SerializeToUtf8Bytes(value, jsonTypeInfo); + accessor.Upsert(this, key, utf8Value); + OnChangeInternal(ArrowDbChangeEventArgs.Upsert); // Trigger change event + return Volatile.Read(ref StateEpoch) == observedEpoch; + } - /// - /// Tries to upsert the specified key with the specified value into the database. - /// - /// The type of the value to upsert - /// The key at which to upsert the value - /// The value to upsert. This cannot be null. - /// The json type info for the value type - /// A conditional check that determines whether this update should be performed - /// True if the value was upserted, false otherwise. - /// - /// - /// The operation will be rejected if the provided is null. - /// - /// - /// can be used to resolve write conflicts, the update will be rejected only if both conditions are met: - /// - /// 1. A value for the specified key exists and successfully deserialized to - /// 2. on the reference value returns false - /// - public bool Upsert(string key, TValue value, JsonTypeInfo jsonTypeInfo, Func updateCondition) { - if (TryGetValue(key, jsonTypeInfo, out TValue existingReference) && - !updateCondition(existingReference)) { - return false; - } - return Upsert(key, value, jsonTypeInfo); - } + /// + /// Tries to upsert the specified key with the specified value into the database. + /// + /// The type of the value to upsert + /// The key at which to upsert the value + /// The value to upsert. This cannot be null. + /// The json type info for the value type + /// A conditional check that determines whether this update should be performed + /// True if the value was upserted, false otherwise. + /// + /// + /// The operation will be rejected if the provided is null. + /// + /// + /// can be used to resolve write conflicts, the update will be rejected only if both conditions are met: + /// + /// 1. A value for the specified key exists and successfully deserialized to + /// 2. on the reference value returns false + /// + public bool Upsert(string key, TValue value, JsonTypeInfo jsonTypeInfo, Func updateCondition) { + if (TryGetValue(key, jsonTypeInfo, out TValue existingReference) && + !updateCondition(existingReference)) { + return false; + } + return Upsert(key, value, jsonTypeInfo); + } /// /// Tries to upsert the specified key with the specified value into the database. @@ -95,72 +96,72 @@ public bool Upsert(string key, TValue value, JsonTypeInfo jsonTy /// 2. on the reference value returns false /// public bool Upsert(string key, TValue value, JsonTypeInfo jsonTypeInfo, Func updateCondition, TArg updateConditionArgument) { - if (TryGetValue(key, jsonTypeInfo, out TValue existingReference) && - !updateCondition(existingReference, updateConditionArgument)) { - return false; - } - return Upsert(key, value, jsonTypeInfo); - } + if (TryGetValue(key, jsonTypeInfo, out TValue existingReference) && + !updateCondition(existingReference, updateConditionArgument)) { + return false; + } + return Upsert(key, value, jsonTypeInfo); + } - /// - /// Tries to upsert the specified key with the specified value into the database. - /// - /// The type of the value to upsert - /// The key at which to upsert the value - /// The value to upsert. This cannot be null. - /// The json type info for the value type - /// A conditional check that determines whether this update should be performed - /// True if the value was upserted, false otherwise. - /// - /// - /// The operation will be rejected if the provided is null. - /// - /// - /// can be used to resolve write conflicts, the update will be rejected only if both conditions are met: - /// - /// 1. A value for the specified key exists and successfully deserialized to - /// 2. on the reference value returns false - /// - /// This method overload which uses ReadOnlySpan{char} will not allocate a new string for the key if it already exists, instead it will directly replace the value - /// - /// - public bool Upsert(ReadOnlySpan key, TValue value, JsonTypeInfo jsonTypeInfo, Func updateCondition) { - if (TryGetValue(key, jsonTypeInfo, out TValue existingReference) && - !updateCondition(existingReference)) { - return false; - } - return Upsert(key, value, jsonTypeInfo); - } + /// + /// Tries to upsert the specified key with the specified value into the database. + /// + /// The type of the value to upsert + /// The key at which to upsert the value + /// The value to upsert. This cannot be null. + /// The json type info for the value type + /// A conditional check that determines whether this update should be performed + /// True if the value was upserted, false otherwise. + /// + /// + /// The operation will be rejected if the provided is null. + /// + /// + /// can be used to resolve write conflicts, the update will be rejected only if both conditions are met: + /// + /// 1. A value for the specified key exists and successfully deserialized to + /// 2. on the reference value returns false + /// + /// This method overload which uses ReadOnlySpan{char} will not allocate a new string for the key if it already exists, instead it will directly replace the value + /// + /// + public bool Upsert(ReadOnlySpan key, TValue value, JsonTypeInfo jsonTypeInfo, Func updateCondition) { + if (TryGetValue(key, jsonTypeInfo, out TValue existingReference) && + !updateCondition(existingReference)) { + return false; + } + return Upsert(key, value, jsonTypeInfo); + } - /// - /// Tries to upsert the specified key with the specified value into the database. - /// - /// The type of the value to upsert - /// The type of the argument for the updateCondition function - /// The key at which to upsert the value - /// The value to upsert. This cannot be null. - /// The json type info for the value type - /// A conditional check that determines whether this update should be performed - /// An argument that could be provided to the updateCondition function to avoid a closure - /// True if the value was upserted, false otherwise. - /// - /// - /// The operation will be rejected if the provided is null. - /// - /// - /// can be used to resolve write conflicts, the update will be rejected only if both conditions are met: - /// - /// 1. A value for the specified key exists and successfully deserialized to - /// 2. on the reference value returns false - /// - /// This method overload which uses ReadOnlySpan{char} will not allocate a new string for the key if it already exists, instead it will directly replace the value - /// - /// - public bool Upsert(ReadOnlySpan key, TValue value, JsonTypeInfo jsonTypeInfo, Func updateCondition, TArg updateConditionArgument) { - if (TryGetValue(key, jsonTypeInfo, out TValue existingReference) && - !updateCondition(existingReference, updateConditionArgument)) { - return false; - } - return Upsert(key, value, jsonTypeInfo); - } -} + /// + /// Tries to upsert the specified key with the specified value into the database. + /// + /// The type of the value to upsert + /// The type of the argument for the updateCondition function + /// The key at which to upsert the value + /// The value to upsert. This cannot be null. + /// The json type info for the value type + /// A conditional check that determines whether this update should be performed + /// An argument that could be provided to the updateCondition function to avoid a closure + /// True if the value was upserted, false otherwise. + /// + /// + /// The operation will be rejected if the provided is null. + /// + /// + /// can be used to resolve write conflicts, the update will be rejected only if both conditions are met: + /// + /// 1. A value for the specified key exists and successfully deserialized to + /// 2. on the reference value returns false + /// + /// This method overload which uses ReadOnlySpan{char} will not allocate a new string for the key if it already exists, instead it will directly replace the value + /// + /// + public bool Upsert(ReadOnlySpan key, TValue value, JsonTypeInfo jsonTypeInfo, Func updateCondition, TArg updateConditionArgument) { + if (TryGetValue(key, jsonTypeInfo, out TValue existingReference) && + !updateCondition(existingReference, updateConditionArgument)) { + return false; + } + return Upsert(key, value, jsonTypeInfo); + } +} \ No newline at end of file diff --git a/src/ArrowDbCore/ArrowDb.cs b/src/ArrowDbCore/ArrowDb.cs index 9d09fb3..5fef6f8 100644 --- a/src/ArrowDbCore/ArrowDb.cs +++ b/src/ArrowDbCore/ArrowDb.cs @@ -7,46 +7,46 @@ namespace ArrowDbCore; /// /// Initialize via the factory methods public sealed partial class ArrowDb { - /// - /// Returns the number of active instances - /// - public static long RunningInstances => Interlocked.Read(ref s_runningInstances); - - /// - /// Tracks the number of running instances - /// - private static long s_runningInstances; - - /// - /// The backing dictionary - /// - internal volatile ConcurrentDictionary Source; - - /// - /// The alternate lookup - /// - internal ConcurrentDictionary.AlternateLookup> Lookup; - - /// - /// The semaphore for maintaining serialization consistency - /// - internal readonly SemaphoreSlim Semaphore; - - /// - /// The serializer - /// - internal readonly IDbSerializer Serializer; - - /// - /// An event that is raised when any operation was performed that changes the database state, i.e, adding, updating, or removing a key, or clearing the database - /// - public event EventHandler? OnChange; + /// + /// Returns the number of active instances + /// + public static long RunningInstances => Interlocked.Read(ref s_runningInstances); + + /// + /// Tracks the number of running instances + /// + private static long s_runningInstances; + + /// + /// The backing dictionary + /// + internal volatile ConcurrentDictionary Source; + + /// + /// The alternate lookup + /// + internal ConcurrentDictionary.AlternateLookup> Lookup; + + /// + /// The semaphore for maintaining serialization consistency + /// + internal readonly SemaphoreSlim Semaphore; + + /// + /// The serializer + /// + internal readonly IDbSerializer Serializer; + + /// + /// An event that is raised when any operation was performed that changes the database state, i.e, adding, updating, or removing a key, or clearing the database + /// + public event EventHandler? OnChange; /// /// Raises the event /// private void OnChangeInternal(ArrowDbChangeEventArgs args) { - Interlocked.Increment(ref _pendingChanges); + Interlocked.Increment(ref _pendingChanges); OnChange?.Invoke(this, args); } @@ -55,43 +55,48 @@ private void OnChangeInternal(ArrowDbChangeEventArgs args) { /// public long PendingChanges => Interlocked.Read(ref _pendingChanges); - /// - /// Thread-safe pending changes tracker - /// - private long _pendingChanges; - - /// - /// Thread-safe transaction depth tracker - /// - internal long TransactionDepth = 0; - - /// - /// Private Ctor - /// - /// A pre-existing or empty dictionary - /// A serializer implementation - private ArrowDb(ConcurrentDictionary source, IDbSerializer serializer) { - Source = source; - Lookup = Source.GetAlternateLookup>(); - Serializer = serializer; - Interlocked.Increment(ref s_runningInstances); - Semaphore = new SemaphoreSlim(1, 1); - } - - /// - /// Finalizer (called when the instance is garbage collected) - /// - ~ArrowDb() { - Interlocked.Decrement(ref s_runningInstances); - Semaphore.Dispose(); - } - - /// - /// Returns a transaction scope that implicitly calls when disposed - /// - /// - /// The implements both and , allowing it to be used in both synchronous and asynchronous contexts. - /// - /// A new instance. - public ArrowDbTransactionScope BeginTransaction() => new(this); -} + /// + /// Thread-safe pending changes tracker + /// + private long _pendingChanges; + + /// + /// Thread-safe transaction depth tracker + /// + internal long TransactionDepth = 0; + + /// + /// A state epoch used to detect concurrent operations in hot write paths. + /// + internal long StateEpoch = 0; + + /// + /// Private Ctor + /// + /// A pre-existing or empty dictionary + /// A serializer implementation + private ArrowDb(ConcurrentDictionary source, IDbSerializer serializer) { + Source = source; + Lookup = Source.GetAlternateLookup>(); + Serializer = serializer; + Interlocked.Increment(ref s_runningInstances); + Semaphore = new SemaphoreSlim(1, 1); + } + + /// + /// Finalizer (called when the instance is garbage collected) + /// + ~ArrowDb() { + Interlocked.Decrement(ref s_runningInstances); + Semaphore.Dispose(); + } + + /// + /// Returns a transaction scope that implicitly calls when disposed + /// + /// + /// The implements both and , allowing it to be used in both synchronous and asynchronous contexts. + /// + /// A new instance. + public ArrowDbTransactionScope BeginTransaction() => new(this); +} \ No newline at end of file diff --git a/src/ArrowDbCore/ArrowDbCore.csproj b/src/ArrowDbCore/ArrowDbCore.csproj index b31e880..0cd47c4 100644 --- a/src/ArrowDbCore/ArrowDbCore.csproj +++ b/src/ArrowDbCore/ArrowDbCore.csproj @@ -1,12 +1,19 @@  - net9.0 + net9.0;net10.0 enable enable - 1.5.0 + 1.6.0 true true + true + true + latest-recommended + true + true + snupkg + true @@ -23,7 +30,6 @@ git Database;HighPerformance;NoSql;Document;KeyValuePair true - true false @@ -32,6 +38,18 @@ + + + + + + portable + true + + $(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb + + + <_Parameter1>ArrowDbCore.Tests.Unit diff --git a/src/ArrowDbCore/ArrowDbJsonContext.cs b/src/ArrowDbCore/ArrowDbJsonContext.cs index f993317..4c01e29 100644 --- a/src/ArrowDbCore/ArrowDbJsonContext.cs +++ b/src/ArrowDbCore/ArrowDbJsonContext.cs @@ -8,4 +8,4 @@ namespace ArrowDbCore; /// [JsonSourceGenerationOptions(WriteIndented = false, AllowTrailingCommas = true, NumberHandling = JsonNumberHandling.AllowReadingFromString, UseStringEnumConverter = true)] [JsonSerializable(typeof(ConcurrentDictionary))] -public partial class ArrowDbJsonContext : JsonSerializerContext; +public partial class ArrowDbJsonContext : JsonSerializerContext; \ No newline at end of file diff --git a/src/ArrowDbCore/ArrowDbTransactionScope.cs b/src/ArrowDbCore/ArrowDbTransactionScope.cs index d135440..17465f6 100644 --- a/src/ArrowDbCore/ArrowDbTransactionScope.cs +++ b/src/ArrowDbCore/ArrowDbTransactionScope.cs @@ -5,39 +5,39 @@ namespace ArrowDbCore; /// Provides a scope that can be used to defer serialization until the scope is disposed /// public sealed class ArrowDbTransactionScope : IAsyncDisposable, IDisposable { - private readonly ArrowDb _database; - private bool _disposed; + private readonly ArrowDb _database; + private bool _disposed; - /// - /// Initializes a new instance of the class. - /// - /// The database instance - internal ArrowDbTransactionScope(ArrowDb database) { - _database = database; - Interlocked.Increment(ref _database.TransactionDepth); - } + /// + /// Initializes a new instance of the class. + /// + /// The database instance + internal ArrowDbTransactionScope(ArrowDb database) { + _database = database; + Interlocked.Increment(ref _database.TransactionDepth); + } - /// - /// Disposes the scope and calls - /// - public async ValueTask DisposeAsync() { - if (_disposed) { - return; - } - if (Interlocked.Decrement(ref _database.TransactionDepth) == 0) { - await _database.SerializeAsync().ConfigureAwait(false); - } - _disposed = true; - } + /// + /// Disposes the scope and calls + /// + public async ValueTask DisposeAsync() { + if (_disposed) { + return; + } + if (Interlocked.Decrement(ref _database.TransactionDepth) == 0) { + await _database.SerializeAsync().ConfigureAwait(false); + } + _disposed = true; + } - /// - /// Disposes the scope and calls in a blocking operation - /// - public void Dispose() { - var task = DisposeAsync(); - if (task.IsCompleted) { - return; - } - task.GetAwaiter().GetResult(); - } -} + /// + /// Disposes the scope and calls in a blocking operation + /// + public void Dispose() { + var task = DisposeAsync(); + if (task.IsCompleted) { + return; + } + task.GetAwaiter().GetResult(); + } +} \ No newline at end of file diff --git a/src/ArrowDbCore/ChangeEventArgs.cs b/src/ArrowDbCore/ChangeEventArgs.cs index 098547e..f61456c 100644 --- a/src/ArrowDbCore/ChangeEventArgs.cs +++ b/src/ArrowDbCore/ChangeEventArgs.cs @@ -4,23 +4,23 @@ namespace ArrowDbCore; /// An argument that is passed to the event /// public sealed class ArrowDbChangeEventArgs : EventArgs { - /// - /// A change event that represents an upsert - /// - public static readonly ArrowDbChangeEventArgs Upsert = new(ArrowDbChangeType.Upsert); - /// - /// A change event that represents a removal - /// - public static readonly ArrowDbChangeEventArgs Remove = new(ArrowDbChangeType.Remove); - /// - /// A change event that represents a clear - /// - public static readonly ArrowDbChangeEventArgs Clear = new(ArrowDbChangeType.Clear); + /// + /// A change event that represents an upsert + /// + public static readonly ArrowDbChangeEventArgs Upsert = new(ArrowDbChangeType.Upsert); + /// + /// A change event that represents a removal + /// + public static readonly ArrowDbChangeEventArgs Remove = new(ArrowDbChangeType.Remove); + /// + /// A change event that represents a clear + /// + public static readonly ArrowDbChangeEventArgs Clear = new(ArrowDbChangeType.Clear); - /// - /// The type of change that occurred - /// - public readonly ArrowDbChangeType ChangeType; + /// + /// The type of change that occurred + /// + public readonly ArrowDbChangeType ChangeType; private ArrowDbChangeEventArgs(ArrowDbChangeType changeType) { ChangeType = changeType; @@ -31,16 +31,16 @@ private ArrowDbChangeEventArgs(ArrowDbChangeType changeType) { /// The type of change that occurred in an instance /// public enum ArrowDbChangeType { - /// - /// An upsert occurred - /// - Upsert, - /// - /// A key was removed - /// - Remove, - /// - /// The db instance was cleared (all entries were removed) - /// - Clear + /// + /// An upsert occurred + /// + Upsert, + /// + /// A key was removed + /// + Remove, + /// + /// The db instance was cleared (all entries were removed) + /// + Clear } \ No newline at end of file diff --git a/src/ArrowDbCore/Extensions.cs b/src/ArrowDbCore/Extensions.cs index c17339b..47d5bc8 100644 --- a/src/ArrowDbCore/Extensions.cs +++ b/src/ArrowDbCore/Extensions.cs @@ -8,17 +8,17 @@ namespace ArrowDbCore; /// Extension methods /// internal static class Extensions { - /// - /// Converts an input into a Hex Hash in an efficient manner - /// - /// - internal static string ToSHA256Hash(string input) { - var inputLength = Encoding.UTF8.GetMaxByteCount(input.Length); - using var memOwner = MemoryPool.Shared.Rent(inputLength); - Span span = memOwner.Memory.Span; - int written = Encoding.UTF8.GetBytes(input, span); - Span hashBuffer = stackalloc byte[32]; - SHA256.HashData(span.Slice(0, written), hashBuffer); - return Convert.ToHexString(hashBuffer); - } + /// + /// Converts an input into a Hex Hash in an efficient manner + /// + /// + internal static string ToSHA256Hash(string input) { + var inputLength = Encoding.UTF8.GetMaxByteCount(input.Length); + using var memOwner = MemoryPool.Shared.Rent(inputLength); + Span span = memOwner.Memory.Span; + int written = Encoding.UTF8.GetBytes(input, span); + Span hashBuffer = stackalloc byte[32]; + SHA256.HashData(span.Slice(0, written), hashBuffer); + return Convert.ToHexString(hashBuffer); + } } \ No newline at end of file diff --git a/src/ArrowDbCore/IDbSerializer.cs b/src/ArrowDbCore/IDbSerializer.cs index 7729048..3352e51 100644 --- a/src/ArrowDbCore/IDbSerializer.cs +++ b/src/ArrowDbCore/IDbSerializer.cs @@ -6,14 +6,14 @@ namespace ArrowDbCore; /// The interface that defines a serializer for ArrowDb /// public interface IDbSerializer { - /// - /// Deserializes the database from the underlying storage - /// - ValueTask> DeserializeAsync(); + /// + /// Deserializes the database from the underlying storage + /// + ValueTask> DeserializeAsync(); - /// - /// Serializes the database to the underlying storage - /// - /// The data to serialize - ValueTask SerializeAsync(ConcurrentDictionary data); -} + /// + /// Serializes the database to the underlying storage + /// + /// The data to serialize + ValueTask SerializeAsync(ConcurrentDictionary data); +} \ No newline at end of file diff --git a/src/ArrowDbCore/Readme.Nuget.md b/src/ArrowDbCore/Readme.Nuget.md index 67b4a7d..b3db0de 100644 --- a/src/ArrowDbCore/Readme.Nuget.md +++ b/src/ArrowDbCore/Readme.Nuget.md @@ -2,9 +2,9 @@ A fast, lightweight, and type-safe key-value database designed for .NET. -* Super-Lightweight (dll size is <= 20KB - approximately 9X smaller than [UltraLiteDb](https://github.com/rejemy/UltraLiteDB)) -* Ultra-Fast (1,000,000 random operations / ~100ms on M2 MacBook Pro) -* Minimal-Allocation (~2KB for serialization of 1,000,000 items) +* Super-Lightweight (dll size is ~19KB - approximately 9X smaller than [UltraLiteDb](https://github.com/rejemy/UltraLiteDB)) +* Ultra-Fast (1,000,000 random operations / ~98ms on M2 MacBook Pro) +* Minimal-Allocation (constant ~520 bytes for serialization any db size) * Thread-Safe and Concurrent * ACID compliant on transaction level * Type-Safe (no reflection - compile-time enforced via source-generated `JsonSerializerContext`) @@ -18,3 +18,7 @@ A fast, lightweight, and type-safe key-value database designed for .NET. This policy does not affect value types (`structs`); their `default` values (e.g., `0` for an `int`) are considered valid. Information on usage can be found in the [README](https://github.com/dusrdev/ArrowDb/blob/stable/README.md). + +## Concurrency note: `GetOrAddAsync` + +`GetOrAddAsync` is intentionally **not atomic**. Under concurrency, the factory may be invoked multiple times for the same key, and the final stored value is last-writer-wins (because the value is persisted via `Upsert`). If you need single-invocation semantics for the factory (e.g. side-effects/expensive work), guard the call site with a keyed lock. diff --git a/src/ArrowDbCore/Serializers/AesFileSerializer.cs b/src/ArrowDbCore/Serializers/AesFileSerializer.cs index f75b7fd..5f01109 100644 --- a/src/ArrowDbCore/Serializers/AesFileSerializer.cs +++ b/src/ArrowDbCore/Serializers/AesFileSerializer.cs @@ -38,4 +38,4 @@ protected override ValueTask> DeserializeDa var res = JsonSerializer.Deserialize(cryptoStream, _jsonTypeInfo); return ValueTask.FromResult(res ?? new ConcurrentDictionary()); } -} +} \ No newline at end of file diff --git a/src/ArrowDbCore/Serializers/BaseFileSerializer.cs b/src/ArrowDbCore/Serializers/BaseFileSerializer.cs index cd30f52..798eba5 100644 --- a/src/ArrowDbCore/Serializers/BaseFileSerializer.cs +++ b/src/ArrowDbCore/Serializers/BaseFileSerializer.cs @@ -47,8 +47,7 @@ public ValueTask> DeserializeAsync() { public ValueTask SerializeAsync(ConcurrentDictionary data) { _mutex.WaitOne(); try { - using (var fileStream = File.Create(_tempFilePath)) - { + using (var fileStream = File.Create(_tempFilePath)) { SerializeData(fileStream, data); } File.Move(_tempFilePath, _dbFilePath, true); diff --git a/src/ArrowDbCore/Serializers/FileSerializer.cs b/src/ArrowDbCore/Serializers/FileSerializer.cs index 390239a..e72db82 100644 --- a/src/ArrowDbCore/Serializers/FileSerializer.cs +++ b/src/ArrowDbCore/Serializers/FileSerializer.cs @@ -30,4 +30,4 @@ protected override ValueTask> DeserializeDa var result = JsonSerializer.Deserialize(stream, _jsonTypeInfo) ?? new(); return ValueTask.FromResult(result); } -} +} \ No newline at end of file diff --git a/src/ArrowDbCore/Serializers/InMemorySerializer.cs b/src/ArrowDbCore/Serializers/InMemorySerializer.cs index da46964..46225bb 100644 --- a/src/ArrowDbCore/Serializers/InMemorySerializer.cs +++ b/src/ArrowDbCore/Serializers/InMemorySerializer.cs @@ -16,4 +16,4 @@ public sealed class InMemorySerializer : IDbSerializer { /// Does nothing /// public ValueTask SerializeAsync(ConcurrentDictionary data) => ValueTask.CompletedTask; -} +} \ No newline at end of file diff --git a/tests/ArrowDbCore.Tests.Analyzers/ArrowDbCore.Tests.Analyzers.csproj b/tests/ArrowDbCore.Tests.Analyzers/ArrowDbCore.Tests.Analyzers.csproj index 0b9b75f..0fef34e 100644 --- a/tests/ArrowDbCore.Tests.Analyzers/ArrowDbCore.Tests.Analyzers.csproj +++ b/tests/ArrowDbCore.Tests.Analyzers/ArrowDbCore.Tests.Analyzers.csproj @@ -2,7 +2,7 @@ Exe - net9.0 + net10.0 enable enable true diff --git a/tests/ArrowDbCore.Tests.Common/ArrowDbCore.Tests.Common.csproj b/tests/ArrowDbCore.Tests.Common/ArrowDbCore.Tests.Common.csproj index 125f4c9..b760144 100644 --- a/tests/ArrowDbCore.Tests.Common/ArrowDbCore.Tests.Common.csproj +++ b/tests/ArrowDbCore.Tests.Common/ArrowDbCore.Tests.Common.csproj @@ -1,7 +1,7 @@  - net9.0 + net10.0 enable enable diff --git a/tests/ArrowDbCore.Tests.Common/Person.cs b/tests/ArrowDbCore.Tests.Common/Person.cs index c0789a3..6b5eef6 100644 --- a/tests/ArrowDbCore.Tests.Common/Person.cs +++ b/tests/ArrowDbCore.Tests.Common/Person.cs @@ -5,4 +5,4 @@ public class Person { public int Age { get; set; } public DateTime BirthDate { get; set; } public bool IsMarried { get; set; } -} +} \ No newline at end of file diff --git a/tests/ArrowDbCore.Tests.Integrity/ArrowDbCore.Tests.Integrity.csproj b/tests/ArrowDbCore.Tests.Integrity/ArrowDbCore.Tests.Integrity.csproj index 4213f81..625bf21 100644 --- a/tests/ArrowDbCore.Tests.Integrity/ArrowDbCore.Tests.Integrity.csproj +++ b/tests/ArrowDbCore.Tests.Integrity/ArrowDbCore.Tests.Integrity.csproj @@ -4,7 +4,7 @@ enable enable Exe - net9.0 + net10.0 true true diff --git a/tests/ArrowDbCore.Tests.Integrity/LargeFile.cs b/tests/ArrowDbCore.Tests.Integrity/LargeFile.cs index d1b55e3..92e88b3 100644 --- a/tests/ArrowDbCore.Tests.Integrity/LargeFile.cs +++ b/tests/ArrowDbCore.Tests.Integrity/LargeFile.cs @@ -24,7 +24,7 @@ private static async Task LargeFile_Passes_OneReadWriteCycle(string path, Funcenable Exe ArrowDbCore.Tests.Unit.Isolated - net9.0 + net10.0 true true diff --git a/tests/ArrowDbCore.Tests.Unit.Isolated/StaticVariables.cs b/tests/ArrowDbCore.Tests.Unit.Isolated/StaticVariables.cs index 898ef61..222eaa6 100644 --- a/tests/ArrowDbCore.Tests.Unit.Isolated/StaticVariables.cs +++ b/tests/ArrowDbCore.Tests.Unit.Isolated/StaticVariables.cs @@ -13,4 +13,4 @@ public async Task Instance_Ids_Match_Running() { } Assert.Equal(count, ArrowDb.RunningInstances); } -} +} \ No newline at end of file diff --git a/tests/ArrowDbCore.Tests.Unit/ArrowDbCore.Tests.Unit.csproj b/tests/ArrowDbCore.Tests.Unit/ArrowDbCore.Tests.Unit.csproj index b64e834..09eae78 100644 --- a/tests/ArrowDbCore.Tests.Unit/ArrowDbCore.Tests.Unit.csproj +++ b/tests/ArrowDbCore.Tests.Unit/ArrowDbCore.Tests.Unit.csproj @@ -5,7 +5,7 @@ enable Exe ArrowDbCore.Tests.Unit - net9.0 + net10.0 true true diff --git a/tests/ArrowDbCore.Tests.Unit/Concurrency.cs b/tests/ArrowDbCore.Tests.Unit/Concurrency.cs index d8b8a35..5af99f7 100644 --- a/tests/ArrowDbCore.Tests.Unit/Concurrency.cs +++ b/tests/ArrowDbCore.Tests.Unit/Concurrency.cs @@ -39,4 +39,4 @@ private async Task CreateDb(string path, bool useAes, Aes? aes = null) return await ArrowDb.CreateFromFile(path); } -} +} \ No newline at end of file diff --git a/tests/ArrowDbCore.Tests.Unit/OnChange.cs b/tests/ArrowDbCore.Tests.Unit/OnChange.cs index 88c50e8..d5cac95 100644 --- a/tests/ArrowDbCore.Tests.Unit/OnChange.cs +++ b/tests/ArrowDbCore.Tests.Unit/OnChange.cs @@ -32,7 +32,7 @@ public async Task OnChange_Clear_Shows_Expected_Change() { db.OnChange += (_, args) => { change = args.ChangeType; }; - db.Clear(); + Assert.True(db.TryClear()); Assert.Equal(ArrowDbChangeType.Clear, change); } } \ No newline at end of file diff --git a/tests/ArrowDbCore.Tests.Unit/Reads.cs b/tests/ArrowDbCore.Tests.Unit/Reads.cs index 1d1078d..c23794a 100644 --- a/tests/ArrowDbCore.Tests.Unit/Reads.cs +++ b/tests/ArrowDbCore.Tests.Unit/Reads.cs @@ -31,4 +31,14 @@ public async Task TryGetValue_WrongType_ThrowsJsonException() { // TryGetValue should throw JsonException as deserialization into incorrect type should fail Assert.Throws(() => db.TryGetValue("ron", JContext.Default.Int32, out _)); } + + [Fact] + public async Task TryGetValue_Struct_ReturnsTrueForDefault() { + var db = await ArrowDb.CreateInMemory(); + Assert.Equal(0, db.Count); + db.Upsert("int", 0, JContext.Default.Int32); + // db should contain the key as a value was upserted + Assert.True(db.TryGetValue("int", JContext.Default.Int32, out int val)); + Assert.Equal(default, val); + } } \ No newline at end of file diff --git a/tests/ArrowDbCore.Tests.Unit/Removes.cs b/tests/ArrowDbCore.Tests.Unit/Removes.cs index 3bd1e29..1ffedc1 100644 --- a/tests/ArrowDbCore.Tests.Unit/Removes.cs +++ b/tests/ArrowDbCore.Tests.Unit/Removes.cs @@ -52,7 +52,7 @@ public async Task Clear_Removes_All_From_Db() { db.Upsert("1", 1, JContext.Default.Int32); db.Upsert("2", 2, JContext.Default.Int32); Assert.Equal(2, db.Count); - db.Clear(); + Assert.True(db.TryClear()); Assert.False(db.ContainsKey("1")); Assert.False(db.ContainsKey("2")); Assert.False(db.TryGetValue("1", JContext.Default.Int32, out _)); diff --git a/tests/ArrowDbCore.Tests.Unit/RollbackRace.cs b/tests/ArrowDbCore.Tests.Unit/RollbackRace.cs new file mode 100644 index 0000000..242d31e --- /dev/null +++ b/tests/ArrowDbCore.Tests.Unit/RollbackRace.cs @@ -0,0 +1,138 @@ +using System.Collections.Concurrent; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace ArrowDbCore.Tests.Unit; + +public class RollbackRace { + [Fact] + public async Task Upsert_WhenRacingWithRollback_EitherPersistsOrSignalsFailure() { + var serializer = new RollbackRaceBlockingSerializer(); + var db = await ArrowDb.CreateCustom(serializer); + + var upsertCommitted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + db.OnChange += (_, args) => { + if (args.ChangeType == ArrowDbChangeType.Upsert) { + upsertCommitted.TrySetResult(); + } + }; + + var hooks = new RollbackRaceHooks(); + RollbackRaceValueConverter.Hooks.Value = hooks; + + try { + Task upsertTask = Task.Run(() => db.Upsert( + "k", + new RollbackRaceValue { X = 1 }, + RollbackRaceJsonContext.Default.RollbackRaceValue)); + + // Ensure Upsert has already passed WaitIfSerializing() and is now blocked inside JSON serialization. + await hooks.UpsertReachedValueSerialization.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + serializer.BlockNextDeserialize(); + Task rollbackTask = db.RollbackAsync(); + + // Ensure rollback acquired the semaphore and is blocked in DeserializeAsync(). + await serializer.RollbackDeserializeStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + // Allow Upsert to proceed to dictionary mutation while rollback is in progress. + hooks.AllowUpsertToProceed.Set(); + await upsertCommitted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + // Now allow rollback to return an empty state, causing a Source swap that can drop the just-written key. + serializer.AllowRollbackDeserializeToReturn.TrySetResult(); + + bool upserted = await upsertTask; + await rollbackTask; + + // Desired contract for a future "write-gate" (epoch) solution: + // If the write is not reliable due to concurrent rollback, the operation should report failure. + // Today, this can be violated (Upsert returns true but the key is dropped by rollback). + Assert.True(db.ContainsKey("k") || !upserted); + } finally { + RollbackRaceValueConverter.Hooks.Value = null; + } + } +} + +internal sealed class RollbackRaceBlockingSerializer : IDbSerializer { + private int _blockNextDeserialize; + + public readonly TaskCompletionSource RollbackDeserializeStarted = new(TaskCreationOptions.RunContinuationsAsynchronously); + public readonly TaskCompletionSource AllowRollbackDeserializeToReturn = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public void BlockNextDeserialize() => Interlocked.Exchange(ref _blockNextDeserialize, 1); + + public ValueTask> DeserializeAsync() { + if (Interlocked.Exchange(ref _blockNextDeserialize, 0) == 0) { + return ValueTask.FromResult(new ConcurrentDictionary()); + } + + RollbackDeserializeStarted.TrySetResult(); + return new ValueTask>(WaitAndReturnEmptyAsync()); + } + + private async Task> WaitAndReturnEmptyAsync() { + await AllowRollbackDeserializeToReturn.Task.ConfigureAwait(false); + return new ConcurrentDictionary(); + } + + public ValueTask SerializeAsync(ConcurrentDictionary data) => ValueTask.CompletedTask; +} + +internal sealed class RollbackRaceHooks { + public readonly TaskCompletionSource UpsertReachedValueSerialization = new(TaskCreationOptions.RunContinuationsAsynchronously); + public readonly ManualResetEventSlim AllowUpsertToProceed = new(false); +} + +[JsonConverter(typeof(RollbackRaceValueConverter))] +internal sealed class RollbackRaceValue { + public int X { get; set; } +} + +internal sealed class RollbackRaceValueConverter : JsonConverter { + public static readonly AsyncLocal Hooks = new(); + + public override RollbackRaceValue Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + if (reader.TokenType != JsonTokenType.StartObject) { + throw new JsonException("Expected StartObject."); + } + int x = 0; + while (reader.Read()) { + if (reader.TokenType == JsonTokenType.EndObject) { + return new RollbackRaceValue { X = x }; + } + if (reader.TokenType != JsonTokenType.PropertyName) { + throw new JsonException("Expected PropertyName."); + } + string propertyName = reader.GetString() ?? string.Empty; + if (!reader.Read()) { + throw new JsonException("Unexpected end of JSON."); + } + if (propertyName == "x") { + x = reader.GetInt32(); + } else { + reader.Skip(); + } + } + throw new JsonException("Unexpected end of JSON."); + } + + public override void Write(Utf8JsonWriter writer, RollbackRaceValue value, JsonSerializerOptions options) { + RollbackRaceHooks? hooks = Hooks.Value; + if (hooks is not null) { + hooks.UpsertReachedValueSerialization.TrySetResult(); + if (!hooks.AllowUpsertToProceed.Wait(TimeSpan.FromSeconds(5))) { + throw new TimeoutException("Timed out waiting for test to allow value serialization to proceed."); + } + } + + writer.WriteStartObject(); + writer.WriteNumber("x", value.X); + writer.WriteEndObject(); + } +} + +[JsonSourceGenerationOptions(WriteIndented = false)] +[JsonSerializable(typeof(RollbackRaceValue))] +internal partial class RollbackRaceJsonContext : JsonSerializerContext { } \ No newline at end of file diff --git a/tests/ArrowDbCore.Tests.Unit/Serialization.cs b/tests/ArrowDbCore.Tests.Unit/Serialization.cs index b5064bf..a0af9c6 100644 --- a/tests/ArrowDbCore.Tests.Unit/Serialization.cs +++ b/tests/ArrowDbCore.Tests.Unit/Serialization.cs @@ -117,7 +117,7 @@ private static async Task File_Serializes_And_Rollback_As_Expected(string path, Assert.Equal(1, db.PendingChanges); await db.SerializeAsync(); // clear the db (critical change) - db.Clear(); + Assert.True(db.TryClear()); Assert.False(db.ContainsKey("1")); Assert.Equal(0, db.Count); Assert.Equal(1, db.PendingChanges); diff --git a/tests/ArrowDbCore.Tests.Unit/SerializationPendingChanges.cs b/tests/ArrowDbCore.Tests.Unit/SerializationPendingChanges.cs new file mode 100644 index 0000000..7f5ec38 --- /dev/null +++ b/tests/ArrowDbCore.Tests.Unit/SerializationPendingChanges.cs @@ -0,0 +1,125 @@ +using System.Collections.Concurrent; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace ArrowDbCore.Tests.Unit; + +public class SerializationPendingChanges { + [Fact] + public async Task SerializeAsync_WhenChangeHappensDuringSerialization_DoesNotClearPendingChanges() { + var serializer = new BlockingSerializer(); + var db = await ArrowDb.CreateCustom(serializer); + + var secondUpsertCommitted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + int upsertEvents = 0; + db.OnChange += (_, args) => { + if (args.ChangeType == ArrowDbChangeType.Upsert) { + if (Interlocked.Increment(ref upsertEvents) == 2) { + secondUpsertCommitted.TrySetResult(); + } + } + }; + + // Ensure SerializeAsync doesn't early-exit: it is a no-op when PendingChanges == 0. + Assert.True(db.Upsert("seed", new PendingChangesDuringSerializeValue { X = 0 }, PendingChangesDuringSerializeJsonContext.Default.PendingChangesDuringSerializeValue)); + + var hooks = new PendingChangesDuringSerializeHooks(); + PendingChangesDuringSerializeValueConverter.Hooks.Value = hooks; + + try { + Task upsertTask = Task.Run(() => db.Upsert("k", new PendingChangesDuringSerializeValue { X = 1 }, PendingChangesDuringSerializeJsonContext.Default.PendingChangesDuringSerializeValue)); + + await hooks.UpsertReachedValueSerialization.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + Task serializeTask = db.SerializeAsync(); + await serializer.SerializeStarted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + hooks.AllowUpsertToProceed.Set(); + await secondUpsertCommitted.Task.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); + + serializer.AllowSerializeToFinish.TrySetResult(); + + bool upserted = await upsertTask; + await serializeTask; + + Assert.True(upserted); + Assert.True(db.PendingChanges > 0); + } finally { + PendingChangesDuringSerializeValueConverter.Hooks.Value = null; + } + } + + private sealed class BlockingSerializer : IDbSerializer { + public readonly TaskCompletionSource SerializeStarted = new(TaskCreationOptions.RunContinuationsAsynchronously); + public readonly TaskCompletionSource AllowSerializeToFinish = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public ValueTask> DeserializeAsync() { + return ValueTask.FromResult(new ConcurrentDictionary()); + } + + public ValueTask SerializeAsync(ConcurrentDictionary data) { + SerializeStarted.TrySetResult(); + return new ValueTask(AllowSerializeToFinish.Task); + } + } + +} + +// These types are intentionally top-level so System.Text.Json source generation runs correctly. + +internal sealed class PendingChangesDuringSerializeHooks { + public readonly TaskCompletionSource UpsertReachedValueSerialization = new(TaskCreationOptions.RunContinuationsAsynchronously); + public readonly ManualResetEventSlim AllowUpsertToProceed = new(false); +} + +[JsonConverter(typeof(PendingChangesDuringSerializeValueConverter))] +internal sealed class PendingChangesDuringSerializeValue { + public int X { get; set; } +} + +internal sealed class PendingChangesDuringSerializeValueConverter : JsonConverter { + public static readonly AsyncLocal Hooks = new(); + + public override PendingChangesDuringSerializeValue Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + if (reader.TokenType != JsonTokenType.StartObject) { + throw new JsonException("Expected StartObject."); + } + int x = 0; + while (reader.Read()) { + if (reader.TokenType == JsonTokenType.EndObject) { + return new PendingChangesDuringSerializeValue { X = x }; + } + if (reader.TokenType != JsonTokenType.PropertyName) { + throw new JsonException("Expected PropertyName."); + } + string propertyName = reader.GetString() ?? string.Empty; + if (!reader.Read()) { + throw new JsonException("Unexpected end of JSON."); + } + if (propertyName == "x") { + x = reader.GetInt32(); + } else { + reader.Skip(); + } + } + throw new JsonException("Unexpected end of JSON."); + } + + public override void Write(Utf8JsonWriter writer, PendingChangesDuringSerializeValue value, JsonSerializerOptions options) { + PendingChangesDuringSerializeHooks? hooks = Hooks.Value; + if (hooks is not null) { + hooks.UpsertReachedValueSerialization.TrySetResult(); + if (!hooks.AllowUpsertToProceed.Wait(TimeSpan.FromSeconds(5))) { + throw new TimeoutException("Timed out waiting for test to allow value serialization to proceed."); + } + } + + writer.WriteStartObject(); + writer.WriteNumber("x", value.X); + writer.WriteEndObject(); + } +} + +[JsonSourceGenerationOptions(WriteIndented = false)] +[JsonSerializable(typeof(PendingChangesDuringSerializeValue))] +internal partial class PendingChangesDuringSerializeJsonContext : JsonSerializerContext { } \ No newline at end of file diff --git a/tests/ArrowDbCore.Tests.Unit/Transactions.cs b/tests/ArrowDbCore.Tests.Unit/Transactions.cs index e0bbd72..055bfcc 100644 --- a/tests/ArrowDbCore.Tests.Unit/Transactions.cs +++ b/tests/ArrowDbCore.Tests.Unit/Transactions.cs @@ -76,4 +76,4 @@ private async Task CreateDb(string path, bool useAes, Aes? aes = null) return await ArrowDb.CreateFromFile(path); } -} +} \ No newline at end of file diff --git a/tests/ArrowDbCore.Tests.Unit/Upserts.Spans.cs b/tests/ArrowDbCore.Tests.Unit/Upserts.Spans.cs index cf0d943..d87aed2 100644 --- a/tests/ArrowDbCore.Tests.Unit/Upserts.Spans.cs +++ b/tests/ArrowDbCore.Tests.Unit/Upserts.Spans.cs @@ -56,6 +56,17 @@ public async Task Conditional_Update_When_Found_And_Invalid_Returns_False() { Assert.Equal(1, value); } + [Fact] + public async Task Conditional_Update_When_Found_Default_ForValueType_StillEvaluatesCondition() { + var db = await ArrowDb.CreateInMemory(); + Assert.Equal(0, db.Count); + ReadOnlySpan key = "1"; + db.Upsert(key, 0, JContext.Default.Int32); + Assert.False(db.Upsert(key, 1, JContext.Default.Int32, reference => reference != 0)); + Assert.True(db.TryGetValue(key, JContext.Default.Int32, out var value)); + Assert.Equal(0, value); + } + [Fact] public async Task Conditional_Update_TArg_When_Not_Found_Inserts() { var db = await ArrowDb.CreateInMemory(); diff --git a/tests/ArrowDbCore.Tests.Unit/Upserts.cs b/tests/ArrowDbCore.Tests.Unit/Upserts.cs index 86d1f9a..1cd87b1 100644 --- a/tests/ArrowDbCore.Tests.Unit/Upserts.cs +++ b/tests/ArrowDbCore.Tests.Unit/Upserts.cs @@ -25,8 +25,7 @@ public async Task Upsert_When_Found_Overwrites() { } [Fact] - public async Task Upsert_NullValue_IsDisallowed() - { + public async Task Upsert_NullValue_IsDisallowed() { // Arrange var db = await ArrowDb.CreateInMemory(); @@ -66,6 +65,16 @@ public async Task Conditional_Update_When_Found_And_Invalid_Returns_False() { Assert.Equal(1, value); } + [Fact] + public async Task Conditional_Update_When_Found_Default_ForValueType_StillEvaluatesCondition() { + var db = await ArrowDb.CreateInMemory(); + Assert.Equal(0, db.Count); + db.Upsert("1", 0, JContext.Default.Int32); + Assert.False(db.Upsert("1", 1, JContext.Default.Int32, reference => reference != 0)); + Assert.True(db.TryGetValue("1", JContext.Default.Int32, out var value)); + Assert.Equal(0, value); + } + [Fact] public async Task Conditional_Update_TArg_When_Not_Found_Inserts() { var db = await ArrowDb.CreateInMemory();