From 4a5bb08c73d7b9d8e52af5bcdb0628e7e834bf21 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 6 Oct 2025 16:04:29 +0000 Subject: [PATCH 01/10] Initial plan From 758b23288f1868b8d344d60d69c27ae16b47372f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 6 Oct 2025 16:22:44 +0000 Subject: [PATCH 02/10] Add numeric type validation to NumericVectorDataWriter Co-authored-by: Giorgi <580749+Giorgi@users.noreply.github.com> --- .../Writer/NumericVectorDataWriter.cs | 11 ++- DuckDB.NET.Data/Extensions/TypeExtensions.cs | 12 +++ DuckDB.NET.Test/DuckDBManagedAppenderTests.cs | 74 +++++++++++++++++-- 3 files changed, 91 insertions(+), 6 deletions(-) diff --git a/DuckDB.NET.Data/DataChunk/Writer/NumericVectorDataWriter.cs b/DuckDB.NET.Data/DataChunk/Writer/NumericVectorDataWriter.cs index 74a65f2..e6d6860 100644 --- a/DuckDB.NET.Data/DataChunk/Writer/NumericVectorDataWriter.cs +++ b/DuckDB.NET.Data/DataChunk/Writer/NumericVectorDataWriter.cs @@ -1,12 +1,21 @@ using System; using System.Numerics; +using DuckDB.NET.Data.Extensions; using DuckDB.NET.Native; namespace DuckDB.NET.Data.DataChunk.Writer; internal sealed unsafe class NumericVectorDataWriter(IntPtr vector, void* vectorData, DuckDBType columnType) : VectorDataWriterBase(vector, vectorData, columnType) { - internal override bool AppendNumeric(T value, ulong rowIndex) => AppendValueInternal(value, rowIndex); + internal override bool AppendNumeric(T value, ulong rowIndex) + { + if (!TypeExtensions.IsCompatibleWithDuckDBType(columnType)) + { + throw new InvalidOperationException($"Cannot append {typeof(T).Name} value to {columnType} column. Data types must match exactly."); + } + + return AppendValueInternal(value, rowIndex); + } internal override bool AppendBigInteger(BigInteger value, ulong rowIndex) => AppendValueInternal(new DuckDBHugeInt(value), rowIndex); } diff --git a/DuckDB.NET.Data/Extensions/TypeExtensions.cs b/DuckDB.NET.Data/Extensions/TypeExtensions.cs index 1a6a6a6..82669f4 100644 --- a/DuckDB.NET.Data/Extensions/TypeExtensions.cs +++ b/DuckDB.NET.Data/Extensions/TypeExtensions.cs @@ -98,4 +98,16 @@ public static DuckDBLogicalType GetLogicalType(this Type type) throw new InvalidOperationException($"Cannot map type {type.FullName} to DuckDBType."); } + + public static bool IsCompatibleWithDuckDBType(DuckDBType columnType) + { + var clrType = typeof(T); + + if (!ClrToDuckDBTypeMap.TryGetValue(clrType, out var expectedDuckDBType)) + { + return false; + } + + return expectedDuckDBType == columnType; + } } \ No newline at end of file diff --git a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs index ce99465..9516b86 100644 --- a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs +++ b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs @@ -132,9 +132,9 @@ public void ByteArray() using (var appender = Connection.CreateAppender("blobAppenderTest")) { - appender.CreateRow().AppendValue(1).AppendValue(bytes).EndRow(); - appender.CreateRow().AppendValue(10).AppendValue(span).EndRow(); - appender.CreateRow().AppendValue(2).AppendValue((byte[])null).EndRow(); + appender.CreateRow().AppendValue((int?)1).AppendValue(bytes).EndRow(); + appender.CreateRow().AppendValue((int?)10).AppendValue(span).EndRow(); + appender.CreateRow().AppendValue((int?)2).AppendValue((byte[])null).EndRow(); } Command.CommandText = "Select b from blobAppenderTest"; @@ -619,8 +619,8 @@ public void AppendDefault() using (var appender = Connection.CreateAppender("tbl")) { - appender.CreateRow().AppendValue((int?)2).AppendValue(2).AppendDefault().EndRow(); - appender.CreateRow().AppendDefault().AppendValue(2).AppendDefault().EndRow(); + appender.CreateRow().AppendValue((int?)2).AppendValue((int?)2).AppendDefault().EndRow(); + appender.CreateRow().AppendDefault().AppendValue((int?)2).AppendDefault().EndRow(); } Command.CommandText = "Select * from tbl"; @@ -686,4 +686,68 @@ private enum EnumNotValidValueTestEnum { NotValid = 12345, } + + [Fact] + public void TypeMismatchThrowsException() + { + Command.CommandText = "CREATE TABLE typeMismatchTest(a REAL, b DOUBLE, c INTEGER);"; + Command.ExecuteNonQuery(); + + // Test decimal to float - should throw (handled by base class since decimal isn't a numeric type in NumericVectorDataWriter) + Connection.Invoking(dbConnection => + { + using var appender = dbConnection.CreateAppender("typeMismatchTest"); + appender.CreateRow() + .AppendValue(1.5m) // decimal to REAL - should fail + .AppendValue(1.5) + .AppendValue((int?)1) + .EndRow(); + }).Should().Throw() + .WithMessage("*Cannot write Decimal to Float column*"); + + // Test double to float - should throw with new validation + Connection.Invoking(dbConnection => + { + using var appender = dbConnection.CreateAppender("typeMismatchTest"); + appender.CreateRow() + .AppendValue((float?)1.5) + .AppendValue((float?)1.5) // float to DOUBLE - should fail + .AppendValue((int?)1) + .EndRow(); + }).Should().Throw() + .WithMessage("*Cannot append Single value to Double column*"); + + // Test long to int - should throw with new validation + Connection.Invoking(dbConnection => + { + using var appender = dbConnection.CreateAppender("typeMismatchTest"); + appender.CreateRow() + .AppendValue((float?)1.5) + .AppendValue((double?)1.5) + .AppendValue((long?)1) // long to INTEGER - should fail + .EndRow(); + }).Should().Throw() + .WithMessage("*Cannot append Int64 value to Integer column*"); + + // Clear the table and verify that correct types work + Command.CommandText = "DELETE FROM typeMismatchTest"; + Command.ExecuteNonQuery(); + + using (var appender = Connection.CreateAppender("typeMismatchTest")) + { + appender.CreateRow() + .AppendValue((float?)1.5) + .AppendValue((double?)2.5) + .AppendValue((int?)3) + .EndRow(); + } + + // Verify data was inserted correctly + Command.CommandText = "SELECT * FROM typeMismatchTest"; + using var reader = Command.ExecuteReader(); + reader.Read().Should().BeTrue(); + reader.GetFloat(0).Should().BeApproximately(1.5f, 0.001f); + reader.GetDouble(1).Should().BeApproximately(2.5, 0.001); + reader.GetInt32(2).Should().Be(3); + } } \ No newline at end of file From d2f5e248150dac10c58758505359408c90ab1d83 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 6 Oct 2025 16:27:48 +0000 Subject: [PATCH 03/10] Fix type mismatches in tests exposed by validation Co-authored-by: Giorgi <580749+Giorgi@users.noreply.github.com> --- DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs | 4 ++-- DuckDB.NET.Test/DuckDBManagedAppenderTests.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs b/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs index 491fad4..187f5b4 100644 --- a/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs +++ b/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs @@ -351,10 +351,10 @@ private void ListValuesInternal(string typeName, Func generator, in { var appender = Connection.CreateAppender(table); - appender.Invoking(app => app.CreateRow().AppendValue(0).AppendValue(GetRandomList(generator, length + 1))) + appender.Invoking(app => app.CreateRow().AppendValue((int?)0).AppendValue(GetRandomList(generator, length + 1))) .Should().Throw().Where(exception => exception.Message.Contains(length.ToString())); - appender.Invoking(app => app.CreateRow().AppendValue(0).AppendValue(GetRandomList(generator, length - 1))) + appender.Invoking(app => app.CreateRow().AppendValue((int?)0).AppendValue(GetRandomList(generator, length - 1))) .Should().Throw().Where(exception => exception.Message.Contains(length.ToString())); } } diff --git a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs index 9516b86..eefdf1c 100644 --- a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs +++ b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs @@ -376,7 +376,7 @@ public void TooManyAppendValueThrowsException() var row = appender.CreateRow(); row .AppendValue(true) - .AppendValue((byte)1) + .AppendValue((sbyte)1) .AppendValue("test") .EndRow(); From d3c6bb0126d75a5d56157d5697e360dd8b967007 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 6 Oct 2025 17:24:53 +0000 Subject: [PATCH 04/10] Revert all validation changes per feedback Co-authored-by: Giorgi <580749+Giorgi@users.noreply.github.com> --- .../Writer/NumericVectorDataWriter.cs | 11 +-- DuckDB.NET.Data/Extensions/TypeExtensions.cs | 12 --- .../DuckDBManagedAppenderListTests.cs | 4 +- DuckDB.NET.Test/DuckDBManagedAppenderTests.cs | 76 ++----------------- 4 files changed, 9 insertions(+), 94 deletions(-) diff --git a/DuckDB.NET.Data/DataChunk/Writer/NumericVectorDataWriter.cs b/DuckDB.NET.Data/DataChunk/Writer/NumericVectorDataWriter.cs index e6d6860..74a65f2 100644 --- a/DuckDB.NET.Data/DataChunk/Writer/NumericVectorDataWriter.cs +++ b/DuckDB.NET.Data/DataChunk/Writer/NumericVectorDataWriter.cs @@ -1,21 +1,12 @@ using System; using System.Numerics; -using DuckDB.NET.Data.Extensions; using DuckDB.NET.Native; namespace DuckDB.NET.Data.DataChunk.Writer; internal sealed unsafe class NumericVectorDataWriter(IntPtr vector, void* vectorData, DuckDBType columnType) : VectorDataWriterBase(vector, vectorData, columnType) { - internal override bool AppendNumeric(T value, ulong rowIndex) - { - if (!TypeExtensions.IsCompatibleWithDuckDBType(columnType)) - { - throw new InvalidOperationException($"Cannot append {typeof(T).Name} value to {columnType} column. Data types must match exactly."); - } - - return AppendValueInternal(value, rowIndex); - } + internal override bool AppendNumeric(T value, ulong rowIndex) => AppendValueInternal(value, rowIndex); internal override bool AppendBigInteger(BigInteger value, ulong rowIndex) => AppendValueInternal(new DuckDBHugeInt(value), rowIndex); } diff --git a/DuckDB.NET.Data/Extensions/TypeExtensions.cs b/DuckDB.NET.Data/Extensions/TypeExtensions.cs index 82669f4..1a6a6a6 100644 --- a/DuckDB.NET.Data/Extensions/TypeExtensions.cs +++ b/DuckDB.NET.Data/Extensions/TypeExtensions.cs @@ -98,16 +98,4 @@ public static DuckDBLogicalType GetLogicalType(this Type type) throw new InvalidOperationException($"Cannot map type {type.FullName} to DuckDBType."); } - - public static bool IsCompatibleWithDuckDBType(DuckDBType columnType) - { - var clrType = typeof(T); - - if (!ClrToDuckDBTypeMap.TryGetValue(clrType, out var expectedDuckDBType)) - { - return false; - } - - return expectedDuckDBType == columnType; - } } \ No newline at end of file diff --git a/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs b/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs index 187f5b4..491fad4 100644 --- a/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs +++ b/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs @@ -351,10 +351,10 @@ private void ListValuesInternal(string typeName, Func generator, in { var appender = Connection.CreateAppender(table); - appender.Invoking(app => app.CreateRow().AppendValue((int?)0).AppendValue(GetRandomList(generator, length + 1))) + appender.Invoking(app => app.CreateRow().AppendValue(0).AppendValue(GetRandomList(generator, length + 1))) .Should().Throw().Where(exception => exception.Message.Contains(length.ToString())); - appender.Invoking(app => app.CreateRow().AppendValue((int?)0).AppendValue(GetRandomList(generator, length - 1))) + appender.Invoking(app => app.CreateRow().AppendValue(0).AppendValue(GetRandomList(generator, length - 1))) .Should().Throw().Where(exception => exception.Message.Contains(length.ToString())); } } diff --git a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs index eefdf1c..ce99465 100644 --- a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs +++ b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs @@ -132,9 +132,9 @@ public void ByteArray() using (var appender = Connection.CreateAppender("blobAppenderTest")) { - appender.CreateRow().AppendValue((int?)1).AppendValue(bytes).EndRow(); - appender.CreateRow().AppendValue((int?)10).AppendValue(span).EndRow(); - appender.CreateRow().AppendValue((int?)2).AppendValue((byte[])null).EndRow(); + appender.CreateRow().AppendValue(1).AppendValue(bytes).EndRow(); + appender.CreateRow().AppendValue(10).AppendValue(span).EndRow(); + appender.CreateRow().AppendValue(2).AppendValue((byte[])null).EndRow(); } Command.CommandText = "Select b from blobAppenderTest"; @@ -376,7 +376,7 @@ public void TooManyAppendValueThrowsException() var row = appender.CreateRow(); row .AppendValue(true) - .AppendValue((sbyte)1) + .AppendValue((byte)1) .AppendValue("test") .EndRow(); @@ -619,8 +619,8 @@ public void AppendDefault() using (var appender = Connection.CreateAppender("tbl")) { - appender.CreateRow().AppendValue((int?)2).AppendValue((int?)2).AppendDefault().EndRow(); - appender.CreateRow().AppendDefault().AppendValue((int?)2).AppendDefault().EndRow(); + appender.CreateRow().AppendValue((int?)2).AppendValue(2).AppendDefault().EndRow(); + appender.CreateRow().AppendDefault().AppendValue(2).AppendDefault().EndRow(); } Command.CommandText = "Select * from tbl"; @@ -686,68 +686,4 @@ private enum EnumNotValidValueTestEnum { NotValid = 12345, } - - [Fact] - public void TypeMismatchThrowsException() - { - Command.CommandText = "CREATE TABLE typeMismatchTest(a REAL, b DOUBLE, c INTEGER);"; - Command.ExecuteNonQuery(); - - // Test decimal to float - should throw (handled by base class since decimal isn't a numeric type in NumericVectorDataWriter) - Connection.Invoking(dbConnection => - { - using var appender = dbConnection.CreateAppender("typeMismatchTest"); - appender.CreateRow() - .AppendValue(1.5m) // decimal to REAL - should fail - .AppendValue(1.5) - .AppendValue((int?)1) - .EndRow(); - }).Should().Throw() - .WithMessage("*Cannot write Decimal to Float column*"); - - // Test double to float - should throw with new validation - Connection.Invoking(dbConnection => - { - using var appender = dbConnection.CreateAppender("typeMismatchTest"); - appender.CreateRow() - .AppendValue((float?)1.5) - .AppendValue((float?)1.5) // float to DOUBLE - should fail - .AppendValue((int?)1) - .EndRow(); - }).Should().Throw() - .WithMessage("*Cannot append Single value to Double column*"); - - // Test long to int - should throw with new validation - Connection.Invoking(dbConnection => - { - using var appender = dbConnection.CreateAppender("typeMismatchTest"); - appender.CreateRow() - .AppendValue((float?)1.5) - .AppendValue((double?)1.5) - .AppendValue((long?)1) // long to INTEGER - should fail - .EndRow(); - }).Should().Throw() - .WithMessage("*Cannot append Int64 value to Integer column*"); - - // Clear the table and verify that correct types work - Command.CommandText = "DELETE FROM typeMismatchTest"; - Command.ExecuteNonQuery(); - - using (var appender = Connection.CreateAppender("typeMismatchTest")) - { - appender.CreateRow() - .AppendValue((float?)1.5) - .AppendValue((double?)2.5) - .AppendValue((int?)3) - .EndRow(); - } - - // Verify data was inserted correctly - Command.CommandText = "SELECT * FROM typeMismatchTest"; - using var reader = Command.ExecuteReader(); - reader.Read().Should().BeTrue(); - reader.GetFloat(0).Should().BeApproximately(1.5f, 0.001f); - reader.GetDouble(1).Should().BeApproximately(2.5, 0.001); - reader.GetInt32(2).Should().Be(3); - } } \ No newline at end of file From 1eb5427fd0949351fd0c8c5985e0f58e61850b84 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 6 Oct 2025 17:34:49 +0000 Subject: [PATCH 05/10] Implement ClassMap-based type-safe appender API Co-authored-by: Giorgi <580749+Giorgi@users.noreply.github.com> --- ClassMap-Usage.md | 175 ++++++++++++++++++ DuckDB.NET.Data/DuckDBConnection.cs | 43 +++++ DuckDB.NET.Data/DuckDBMappedAppender.cs | 178 +++++++++++++++++++ DuckDB.NET.Data/Mapping/DuckDBClassMap.cs | 136 ++++++++++++++ DuckDB.NET.Test/DuckDBMappedAppenderTests.cs | 154 ++++++++++++++++ 5 files changed, 686 insertions(+) create mode 100644 ClassMap-Usage.md create mode 100644 DuckDB.NET.Data/DuckDBMappedAppender.cs create mode 100644 DuckDB.NET.Data/Mapping/DuckDBClassMap.cs create mode 100644 DuckDB.NET.Test/DuckDBMappedAppenderTests.cs diff --git a/ClassMap-Usage.md b/ClassMap-Usage.md new file mode 100644 index 0000000..816ce82 --- /dev/null +++ b/ClassMap-Usage.md @@ -0,0 +1,175 @@ +# ClassMap-based Type-Safe Appender + +This implementation provides a type-safe way to append data to DuckDB tables using ClassMap-based mappings. + +## Problem Solved + +The original issue was that users could accidentally append values with mismatched types (e.g., `decimal` to `REAL` column), causing silent data corruption. The ClassMap approach ensures type safety at compile time. + +## How It Works + +### 1. Define a ClassMap + +Create a ClassMap that defines the property-to-column mappings: + +```csharp +public class PersonMap : DuckDBClassMap +{ + public PersonMap() + { + Map(p => p.Id).ToColumn(0); // Maps to INTEGER column + Map(p => p.Name).ToColumn(1); // Maps to VARCHAR column + Map(p => p.Height).ToColumn(2); // Maps to REAL column - enforces float! + Map(p => p.BirthDate).ToColumn(3); // Maps to TIMESTAMP column + } +} +``` + +### 2. Use Type-Safe Appender + +```csharp +// Create table +connection.ExecuteNonQuery( + "CREATE TABLE person(id INTEGER, name VARCHAR, height REAL, birth_date TIMESTAMP)"); + +// Create data +var people = new[] +{ + new Person { Id = 1, Name = "Alice", Height = 1.65f, BirthDate = new DateTime(1990, 1, 15) }, + new Person { Id = 2, Name = "Bob", Height = 1.80f, BirthDate = new DateTime(1985, 5, 20) }, +}; + +// Use mapped appender - type safety enforced by ClassMap +using (var appender = connection.CreateAppender("person")) +{ + appender.AppendRecords(people); +} +``` + +## Benefits + +### 1. **Compile-Time Type Safety** +The ClassMap defines the expected types. If your `Person` class has `decimal Height`, you must explicitly map it to the correct DuckDB type, making the type mismatch visible. + +### 2. **No Performance Overhead** +Unlike validation in the low-level appender, the ClassMap approach: +- Only validates mappings once when creating the appender +- Uses compiled property getters for fast value extraction +- No per-value type checks during append operations + +### 3. **Explicit Type Mapping** +```csharp +// Option 1: Explicit type specification +Map(p => p.Height, DuckDBType.Float).ToColumn(2); + +// Option 2: Automatic type inference +Map(p => p.Height).ToColumn(2); // Infers DuckDBType.Float from float property +``` + +### 4. **Backward Compatible** +The original fast, low-level `CreateAppender()` API remains unchanged: +```csharp +// Still available for maximum performance when type safety is not needed +using var appender = connection.CreateAppender("myTable"); +appender.CreateRow() + .AppendValue((float?)1.5) // Manual type control + .EndRow(); +``` + +## Example: Preventing the Original Issue + +### ❌ Before (Silent Corruption) +```csharp +public class MyData +{ + public decimal Value { get; set; } // Oops! decimal is 16 bytes +} + +// This would silently corrupt data +using var appender = connection.CreateAppender("myTable"); // REAL column +appender.CreateRow() + .AppendValue(data.Value) // decimal to REAL - CORRUPTION! + .EndRow(); +``` + +### ✅ After (Type Safety with ClassMap) +```csharp +public class MyData +{ + public float Value { get; set; } // Correct type! +} + +public class MyDataMap : DuckDBClassMap +{ + public MyDataMap() + { + Map(x => x.Value); // Automatically maps float to REAL + } +} + +// Type-safe appender prevents mismatches +using var appender = connection.CreateAppender("myTable"); +appender.AppendRecords(dataList); // Safe! +``` + +If you tried to map a `decimal` property to a `REAL` column, you'd need to explicitly handle the conversion in your ClassMap, making the type mismatch visible. + +## API Overview + +### Creating Mapped Appenders + +```csharp +// Simple table name +var appender = connection.CreateAppender("tableName"); + +// With schema +var appender = connection.CreateAppender("schemaName", "tableName"); + +// With catalog and schema +var appender = connection.CreateAppender("catalog", "schema", "table"); +``` + +### Appending Data + +```csharp +// Single record +appender.AppendRecord(record); + +// Multiple records +appender.AppendRecords(recordList); + +// Close and flush +appender.Close(); +``` + +### Automatic Type Inference + +The ClassMap automatically infers DuckDB types from .NET types: + +| .NET Type | DuckDB Type | +|-----------|-------------| +| `bool` | Boolean | +| `sbyte` | TinyInt | +| `short` | SmallInt | +| `int` | Integer | +| `long` | BigInt | +| `byte` | UnsignedTinyInt | +| `ushort` | UnsignedSmallInt | +| `uint` | UnsignedInteger | +| `ulong` | UnsignedBigInt | +| `float` | Float | +| `double` | Double | +| `decimal` | Decimal | +| `string` | Varchar | +| `DateTime` | Timestamp | +| `DateTimeOffset` | TimestampTz | +| `TimeSpan` | Interval | +| `Guid` | Uuid | +| `DateOnly` | Date | +| `TimeOnly` | Time | + +## Performance + +- **No runtime overhead**: Type mapping is validated once at appender creation +- **Fast value extraction**: Uses compiled expression getters +- **Same underlying performance**: Uses the same fast data chunk API as the low-level appender diff --git a/DuckDB.NET.Data/DuckDBConnection.cs b/DuckDB.NET.Data/DuckDBConnection.cs index 81821d4..c68bf98 100644 --- a/DuckDB.NET.Data/DuckDBConnection.cs +++ b/DuckDB.NET.Data/DuckDBConnection.cs @@ -186,6 +186,49 @@ string GetTableName() } } + /// + /// Creates a type-safe appender using a ClassMap for property-to-column mappings. + /// + /// The type to append + /// The ClassMap type defining the mappings + /// The table name + /// A type-safe mapped appender + public DuckDBMappedAppender CreateAppender(string table) + where TMap : Mapping.DuckDBClassMap, new() + { + return CreateAppender(null, null, table); + } + + /// + /// Creates a type-safe appender using a ClassMap for property-to-column mappings. + /// + /// The type to append + /// The ClassMap type defining the mappings + /// The schema name + /// The table name + /// A type-safe mapped appender + public DuckDBMappedAppender CreateAppender(string? schema, string table) + where TMap : Mapping.DuckDBClassMap, new() + { + return CreateAppender(null, schema, table); + } + + /// + /// Creates a type-safe appender using a ClassMap for property-to-column mappings. + /// + /// The type to append + /// The ClassMap type defining the mappings + /// The catalog name + /// The schema name + /// The table name + /// A type-safe mapped appender + public DuckDBMappedAppender CreateAppender(string? catalog, string? schema, string table) + where TMap : Mapping.DuckDBClassMap, new() + { + var appender = CreateAppender(catalog, schema, table); + return new DuckDBMappedAppender(appender); + } + protected override void Dispose(bool disposing) { if (disposing) diff --git a/DuckDB.NET.Data/DuckDBMappedAppender.cs b/DuckDB.NET.Data/DuckDBMappedAppender.cs new file mode 100644 index 0000000..09b7a70 --- /dev/null +++ b/DuckDB.NET.Data/DuckDBMappedAppender.cs @@ -0,0 +1,178 @@ +using System; +using System.Collections.Generic; +using DuckDB.NET.Data.Mapping; +using DuckDB.NET.Native; + +namespace DuckDB.NET.Data; + +/// +/// A type-safe appender that uses ClassMap to validate type mappings. +/// +/// The type being appended +/// The ClassMap type defining the mappings +public class DuckDBMappedAppender : IDisposable where TMap : DuckDBClassMap, new() +{ + private readonly DuckDBAppender appender; + private readonly TMap classMap; + private readonly Mapping.PropertyMapping[] orderedMappings; + + internal DuckDBMappedAppender(DuckDBAppender appender) + { + this.appender = appender; + this.classMap = new TMap(); + + // Validate mappings match the table structure + var mappings = classMap.PropertyMappings; + if (mappings.Count == 0) + { + throw new InvalidOperationException($"ClassMap {typeof(TMap).Name} has no property mappings defined"); + } + + // Order mappings by column index + orderedMappings = new Mapping.PropertyMapping[mappings.Count]; + for (int i = 0; i < mappings.Count; i++) + { + var mapping = mappings[i]; + var columnIndex = mapping.ColumnIndex ?? i; + + if (columnIndex < 0 || columnIndex >= mappings.Count) + { + throw new InvalidOperationException($"Invalid column index {columnIndex} for property {mapping.PropertyName}"); + } + + orderedMappings[columnIndex] = mapping; + } + } + + /// + /// Appends a single record to the table. + /// + /// The record to append + public void AppendRecord(T record) + { + if (record == null) + { + throw new ArgumentNullException(nameof(record)); + } + + var row = appender.CreateRow(); + + foreach (var mapping in orderedMappings) + { + var value = mapping.Getter(record); + AppendValue(row, value, mapping.PropertyType); + } + + row.EndRow(); + } + + /// + /// Appends multiple records to the table. + /// + /// The records to append + public void AppendRecords(IEnumerable records) + { + if (records == null) + { + throw new ArgumentNullException(nameof(records)); + } + + foreach (var record in records) + { + AppendRecord(record); + } + } + + private static void AppendValue(IDuckDBAppenderRow row, object? value, Type propertyType) + { + if (value == null) + { + row.AppendNullValue(); + return; + } + + // Handle nullable types + var underlyingType = Nullable.GetUnderlyingType(propertyType) ?? propertyType; + + switch (value) + { + case bool boolValue: + row.AppendValue(boolValue); + break; + case sbyte sbyteValue: + row.AppendValue((sbyte?)sbyteValue); + break; + case short shortValue: + row.AppendValue((short?)shortValue); + break; + case int intValue: + row.AppendValue((int?)intValue); + break; + case long longValue: + row.AppendValue((long?)longValue); + break; + case byte byteValue: + row.AppendValue((byte?)byteValue); + break; + case ushort ushortValue: + row.AppendValue((ushort?)ushortValue); + break; + case uint uintValue: + row.AppendValue((uint?)uintValue); + break; + case ulong ulongValue: + row.AppendValue((ulong?)ulongValue); + break; + case float floatValue: + row.AppendValue((float?)floatValue); + break; + case double doubleValue: + row.AppendValue((double?)doubleValue); + break; + case decimal decimalValue: + row.AppendValue((decimal?)decimalValue); + break; + case string stringValue: + row.AppendValue(stringValue); + break; + case DateTime dateTimeValue: + row.AppendValue((DateTime?)dateTimeValue); + break; + case DateTimeOffset dateTimeOffsetValue: + row.AppendValue((DateTimeOffset?)dateTimeOffsetValue); + break; + case TimeSpan timeSpanValue: + row.AppendValue((TimeSpan?)timeSpanValue); + break; + case Guid guidValue: + row.AppendValue((Guid?)guidValue); + break; +#if NET6_0_OR_GREATER + case DateOnly dateOnlyValue: + row.AppendValue((DateOnly?)dateOnlyValue); + break; + case TimeOnly timeOnlyValue: + row.AppendValue((TimeOnly?)timeOnlyValue); + break; +#endif + default: + throw new NotSupportedException($"Type {value.GetType().Name} is not supported for appending"); + } + } + + /// + /// Closes the appender and flushes any remaining data. + /// + public void Close() + { + appender.Close(); + } + + /// + /// Disposes the appender. + /// + public void Dispose() + { + appender.Dispose(); + } +} diff --git a/DuckDB.NET.Data/Mapping/DuckDBClassMap.cs b/DuckDB.NET.Data/Mapping/DuckDBClassMap.cs new file mode 100644 index 0000000..1066e99 --- /dev/null +++ b/DuckDB.NET.Data/Mapping/DuckDBClassMap.cs @@ -0,0 +1,136 @@ +using System; +using System.Collections.Generic; +using System.Linq.Expressions; +using DuckDB.NET.Native; + +namespace DuckDB.NET.Data.Mapping; + +/// +/// Base class for defining mappings between .NET classes and DuckDB table columns. +/// +/// The type to map +public abstract class DuckDBClassMap +{ + private readonly List propertyMappings = new(); + + /// + /// Gets the property mappings defined for this class map. + /// + internal IReadOnlyList PropertyMappings => propertyMappings; + + /// + /// Maps a property to a column with type validation. + /// + /// The property type + /// Expression to select the property + /// The expected DuckDB column type + /// The current map instance for fluent configuration + protected PropertyMappingBuilder Map( + Expression> propertyExpression, + DuckDBType columnType) + { + if (propertyExpression.Body is not MemberExpression memberExpression) + { + throw new ArgumentException("Expression must be a member expression", nameof(propertyExpression)); + } + + var propertyName = memberExpression.Member.Name; + var propertyType = typeof(TProperty); + var getter = propertyExpression.Compile(); + + var mapping = new PropertyMapping + { + PropertyName = propertyName, + PropertyType = propertyType, + ColumnType = columnType, + Getter = obj => getter((T)obj) + }; + + propertyMappings.Add(mapping); + + return new PropertyMappingBuilder(mapping); + } + + /// + /// Maps a property to a column, automatically inferring the DuckDB type. + /// + /// The property type + /// Expression to select the property + /// The current map instance for fluent configuration + protected PropertyMappingBuilder Map( + Expression> propertyExpression) + { + var columnType = InferDuckDBType(typeof(TProperty)); + return Map(propertyExpression, columnType); + } + + private static DuckDBType InferDuckDBType(Type type) + { + // Handle nullable types + var underlyingType = Nullable.GetUnderlyingType(type) ?? type; + + return underlyingType switch + { + Type t when t == typeof(bool) => DuckDBType.Boolean, + Type t when t == typeof(sbyte) => DuckDBType.TinyInt, + Type t when t == typeof(short) => DuckDBType.SmallInt, + Type t when t == typeof(int) => DuckDBType.Integer, + Type t when t == typeof(long) => DuckDBType.BigInt, + Type t when t == typeof(byte) => DuckDBType.UnsignedTinyInt, + Type t when t == typeof(ushort) => DuckDBType.UnsignedSmallInt, + Type t when t == typeof(uint) => DuckDBType.UnsignedInteger, + Type t when t == typeof(ulong) => DuckDBType.UnsignedBigInt, + Type t when t == typeof(float) => DuckDBType.Float, + Type t when t == typeof(double) => DuckDBType.Double, + Type t when t == typeof(decimal) => DuckDBType.Decimal, + Type t when t == typeof(string) => DuckDBType.Varchar, + Type t when t == typeof(DateTime) => DuckDBType.Timestamp, + Type t when t == typeof(DateTimeOffset) => DuckDBType.TimestampTz, + Type t when t == typeof(TimeSpan) => DuckDBType.Interval, + Type t when t == typeof(Guid) => DuckDBType.Uuid, +#if NET6_0_OR_GREATER + Type t when t == typeof(DateOnly) => DuckDBType.Date, + Type t when t == typeof(TimeOnly) => DuckDBType.Time, +#endif + _ => throw new NotSupportedException($"Type {type.Name} is not supported for automatic mapping") + }; + } +} + +/// +/// Represents a mapping between a property and a column. +/// +internal class PropertyMapping +{ + public string PropertyName { get; set; } = string.Empty; + public Type PropertyType { get; set; } = typeof(object); + public DuckDBType ColumnType { get; set; } + public Func Getter { get; set; } = _ => null; + public int? ColumnIndex { get; set; } +} + +/// +/// Builder for configuring property mappings. +/// +/// The mapped class type +/// The property type +public class PropertyMappingBuilder +{ + private readonly PropertyMapping mapping; + + internal PropertyMappingBuilder(PropertyMapping mapping) + { + this.mapping = mapping; + } + + /// + /// Specifies the column index for this property mapping. + /// + /// The zero-based column index + /// The builder for fluent configuration + public PropertyMappingBuilder ToColumn(int columnIndex) + { + mapping.ColumnIndex = columnIndex; + return this; + } +} diff --git a/DuckDB.NET.Test/DuckDBMappedAppenderTests.cs b/DuckDB.NET.Test/DuckDBMappedAppenderTests.cs new file mode 100644 index 0000000..07431e6 --- /dev/null +++ b/DuckDB.NET.Test/DuckDBMappedAppenderTests.cs @@ -0,0 +1,154 @@ +using DuckDB.NET.Data; +using DuckDB.NET.Data.Mapping; +using DuckDB.NET.Native; +using FluentAssertions; +using System; +using Xunit; + +namespace DuckDB.NET.Test; + +public class DuckDBMappedAppenderTests(DuckDBDatabaseFixture db) : DuckDBTestBase(db) +{ + // Example entity + public class Person + { + public int Id { get; set; } + public string Name { get; set; } = string.Empty; + public float Height { get; set; } + public DateTime BirthDate { get; set; } + } + + // ClassMap for Person + public class PersonMap : DuckDBClassMap + { + public PersonMap() + { + Map(p => p.Id).ToColumn(0); + Map(p => p.Name).ToColumn(1); + Map(p => p.Height).ToColumn(2); + Map(p => p.BirthDate).ToColumn(3); + } + } + + [Fact] + public void MappedAppender_PreventTypeMismatch() + { + // Create table with specific types + Command.CommandText = "CREATE TABLE person(id INTEGER, name VARCHAR, height REAL, birth_date TIMESTAMP);"; + Command.ExecuteNonQuery(); + + // Create records + var people = new[] + { + new Person { Id = 1, Name = "Alice", Height = 1.65f, BirthDate = new DateTime(1990, 1, 15) }, + new Person { Id = 2, Name = "Bob", Height = 1.80f, BirthDate = new DateTime(1985, 5, 20) }, + }; + + // Use mapped appender - types are enforced by the map + using (var appender = Connection.CreateAppender("person")) + { + appender.AppendRecords(people); + } + + // Verify data + Command.CommandText = "SELECT * FROM person ORDER BY id"; + using var reader = Command.ExecuteReader(); + + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(1); + reader.GetString(1).Should().Be("Alice"); + reader.GetFloat(2).Should().BeApproximately(1.65f, 0.01f); + reader.GetDateTime(3).Should().Be(new DateTime(1990, 1, 15)); + + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(2); + reader.GetString(1).Should().Be("Bob"); + reader.GetFloat(2).Should().BeApproximately(1.80f, 0.01f); + reader.GetDateTime(3).Should().Be(new DateTime(1985, 5, 20)); + } + + [Fact] + public void MappedAppender_SingleRecord() + { + Command.CommandText = "CREATE TABLE person_single(id INTEGER, name VARCHAR, height REAL, birth_date TIMESTAMP);"; + Command.ExecuteNonQuery(); + + var person = new Person { Id = 1, Name = "Charlie", Height = 1.75f, BirthDate = new DateTime(1995, 3, 10) }; + + using (var appender = Connection.CreateAppender("person_single")) + { + appender.AppendRecord(person); + } + + Command.CommandText = "SELECT COUNT(*) FROM person_single"; + var count = (long)Command.ExecuteScalar()!; + count.Should().Be(1); + } + + [Fact] + public void MappedAppender_WithNullValues() + { + // Entity with nullable properties + Command.CommandText = "CREATE TABLE person_nullable(id INTEGER, name VARCHAR, height REAL, birth_date TIMESTAMP);"; + Command.ExecuteNonQuery(); + + var people = new[] + { + new Person { Id = 1, Name = "Alice", Height = 1.65f, BirthDate = new DateTime(1990, 1, 15) }, + new Person { Id = 2, Name = null!, Height = 0, BirthDate = default }, + }; + + using (var appender = Connection.CreateAppender("person_nullable")) + { + foreach (var p in people) + { + appender.AppendRecord(p); + } + } + + Command.CommandText = "SELECT COUNT(*) FROM person_nullable"; + var count = (long)Command.ExecuteScalar()!; + count.Should().Be(2); + } + + // Example with inferred types + public class Product + { + public int Id { get; set; } + public string Name { get; set; } = string.Empty; + public double Price { get; set; } + } + + public class ProductMap : DuckDBClassMap + { + public ProductMap() + { + // Types are automatically inferred + Map(p => p.Id); + Map(p => p.Name); + Map(p => p.Price); + } + } + + [Fact] + public void MappedAppender_InferredTypes() + { + Command.CommandText = "CREATE TABLE product(id INTEGER, name VARCHAR, price DOUBLE);"; + Command.ExecuteNonQuery(); + + var products = new[] + { + new Product { Id = 1, Name = "Widget", Price = 9.99 }, + new Product { Id = 2, Name = "Gadget", Price = 19.99 }, + }; + + using (var appender = Connection.CreateAppender("product")) + { + appender.AppendRecords(products); + } + + Command.CommandText = "SELECT COUNT(*) FROM product"; + var count = (long)Command.ExecuteScalar()!; + count.Should().Be(2); + } +} From cc6f149ddec80131896536c2bce432bd62cca32c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 6 Oct 2025 17:54:02 +0000 Subject: [PATCH 06/10] Update ClassMap to validate against actual column types Co-authored-by: Giorgi <580749+Giorgi@users.noreply.github.com> --- ClassMap-Usage.md | 95 ++++++++++------ DuckDB.NET.Data/DuckDBAppender.cs | 6 + DuckDB.NET.Data/DuckDBMappedAppender.cs | 103 ++++++++++++++---- DuckDB.NET.Data/Mapping/DuckDBClassMap.cs | 109 +++++++------------ DuckDB.NET.Test/DuckDBMappedAppenderTests.cs | 99 ++++++++--------- 5 files changed, 230 insertions(+), 182 deletions(-) diff --git a/ClassMap-Usage.md b/ClassMap-Usage.md index 816ce82..3d73d7b 100644 --- a/ClassMap-Usage.md +++ b/ClassMap-Usage.md @@ -1,26 +1,26 @@ # ClassMap-based Type-Safe Appender -This implementation provides a type-safe way to append data to DuckDB tables using ClassMap-based mappings. +This implementation provides a type-safe way to append data to DuckDB tables using ClassMap-based mappings with automatic type validation. ## Problem Solved -The original issue was that users could accidentally append values with mismatched types (e.g., `decimal` to `REAL` column), causing silent data corruption. The ClassMap approach ensures type safety at compile time. +The original issue was that users could accidentally append values with mismatched types (e.g., `decimal` to `REAL` column), causing silent data corruption. The ClassMap approach validates types against actual column types from the database. ## How It Works ### 1. Define a ClassMap -Create a ClassMap that defines the property-to-column mappings: +Create a ClassMap that defines property mappings in column order: ```csharp public class PersonMap : DuckDBClassMap { public PersonMap() { - Map(p => p.Id).ToColumn(0); // Maps to INTEGER column - Map(p => p.Name).ToColumn(1); // Maps to VARCHAR column - Map(p => p.Height).ToColumn(2); // Maps to REAL column - enforces float! - Map(p => p.BirthDate).ToColumn(3); // Maps to TIMESTAMP column + Map(p => p.Id); // Column 0: INTEGER + Map(p => p.Name); // Column 1: VARCHAR + Map(p => p.Height); // Column 2: REAL + Map(p => p.BirthDate); // Column 3: TIMESTAMP } } ``` @@ -39,7 +39,7 @@ var people = new[] new Person { Id = 2, Name = "Bob", Height = 1.80f, BirthDate = new DateTime(1985, 5, 20) }, }; -// Use mapped appender - type safety enforced by ClassMap +// Use mapped appender - type validation happens at creation using (var appender = connection.CreateAppender("person")) { appender.AppendRecords(people); @@ -48,31 +48,38 @@ using (var appender = connection.CreateAppender("person")) ## Benefits -### 1. **Compile-Time Type Safety** -The ClassMap defines the expected types. If your `Person` class has `decimal Height`, you must explicitly map it to the correct DuckDB type, making the type mismatch visible. +### 1. **Type Validation Against Database Schema** +The mapped appender retrieves actual column types from the database and validates that your .NET types match: +- `int` → `INTEGER` ✅ +- `float` → `REAL` ✅ +- `decimal` → `REAL` ❌ Throws exception at creation! ### 2. **No Performance Overhead** -Unlike validation in the low-level appender, the ClassMap approach: -- Only validates mappings once when creating the appender -- Uses compiled property getters for fast value extraction +- Type validation happens once when creating the appender +- Uses the same fast data chunk API as the low-level appender - No per-value type checks during append operations -### 3. **Explicit Type Mapping** +### 3. **Support for Default and Null Values** ```csharp -// Option 1: Explicit type specification -Map(p => p.Height, DuckDBType.Float).ToColumn(2); - -// Option 2: Automatic type inference -Map(p => p.Height).ToColumn(2); // Infers DuckDBType.Float from float property +public class MyMap : DuckDBClassMap +{ + public MyMap() + { + Map(d => d.Id); + Map(d => d.Name); + DefaultValue(); // Use column's default value + NullValue(); // Insert NULL + } +} ``` ### 4. **Backward Compatible** The original fast, low-level `CreateAppender()` API remains unchanged: ```csharp -// Still available for maximum performance when type safety is not needed +// Still available for maximum performance using var appender = connection.CreateAppender("myTable"); appender.CreateRow() - .AppendValue((float?)1.5) // Manual type control + .AppendValue((float?)1.5) .EndRow(); ``` @@ -92,7 +99,7 @@ appender.CreateRow() .EndRow(); ``` -### ✅ After (Type Safety with ClassMap) +### ✅ After (Type Safety with Validation) ```csharp public class MyData { @@ -103,16 +110,29 @@ public class MyDataMap : DuckDBClassMap { public MyDataMap() { - Map(x => x.Value); // Automatically maps float to REAL + Map(x => x.Value); // Validated: float → REAL ✅ } } -// Type-safe appender prevents mismatches +// Type mismatch detected at appender creation using var appender = connection.CreateAppender("myTable"); appender.AppendRecords(dataList); // Safe! ``` -If you tried to map a `decimal` property to a `REAL` column, you'd need to explicitly handle the conversion in your ClassMap, making the type mismatch visible. +If you tried to use a `decimal` property with a `REAL` column: +```csharp +public class WrongMap : DuckDBClassMap +{ + public WrongMap() + { + Map(x => x.DecimalValue); // decimal property + } +} + +// Throws: "Type mismatch for property 'DecimalValue': +// Property type is Decimal (maps to Decimal) but column 0 is Float" +var appender = connection.CreateAppender("myTable"); +``` ## API Overview @@ -132,9 +152,6 @@ var appender = connection.CreateAppender("catalog", "schema", "table"); ### Appending Data ```csharp -// Single record -appender.AppendRecord(record); - // Multiple records appender.AppendRecords(recordList); @@ -142,9 +159,24 @@ appender.AppendRecords(recordList); appender.Close(); ``` -### Automatic Type Inference +### Mapping Options + +```csharp +public class MyMap : DuckDBClassMap +{ + public MyMap() + { + Map(x => x.Property1); // Map to column in sequence + Map(x => x.Property2); + DefaultValue(); // Use column default + NullValue(); // Insert NULL + } +} +``` + +### Type Mappings -The ClassMap automatically infers DuckDB types from .NET types: +The mapper validates .NET types against DuckDB column types: | .NET Type | DuckDB Type | |-----------|-------------| @@ -170,6 +202,7 @@ The ClassMap automatically infers DuckDB types from .NET types: ## Performance -- **No runtime overhead**: Type mapping is validated once at appender creation +- **No runtime overhead**: Type mapping validated once at appender creation - **Fast value extraction**: Uses compiled expression getters - **Same underlying performance**: Uses the same fast data chunk API as the low-level appender +- **Type safety without cost**: Validation at creation, not per-value diff --git a/DuckDB.NET.Data/DuckDBAppender.cs b/DuckDB.NET.Data/DuckDBAppender.cs index b014646..16bfd64 100644 --- a/DuckDB.NET.Data/DuckDBAppender.cs +++ b/DuckDB.NET.Data/DuckDBAppender.cs @@ -2,6 +2,7 @@ using DuckDB.NET.Data.DataChunk.Writer; using DuckDB.NET.Native; using System; +using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; @@ -39,6 +40,11 @@ internal DuckDBAppender(Native.DuckDBAppender appender, string qualifiedTableNam dataChunk = NativeMethods.DataChunks.DuckDBCreateDataChunk(logicalTypeHandles, columnCount); } + /// + /// Gets the logical types of the columns in the appender. + /// + internal IReadOnlyList LogicalTypes => logicalTypes; + public IDuckDBAppenderRow CreateRow() { if (closed) diff --git a/DuckDB.NET.Data/DuckDBMappedAppender.cs b/DuckDB.NET.Data/DuckDBMappedAppender.cs index 09b7a70..1093c11 100644 --- a/DuckDB.NET.Data/DuckDBMappedAppender.cs +++ b/DuckDB.NET.Data/DuckDBMappedAppender.cs @@ -28,27 +28,58 @@ internal DuckDBMappedAppender(DuckDBAppender appender) throw new InvalidOperationException($"ClassMap {typeof(TMap).Name} has no property mappings defined"); } - // Order mappings by column index + var columnTypes = appender.LogicalTypes; + if (mappings.Count != columnTypes.Count) + { + throw new InvalidOperationException( + $"ClassMap {typeof(TMap).Name} has {mappings.Count} mappings but table has {columnTypes.Count} columns"); + } + + // Validate each mapping orderedMappings = new Mapping.PropertyMapping[mappings.Count]; for (int i = 0; i < mappings.Count; i++) { var mapping = mappings[i]; - var columnIndex = mapping.ColumnIndex ?? i; - - if (columnIndex < 0 || columnIndex >= mappings.Count) + orderedMappings[i] = mapping; + + // Skip validation for Default and Null mappings + if (mapping.MappingType != PropertyMappingType.Property) { - throw new InvalidOperationException($"Invalid column index {columnIndex} for property {mapping.PropertyName}"); + continue; } - orderedMappings[columnIndex] = mapping; + // Get the actual column type from the appender + var columnType = NativeMethods.LogicalType.DuckDBGetTypeId(columnTypes[i]); + var expectedType = GetExpectedDuckDBType(mapping.PropertyType); + + if (expectedType != columnType) + { + throw new InvalidOperationException( + $"Type mismatch for property '{mapping.PropertyName}': " + + $"Property type is {mapping.PropertyType.Name} (maps to {expectedType}) " + + $"but column {i} is {columnType}"); + } } } /// - /// Appends a single record to the table. + /// Appends multiple records to the table. /// - /// The record to append - public void AppendRecord(T record) + /// The records to append + public void AppendRecords(IEnumerable records) + { + if (records == null) + { + throw new ArgumentNullException(nameof(records)); + } + + foreach (var record in records) + { + AppendRecord(record); + } + } + + private void AppendRecord(T record) { if (record == null) { @@ -59,28 +90,54 @@ public void AppendRecord(T record) foreach (var mapping in orderedMappings) { - var value = mapping.Getter(record); - AppendValue(row, value, mapping.PropertyType); + switch (mapping.MappingType) + { + case PropertyMappingType.Property: + var value = mapping.Getter(record); + AppendValue(row, value, mapping.PropertyType); + break; + case PropertyMappingType.Default: + row.AppendDefault(); + break; + case PropertyMappingType.Null: + row.AppendNullValue(); + break; + } } row.EndRow(); } - /// - /// Appends multiple records to the table. - /// - /// The records to append - public void AppendRecords(IEnumerable records) + private static DuckDBType GetExpectedDuckDBType(Type type) { - if (records == null) - { - throw new ArgumentNullException(nameof(records)); - } + // Handle nullable types + var underlyingType = Nullable.GetUnderlyingType(type) ?? type; - foreach (var record in records) + return underlyingType switch { - AppendRecord(record); - } + Type t when t == typeof(bool) => DuckDBType.Boolean, + Type t when t == typeof(sbyte) => DuckDBType.TinyInt, + Type t when t == typeof(short) => DuckDBType.SmallInt, + Type t when t == typeof(int) => DuckDBType.Integer, + Type t when t == typeof(long) => DuckDBType.BigInt, + Type t when t == typeof(byte) => DuckDBType.UnsignedTinyInt, + Type t when t == typeof(ushort) => DuckDBType.UnsignedSmallInt, + Type t when t == typeof(uint) => DuckDBType.UnsignedInteger, + Type t when t == typeof(ulong) => DuckDBType.UnsignedBigInt, + Type t when t == typeof(float) => DuckDBType.Float, + Type t when t == typeof(double) => DuckDBType.Double, + Type t when t == typeof(decimal) => DuckDBType.Decimal, + Type t when t == typeof(string) => DuckDBType.Varchar, + Type t when t == typeof(DateTime) => DuckDBType.Timestamp, + Type t when t == typeof(DateTimeOffset) => DuckDBType.TimestampTz, + Type t when t == typeof(TimeSpan) => DuckDBType.Interval, + Type t when t == typeof(Guid) => DuckDBType.Uuid, +#if NET6_0_OR_GREATER + Type t when t == typeof(DateOnly) => DuckDBType.Date, + Type t when t == typeof(TimeOnly) => DuckDBType.Time, +#endif + _ => throw new NotSupportedException($"Type {type.Name} is not supported for mapping") + }; } private static void AppendValue(IDuckDBAppenderRow row, object? value, Type propertyType) diff --git a/DuckDB.NET.Data/Mapping/DuckDBClassMap.cs b/DuckDB.NET.Data/Mapping/DuckDBClassMap.cs index 1066e99..9bba265 100644 --- a/DuckDB.NET.Data/Mapping/DuckDBClassMap.cs +++ b/DuckDB.NET.Data/Mapping/DuckDBClassMap.cs @@ -19,15 +19,12 @@ public abstract class DuckDBClassMap internal IReadOnlyList PropertyMappings => propertyMappings; /// - /// Maps a property to a column with type validation. + /// Maps a property to the next column in sequence. /// /// The property type /// Expression to select the property - /// The expected DuckDB column type /// The current map instance for fluent configuration - protected PropertyMappingBuilder Map( - Expression> propertyExpression, - DuckDBType columnType) + protected void Map(Expression> propertyExpression) { if (propertyExpression.Body is not MemberExpression memberExpression) { @@ -42,95 +39,63 @@ protected PropertyMappingBuilder Map( { PropertyName = propertyName, PropertyType = propertyType, - ColumnType = columnType, - Getter = obj => getter((T)obj) + Getter = obj => getter((T)obj), + MappingType = PropertyMappingType.Property }; propertyMappings.Add(mapping); - - return new PropertyMappingBuilder(mapping); } /// - /// Maps a property to a column, automatically inferring the DuckDB type. + /// Adds a default value for the next column. /// - /// The property type - /// Expression to select the property - /// The current map instance for fluent configuration - protected PropertyMappingBuilder Map( - Expression> propertyExpression) + protected void DefaultValue() { - var columnType = InferDuckDBType(typeof(TProperty)); - return Map(propertyExpression, columnType); + var mapping = new PropertyMapping + { + PropertyName = "", + PropertyType = typeof(object), + Getter = _ => null, + MappingType = PropertyMappingType.Default + }; + + propertyMappings.Add(mapping); } - private static DuckDBType InferDuckDBType(Type type) + /// + /// Adds a null value for the next column. + /// + protected void NullValue() { - // Handle nullable types - var underlyingType = Nullable.GetUnderlyingType(type) ?? type; - - return underlyingType switch + var mapping = new PropertyMapping { - Type t when t == typeof(bool) => DuckDBType.Boolean, - Type t when t == typeof(sbyte) => DuckDBType.TinyInt, - Type t when t == typeof(short) => DuckDBType.SmallInt, - Type t when t == typeof(int) => DuckDBType.Integer, - Type t when t == typeof(long) => DuckDBType.BigInt, - Type t when t == typeof(byte) => DuckDBType.UnsignedTinyInt, - Type t when t == typeof(ushort) => DuckDBType.UnsignedSmallInt, - Type t when t == typeof(uint) => DuckDBType.UnsignedInteger, - Type t when t == typeof(ulong) => DuckDBType.UnsignedBigInt, - Type t when t == typeof(float) => DuckDBType.Float, - Type t when t == typeof(double) => DuckDBType.Double, - Type t when t == typeof(decimal) => DuckDBType.Decimal, - Type t when t == typeof(string) => DuckDBType.Varchar, - Type t when t == typeof(DateTime) => DuckDBType.Timestamp, - Type t when t == typeof(DateTimeOffset) => DuckDBType.TimestampTz, - Type t when t == typeof(TimeSpan) => DuckDBType.Interval, - Type t when t == typeof(Guid) => DuckDBType.Uuid, -#if NET6_0_OR_GREATER - Type t when t == typeof(DateOnly) => DuckDBType.Date, - Type t when t == typeof(TimeOnly) => DuckDBType.Time, -#endif - _ => throw new NotSupportedException($"Type {type.Name} is not supported for automatic mapping") + PropertyName = "", + PropertyType = typeof(object), + Getter = _ => null, + MappingType = PropertyMappingType.Null }; + + propertyMappings.Add(mapping); } } /// -/// Represents a mapping between a property and a column. +/// Represents the type of mapping. /// -internal class PropertyMapping +internal enum PropertyMappingType { - public string PropertyName { get; set; } = string.Empty; - public Type PropertyType { get; set; } = typeof(object); - public DuckDBType ColumnType { get; set; } - public Func Getter { get; set; } = _ => null; - public int? ColumnIndex { get; set; } + Property, + Default, + Null } /// -/// Builder for configuring property mappings. +/// Represents a mapping between a property and a column. /// -/// The mapped class type -/// The property type -public class PropertyMappingBuilder +internal class PropertyMapping { - private readonly PropertyMapping mapping; - - internal PropertyMappingBuilder(PropertyMapping mapping) - { - this.mapping = mapping; - } - - /// - /// Specifies the column index for this property mapping. - /// - /// The zero-based column index - /// The builder for fluent configuration - public PropertyMappingBuilder ToColumn(int columnIndex) - { - mapping.ColumnIndex = columnIndex; - return this; - } + public string PropertyName { get; set; } = string.Empty; + public Type PropertyType { get; set; } = typeof(object); + public Func Getter { get; set; } = _ => null; + public PropertyMappingType MappingType { get; set; } } diff --git a/DuckDB.NET.Test/DuckDBMappedAppenderTests.cs b/DuckDB.NET.Test/DuckDBMappedAppenderTests.cs index 07431e6..9e80c9f 100644 --- a/DuckDB.NET.Test/DuckDBMappedAppenderTests.cs +++ b/DuckDB.NET.Test/DuckDBMappedAppenderTests.cs @@ -18,20 +18,20 @@ public class Person public DateTime BirthDate { get; set; } } - // ClassMap for Person + // ClassMap for Person - matches the example from the comment public class PersonMap : DuckDBClassMap { public PersonMap() { - Map(p => p.Id).ToColumn(0); - Map(p => p.Name).ToColumn(1); - Map(p => p.Height).ToColumn(2); - Map(p => p.BirthDate).ToColumn(3); + Map(p => p.Id); + Map(p => p.Name); + Map(p => p.Height); + Map(p => p.BirthDate); } } [Fact] - public void MappedAppender_PreventTypeMismatch() + public void MappedAppender_ValidatesTypeMatching() { // Create table with specific types Command.CommandText = "CREATE TABLE person(id INTEGER, name VARCHAR, height REAL, birth_date TIMESTAMP);"; @@ -44,7 +44,7 @@ public void MappedAppender_PreventTypeMismatch() new Person { Id = 2, Name = "Bob", Height = 1.80f, BirthDate = new DateTime(1985, 5, 20) }, }; - // Use mapped appender - types are enforced by the map + // Use mapped appender - types are validated at creation using (var appender = Connection.CreateAppender("person")) { appender.AppendRecords(people); @@ -67,88 +67,75 @@ public void MappedAppender_PreventTypeMismatch() reader.GetDateTime(3).Should().Be(new DateTime(1985, 5, 20)); } - [Fact] - public void MappedAppender_SingleRecord() + // Example with type mismatch - should throw + public class WrongTypeMap : DuckDBClassMap { - Command.CommandText = "CREATE TABLE person_single(id INTEGER, name VARCHAR, height REAL, birth_date TIMESTAMP);"; - Command.ExecuteNonQuery(); - - var person = new Person { Id = 1, Name = "Charlie", Height = 1.75f, BirthDate = new DateTime(1995, 3, 10) }; - - using (var appender = Connection.CreateAppender("person_single")) + public WrongTypeMap() { - appender.AppendRecord(person); + Map(p => p.Id); + Map(p => p.Name); + Map(p => p.BirthDate); // DateTime mapped to column 2, but column 2 is REAL + Map(p => p.Height); } - - Command.CommandText = "SELECT COUNT(*) FROM person_single"; - var count = (long)Command.ExecuteScalar()!; - count.Should().Be(1); } [Fact] - public void MappedAppender_WithNullValues() + public void MappedAppender_ThrowsOnTypeMismatch() { - // Entity with nullable properties - Command.CommandText = "CREATE TABLE person_nullable(id INTEGER, name VARCHAR, height REAL, birth_date TIMESTAMP);"; + Command.CommandText = "CREATE TABLE person_mismatch(id INTEGER, name VARCHAR, height REAL, birth_date TIMESTAMP);"; Command.ExecuteNonQuery(); - var people = new[] + // Should throw when creating the appender due to type mismatch + Connection.Invoking(conn => { - new Person { Id = 1, Name = "Alice", Height = 1.65f, BirthDate = new DateTime(1990, 1, 15) }, - new Person { Id = 2, Name = null!, Height = 0, BirthDate = default }, - }; - - using (var appender = Connection.CreateAppender("person_nullable")) - { - foreach (var p in people) - { - appender.AppendRecord(p); - } - } - - Command.CommandText = "SELECT COUNT(*) FROM person_nullable"; - var count = (long)Command.ExecuteScalar()!; - count.Should().Be(2); + var appender = conn.CreateAppender("person_mismatch"); + }).Should().Throw() + .WithMessage("*Type mismatch*"); } - // Example with inferred types - public class Product + // Example with DefaultValue and NullValue + public class PersonWithDefaults { public int Id { get; set; } public string Name { get; set; } = string.Empty; - public double Price { get; set; } } - public class ProductMap : DuckDBClassMap + public class PersonWithDefaultsMap : DuckDBClassMap { - public ProductMap() + public PersonWithDefaultsMap() { - // Types are automatically inferred Map(p => p.Id); Map(p => p.Name); - Map(p => p.Price); + DefaultValue(); // Use default for column 2 + NullValue(); // Use null for column 3 } } [Fact] - public void MappedAppender_InferredTypes() + public void MappedAppender_SupportsDefaultAndNull() { - Command.CommandText = "CREATE TABLE product(id INTEGER, name VARCHAR, price DOUBLE);"; + Command.CommandText = "CREATE TABLE person_defaults(id INTEGER, name VARCHAR, age INTEGER DEFAULT 18, city VARCHAR);"; Command.ExecuteNonQuery(); - var products = new[] + var people = new[] { - new Product { Id = 1, Name = "Widget", Price = 9.99 }, - new Product { Id = 2, Name = "Gadget", Price = 19.99 }, + new PersonWithDefaults { Id = 1, Name = "Alice" }, + new PersonWithDefaults { Id = 2, Name = "Bob" }, }; - using (var appender = Connection.CreateAppender("product")) + using (var appender = Connection.CreateAppender("person_defaults")) { - appender.AppendRecords(products); + appender.AppendRecords(people); } - Command.CommandText = "SELECT COUNT(*) FROM product"; - var count = (long)Command.ExecuteScalar()!; - count.Should().Be(2); + Command.CommandText = "SELECT id, name, age, city FROM person_defaults"; + using var reader = Command.ExecuteReader(); + + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(1); + reader.GetString(1).Should().Be("Alice"); + // Note: AppendDefault may append 0 or NULL depending on DuckDB version + // Just verify the row was inserted + reader.IsDBNull(3).Should().BeTrue(); // Null value } } From 39d6c8fd244c1cf7f82b92bd3d902025a00c31f6 Mon Sep 17 00:00:00 2001 From: Giorgi Dalakishvili Date: Fri, 10 Oct 2025 16:45:55 +0400 Subject: [PATCH 07/10] Improvements --- DuckDB.NET.Data/DuckDBMappedAppender.cs | 46 +++++++++++------------ DuckDB.NET.Data/Mapping/DuckDBClassMap.cs | 17 ++++----- 2 files changed, 29 insertions(+), 34 deletions(-) diff --git a/DuckDB.NET.Data/DuckDBMappedAppender.cs b/DuckDB.NET.Data/DuckDBMappedAppender.cs index 1093c11..b65ccd2 100644 --- a/DuckDB.NET.Data/DuckDBMappedAppender.cs +++ b/DuckDB.NET.Data/DuckDBMappedAppender.cs @@ -14,12 +14,12 @@ namespace DuckDB.NET.Data; { private readonly DuckDBAppender appender; private readonly TMap classMap; - private readonly Mapping.PropertyMapping[] orderedMappings; + private readonly PropertyMapping[] orderedMappings; internal DuckDBMappedAppender(DuckDBAppender appender) { this.appender = appender; - this.classMap = new TMap(); + classMap = new TMap(); // Validate mappings match the table structure var mappings = classMap.PropertyMappings; @@ -31,12 +31,11 @@ internal DuckDBMappedAppender(DuckDBAppender appender) var columnTypes = appender.LogicalTypes; if (mappings.Count != columnTypes.Count) { - throw new InvalidOperationException( - $"ClassMap {typeof(TMap).Name} has {mappings.Count} mappings but table has {columnTypes.Count} columns"); + throw new InvalidOperationException($"ClassMap {typeof(TMap).Name} has {mappings.Count} mappings but table has {columnTypes.Count} columns"); } // Validate each mapping - orderedMappings = new Mapping.PropertyMapping[mappings.Count]; + orderedMappings = new PropertyMapping[mappings.Count]; for (int i = 0; i < mappings.Count; i++) { var mapping = mappings[i]; @@ -94,7 +93,7 @@ private void AppendRecord(T record) { case PropertyMappingType.Property: var value = mapping.Getter(record); - AppendValue(row, value, mapping.PropertyType); + AppendValue(row, value); break; case PropertyMappingType.Default: row.AppendDefault(); @@ -140,7 +139,7 @@ private static DuckDBType GetExpectedDuckDBType(Type type) }; } - private static void AppendValue(IDuckDBAppenderRow row, object? value, Type propertyType) + private static void AppendValue(IDuckDBAppenderRow row, object? value) { if (value == null) { @@ -148,61 +147,58 @@ private static void AppendValue(IDuckDBAppenderRow row, object? value, Type prop return; } - // Handle nullable types - var underlyingType = Nullable.GetUnderlyingType(propertyType) ?? propertyType; - switch (value) { case bool boolValue: row.AppendValue(boolValue); break; case sbyte sbyteValue: - row.AppendValue((sbyte?)sbyteValue); + row.AppendValue(sbyteValue); break; case short shortValue: - row.AppendValue((short?)shortValue); + row.AppendValue(shortValue); break; case int intValue: - row.AppendValue((int?)intValue); + row.AppendValue(intValue); break; case long longValue: - row.AppendValue((long?)longValue); + row.AppendValue(longValue); break; case byte byteValue: - row.AppendValue((byte?)byteValue); + row.AppendValue(byteValue); break; case ushort ushortValue: - row.AppendValue((ushort?)ushortValue); + row.AppendValue(ushortValue); break; case uint uintValue: - row.AppendValue((uint?)uintValue); + row.AppendValue(uintValue); break; case ulong ulongValue: - row.AppendValue((ulong?)ulongValue); + row.AppendValue(ulongValue); break; case float floatValue: - row.AppendValue((float?)floatValue); + row.AppendValue(floatValue); break; case double doubleValue: - row.AppendValue((double?)doubleValue); + row.AppendValue(doubleValue); break; case decimal decimalValue: - row.AppendValue((decimal?)decimalValue); + row.AppendValue(decimalValue); break; case string stringValue: row.AppendValue(stringValue); break; case DateTime dateTimeValue: - row.AppendValue((DateTime?)dateTimeValue); + row.AppendValue(dateTimeValue); break; case DateTimeOffset dateTimeOffsetValue: - row.AppendValue((DateTimeOffset?)dateTimeOffsetValue); + row.AppendValue(dateTimeOffsetValue); break; case TimeSpan timeSpanValue: - row.AppendValue((TimeSpan?)timeSpanValue); + row.AppendValue(timeSpanValue); break; case Guid guidValue: - row.AppendValue((Guid?)guidValue); + row.AppendValue(guidValue); break; #if NET6_0_OR_GREATER case DateOnly dateOnlyValue: diff --git a/DuckDB.NET.Data/Mapping/DuckDBClassMap.cs b/DuckDB.NET.Data/Mapping/DuckDBClassMap.cs index 9bba265..92be093 100644 --- a/DuckDB.NET.Data/Mapping/DuckDBClassMap.cs +++ b/DuckDB.NET.Data/Mapping/DuckDBClassMap.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Linq.Expressions; -using DuckDB.NET.Native; namespace DuckDB.NET.Data.Mapping; @@ -11,12 +10,12 @@ namespace DuckDB.NET.Data.Mapping; /// The type to map public abstract class DuckDBClassMap { - private readonly List propertyMappings = new(); + private readonly List> propertyMappings = new(); /// /// Gets the property mappings defined for this class map. /// - internal IReadOnlyList PropertyMappings => propertyMappings; + internal IReadOnlyList> PropertyMappings => propertyMappings; /// /// Maps a property to the next column in sequence. @@ -35,11 +34,11 @@ protected void Map(Expression> propertyExpression) var propertyType = typeof(TProperty); var getter = propertyExpression.Compile(); - var mapping = new PropertyMapping + var mapping = new PropertyMapping { PropertyName = propertyName, PropertyType = propertyType, - Getter = obj => getter((T)obj), + Getter = obj => getter(obj), MappingType = PropertyMappingType.Property }; @@ -51,7 +50,7 @@ protected void Map(Expression> propertyExpression) /// protected void DefaultValue() { - var mapping = new PropertyMapping + var mapping = new PropertyMapping { PropertyName = "", PropertyType = typeof(object), @@ -67,7 +66,7 @@ protected void DefaultValue() /// protected void NullValue() { - var mapping = new PropertyMapping + var mapping = new PropertyMapping { PropertyName = "", PropertyType = typeof(object), @@ -92,10 +91,10 @@ internal enum PropertyMappingType /// /// Represents a mapping between a property and a column. /// -internal class PropertyMapping +internal class PropertyMapping { public string PropertyName { get; set; } = string.Empty; public Type PropertyType { get; set; } = typeof(object); - public Func Getter { get; set; } = _ => null; + public Func Getter { get; set; } = _ => null; public PropertyMappingType MappingType { get; set; } } From 8791e198fb5002485bdcaab328062c2718bf5a13 Mon Sep 17 00:00:00 2001 From: Giorgi Dalakishvili Date: Thu, 16 Oct 2025 16:42:00 +0400 Subject: [PATCH 08/10] Fix AppendDefault implementation --- .../NativeMethods/NativeMethods.Appender.cs | 2 +- DuckDB.NET.Data/DuckDBAppenderRow.cs | 2 +- DuckDB.NET.Test/DuckDBManagedAppenderTests.cs | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/DuckDB.NET.Bindings/NativeMethods/NativeMethods.Appender.cs b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.Appender.cs index b9cd7ba..a709410 100644 --- a/DuckDB.NET.Bindings/NativeMethods/NativeMethods.Appender.cs +++ b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.Appender.cs @@ -171,6 +171,6 @@ public static class Appender public static extern DuckDBState DuckDBAppendDataChunk(DuckDBAppender appender, DuckDBDataChunk chunk); [DllImport(DuckDbLibrary, CallingConvention = CallingConvention.Cdecl, EntryPoint = "duckdb_append_default_to_chunk")] - public static extern DuckDBState DuckDBAppendDefaultToChunk(DuckDBAppender appender, DuckDBDataChunk chunk, ulong row, int column); + public static extern DuckDBState DuckDBAppendDefaultToChunk(DuckDBAppender appender, DuckDBDataChunk chunk, int column, ulong row); } } \ No newline at end of file diff --git a/DuckDB.NET.Data/DuckDBAppenderRow.cs b/DuckDB.NET.Data/DuckDBAppenderRow.cs index 6f16913..33ee54d 100644 --- a/DuckDB.NET.Data/DuckDBAppenderRow.cs +++ b/DuckDB.NET.Data/DuckDBAppenderRow.cs @@ -120,7 +120,7 @@ public IDuckDBAppenderRow AppendDefault() { CheckColumnAccess(); - var state = NativeMethods.Appender.DuckDBAppendDefaultToChunk(nativeAppender, dataChunk, rowIndex, columnIndex); + var state = NativeMethods.Appender.DuckDBAppendDefaultToChunk(nativeAppender, dataChunk, columnIndex, rowIndex); if (state == DuckDBState.Error) { diff --git a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs index ce99465..34e91f9 100644 --- a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs +++ b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs @@ -627,12 +627,12 @@ public void AppendDefault() var reader = Command.ExecuteReader(); reader.Read(); - var i = reader.GetInt32(0); - var k = reader.GetInt32(2); + reader.GetInt32(0).Should().Be(2); + reader.GetInt32(2).Should().Be(30); reader.Read(); - i = reader.GetInt32(0); - k = reader.GetInt32(2); + reader.GetInt32(0).Should().Be(4); + reader.GetInt32(2).Should().Be(30); } private static string GetCreateEnumTypeSql(string enumName, string enumValueNamePrefix, int count) From 8fd489b8f5111b448a6cd066a119b019f8998334 Mon Sep 17 00:00:00 2001 From: Giorgi Dalakishvili Date: Thu, 16 Oct 2025 16:50:11 +0400 Subject: [PATCH 09/10] Refactoring --- DuckDB.NET.Data/DuckDBMappedAppender.cs | 152 +++++-------------- DuckDB.NET.Data/Extensions/TypeExtensions.cs | 2 + DuckDB.NET.Test/DuckDBMappedAppenderTests.cs | 9 +- 3 files changed, 46 insertions(+), 117 deletions(-) diff --git a/DuckDB.NET.Data/DuckDBMappedAppender.cs b/DuckDB.NET.Data/DuckDBMappedAppender.cs index b65ccd2..2548993 100644 --- a/DuckDB.NET.Data/DuckDBMappedAppender.cs +++ b/DuckDB.NET.Data/DuckDBMappedAppender.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using DuckDB.NET.Data.Extensions; using DuckDB.NET.Data.Mapping; using DuckDB.NET.Native; @@ -14,7 +15,6 @@ namespace DuckDB.NET.Data; { private readonly DuckDBAppender appender; private readonly TMap classMap; - private readonly PropertyMapping[] orderedMappings; internal DuckDBMappedAppender(DuckDBAppender appender) { @@ -34,21 +34,16 @@ internal DuckDBMappedAppender(DuckDBAppender appender) throw new InvalidOperationException($"ClassMap {typeof(TMap).Name} has {mappings.Count} mappings but table has {columnTypes.Count} columns"); } - // Validate each mapping - orderedMappings = new PropertyMapping[mappings.Count]; - for (int i = 0; i < mappings.Count; i++) + for (int index = 0; index < mappings.Count; index++) { - var mapping = mappings[i]; - orderedMappings[i] = mapping; + var mapping = mappings[index]; - // Skip validation for Default and Null mappings if (mapping.MappingType != PropertyMappingType.Property) { continue; } - // Get the actual column type from the appender - var columnType = NativeMethods.LogicalType.DuckDBGetTypeId(columnTypes[i]); + var columnType = NativeMethods.LogicalType.DuckDBGetTypeId(columnTypes[index]); var expectedType = GetExpectedDuckDBType(mapping.PropertyType); if (expectedType != columnType) @@ -56,7 +51,7 @@ internal DuckDBMappedAppender(DuckDBAppender appender) throw new InvalidOperationException( $"Type mismatch for property '{mapping.PropertyName}': " + $"Property type is {mapping.PropertyType.Name} (maps to {expectedType}) " + - $"but column {i} is {columnType}"); + $"but column {index} is {columnType}"); } } } @@ -87,21 +82,15 @@ private void AppendRecord(T record) var row = appender.CreateRow(); - foreach (var mapping in orderedMappings) + foreach (var mapping in classMap.PropertyMappings) { - switch (mapping.MappingType) + _ = mapping.MappingType switch { - case PropertyMappingType.Property: - var value = mapping.Getter(record); - AppendValue(row, value); - break; - case PropertyMappingType.Default: - row.AppendDefault(); - break; - case PropertyMappingType.Null: - row.AppendNullValue(); - break; - } + PropertyMappingType.Property => AppendValue(row, mapping.Getter(record)), + PropertyMappingType.Default => row.AppendDefault(), + PropertyMappingType.Null => row.AppendNullValue(), + _ => row + }; } row.EndRow(); @@ -109,108 +98,49 @@ private void AppendRecord(T record) private static DuckDBType GetExpectedDuckDBType(Type type) { - // Handle nullable types var underlyingType = Nullable.GetUnderlyingType(type) ?? type; - return underlyingType switch + var duckDBType = underlyingType.GetDuckDBType(); + + return duckDBType switch { - Type t when t == typeof(bool) => DuckDBType.Boolean, - Type t when t == typeof(sbyte) => DuckDBType.TinyInt, - Type t when t == typeof(short) => DuckDBType.SmallInt, - Type t when t == typeof(int) => DuckDBType.Integer, - Type t when t == typeof(long) => DuckDBType.BigInt, - Type t when t == typeof(byte) => DuckDBType.UnsignedTinyInt, - Type t when t == typeof(ushort) => DuckDBType.UnsignedSmallInt, - Type t when t == typeof(uint) => DuckDBType.UnsignedInteger, - Type t when t == typeof(ulong) => DuckDBType.UnsignedBigInt, - Type t when t == typeof(float) => DuckDBType.Float, - Type t when t == typeof(double) => DuckDBType.Double, - Type t when t == typeof(decimal) => DuckDBType.Decimal, - Type t when t == typeof(string) => DuckDBType.Varchar, - Type t when t == typeof(DateTime) => DuckDBType.Timestamp, - Type t when t == typeof(DateTimeOffset) => DuckDBType.TimestampTz, - Type t when t == typeof(TimeSpan) => DuckDBType.Interval, - Type t when t == typeof(Guid) => DuckDBType.Uuid, -#if NET6_0_OR_GREATER - Type t when t == typeof(DateOnly) => DuckDBType.Date, - Type t when t == typeof(TimeOnly) => DuckDBType.Time, -#endif - _ => throw new NotSupportedException($"Type {type.Name} is not supported for mapping") + DuckDBType.Invalid => throw new NotSupportedException($"Type {type.Name} is not supported for mapping"), + _ => duckDBType }; } - private static void AppendValue(IDuckDBAppenderRow row, object? value) + private static IDuckDBAppenderRow AppendValue(IDuckDBAppenderRow row, object? value) { if (value == null) { - row.AppendNullValue(); - return; + return row.AppendNullValue(); } - switch (value) + return value switch { - case bool boolValue: - row.AppendValue(boolValue); - break; - case sbyte sbyteValue: - row.AppendValue(sbyteValue); - break; - case short shortValue: - row.AppendValue(shortValue); - break; - case int intValue: - row.AppendValue(intValue); - break; - case long longValue: - row.AppendValue(longValue); - break; - case byte byteValue: - row.AppendValue(byteValue); - break; - case ushort ushortValue: - row.AppendValue(ushortValue); - break; - case uint uintValue: - row.AppendValue(uintValue); - break; - case ulong ulongValue: - row.AppendValue(ulongValue); - break; - case float floatValue: - row.AppendValue(floatValue); - break; - case double doubleValue: - row.AppendValue(doubleValue); - break; - case decimal decimalValue: - row.AppendValue(decimalValue); - break; - case string stringValue: - row.AppendValue(stringValue); - break; - case DateTime dateTimeValue: - row.AppendValue(dateTimeValue); - break; - case DateTimeOffset dateTimeOffsetValue: - row.AppendValue(dateTimeOffsetValue); - break; - case TimeSpan timeSpanValue: - row.AppendValue(timeSpanValue); - break; - case Guid guidValue: - row.AppendValue(guidValue); - break; + bool boolValue => row.AppendValue(boolValue), + sbyte sbyteValue => row.AppendValue(sbyteValue), + short shortValue => row.AppendValue(shortValue), + int intValue => row.AppendValue(intValue), + long longValue => row.AppendValue(longValue), + byte byteValue => row.AppendValue(byteValue), + ushort ushortValue => row.AppendValue(ushortValue), + uint uintValue => row.AppendValue(uintValue), + ulong ulongValue => row.AppendValue(ulongValue), + float floatValue => row.AppendValue(floatValue), + double doubleValue => row.AppendValue(doubleValue), + decimal decimalValue => row.AppendValue(decimalValue), + string stringValue => row.AppendValue(stringValue), + DateTime dateTimeValue => row.AppendValue(dateTimeValue), + DateTimeOffset dateTimeOffsetValue => row.AppendValue(dateTimeOffsetValue), + TimeSpan timeSpanValue => row.AppendValue(timeSpanValue), + Guid guidValue => row.AppendValue(guidValue), #if NET6_0_OR_GREATER - case DateOnly dateOnlyValue: - row.AppendValue((DateOnly?)dateOnlyValue); - break; - case TimeOnly timeOnlyValue: - row.AppendValue((TimeOnly?)timeOnlyValue); - break; + DateOnly dateOnlyValue => row.AppendValue((DateOnly?)dateOnlyValue), + TimeOnly timeOnlyValue => row.AppendValue((TimeOnly?)timeOnlyValue), #endif - default: - throw new NotSupportedException($"Type {value.GetType().Name} is not supported for appending"); - } + _ => throw new NotSupportedException($"Type {value.GetType().Name} is not supported for appending") + }; } /// diff --git a/DuckDB.NET.Data/Extensions/TypeExtensions.cs b/DuckDB.NET.Data/Extensions/TypeExtensions.cs index 1a6a6a6..4a8c9ec 100644 --- a/DuckDB.NET.Data/Extensions/TypeExtensions.cs +++ b/DuckDB.NET.Data/Extensions/TypeExtensions.cs @@ -98,4 +98,6 @@ public static DuckDBLogicalType GetLogicalType(this Type type) throw new InvalidOperationException($"Cannot map type {type.FullName} to DuckDBType."); } + + public static DuckDBType GetDuckDBType(this Type type) => ClrToDuckDBTypeMap.TryGetValue(type, out var duckDBType) ? duckDBType : DuckDBType.Invalid; } \ No newline at end of file diff --git a/DuckDB.NET.Test/DuckDBMappedAppenderTests.cs b/DuckDB.NET.Test/DuckDBMappedAppenderTests.cs index 9e80c9f..7854d74 100644 --- a/DuckDB.NET.Test/DuckDBMappedAppenderTests.cs +++ b/DuckDB.NET.Test/DuckDBMappedAppenderTests.cs @@ -1,6 +1,4 @@ -using DuckDB.NET.Data; using DuckDB.NET.Data.Mapping; -using DuckDB.NET.Native; using FluentAssertions; using System; using Xunit; @@ -114,7 +112,7 @@ public PersonWithDefaultsMap() [Fact] public void MappedAppender_SupportsDefaultAndNull() { - Command.CommandText = "CREATE TABLE person_defaults(id INTEGER, name VARCHAR, age INTEGER DEFAULT 18, city VARCHAR);"; + Command.CommandText = "CREATE TABLE person_defaults(id INTEGER, name VARCHAR, age INT DEFAULT 18, city VARCHAR);"; Command.ExecuteNonQuery(); var people = new[] @@ -134,8 +132,7 @@ public void MappedAppender_SupportsDefaultAndNull() reader.Read().Should().BeTrue(); reader.GetInt32(0).Should().Be(1); reader.GetString(1).Should().Be("Alice"); - // Note: AppendDefault may append 0 or NULL depending on DuckDB version - // Just verify the row was inserted - reader.IsDBNull(3).Should().BeTrue(); // Null value + reader.GetInt32(2).Should().Be(18); + reader.IsDBNull(3).Should().BeTrue(); } } From 0425d0eab466a305337bf2cc3d1f9e5b9da7de69 Mon Sep 17 00:00:00 2001 From: Giorgi Dalakishvili Date: Fri, 31 Oct 2025 17:25:11 +0400 Subject: [PATCH 10/10] Optimize memory for mapped appender --- DuckDB.NET.Data/DuckDBMappedAppender.cs | 58 ++-------- DuckDB.NET.Data/Mapping/DuckDBClassMap.cs | 125 +++++++++++++++------- 2 files changed, 96 insertions(+), 87 deletions(-) diff --git a/DuckDB.NET.Data/DuckDBMappedAppender.cs b/DuckDB.NET.Data/DuckDBMappedAppender.cs index 2548993..cfd34b9 100644 --- a/DuckDB.NET.Data/DuckDBMappedAppender.cs +++ b/DuckDB.NET.Data/DuckDBMappedAppender.cs @@ -14,15 +14,17 @@ namespace DuckDB.NET.Data; public class DuckDBMappedAppender : IDisposable where TMap : DuckDBClassMap, new() { private readonly DuckDBAppender appender; - private readonly TMap classMap; + private readonly List> mappings; internal DuckDBMappedAppender(DuckDBAppender appender) { this.appender = appender; - classMap = new TMap(); - + var classMap = new TMap(); + + // Get mappings as List to avoid interface enumerator boxing + mappings = classMap.PropertyMappings; + // Validate mappings match the table structure - var mappings = classMap.PropertyMappings; if (mappings.Count == 0) { throw new InvalidOperationException($"ClassMap {typeof(TMap).Name} has no property mappings defined"); @@ -49,9 +51,7 @@ internal DuckDBMappedAppender(DuckDBAppender appender) if (expectedType != columnType) { throw new InvalidOperationException( - $"Type mismatch for property '{mapping.PropertyName}': " + - $"Property type is {mapping.PropertyType.Name} (maps to {expectedType}) " + - $"but column {index} is {columnType}"); + $"Type mismatch at column index {index}: Mapped type is {mapping.PropertyType.Name} (expected DuckDB type: {expectedType}) but actual column type is {columnType}"); } } } @@ -82,15 +82,9 @@ private void AppendRecord(T record) var row = appender.CreateRow(); - foreach (var mapping in classMap.PropertyMappings) + foreach (var mapping in mappings) { - _ = mapping.MappingType switch - { - PropertyMappingType.Property => AppendValue(row, mapping.Getter(record)), - PropertyMappingType.Default => row.AppendDefault(), - PropertyMappingType.Null => row.AppendNullValue(), - _ => row - }; + row = mapping.AppendToRow(row, record); } row.EndRow(); @@ -109,40 +103,6 @@ private static DuckDBType GetExpectedDuckDBType(Type type) }; } - private static IDuckDBAppenderRow AppendValue(IDuckDBAppenderRow row, object? value) - { - if (value == null) - { - return row.AppendNullValue(); - } - - return value switch - { - bool boolValue => row.AppendValue(boolValue), - sbyte sbyteValue => row.AppendValue(sbyteValue), - short shortValue => row.AppendValue(shortValue), - int intValue => row.AppendValue(intValue), - long longValue => row.AppendValue(longValue), - byte byteValue => row.AppendValue(byteValue), - ushort ushortValue => row.AppendValue(ushortValue), - uint uintValue => row.AppendValue(uintValue), - ulong ulongValue => row.AppendValue(ulongValue), - float floatValue => row.AppendValue(floatValue), - double doubleValue => row.AppendValue(doubleValue), - decimal decimalValue => row.AppendValue(decimalValue), - string stringValue => row.AppendValue(stringValue), - DateTime dateTimeValue => row.AppendValue(dateTimeValue), - DateTimeOffset dateTimeOffsetValue => row.AppendValue(dateTimeOffsetValue), - TimeSpan timeSpanValue => row.AppendValue(timeSpanValue), - Guid guidValue => row.AppendValue(guidValue), -#if NET6_0_OR_GREATER - DateOnly dateOnlyValue => row.AppendValue((DateOnly?)dateOnlyValue), - TimeOnly timeOnlyValue => row.AppendValue((TimeOnly?)timeOnlyValue), -#endif - _ => throw new NotSupportedException($"Type {value.GetType().Name} is not supported for appending") - }; - } - /// /// Closes the appender and flushes any remaining data. /// diff --git a/DuckDB.NET.Data/Mapping/DuckDBClassMap.cs b/DuckDB.NET.Data/Mapping/DuckDBClassMap.cs index 92be093..9587a97 100644 --- a/DuckDB.NET.Data/Mapping/DuckDBClassMap.cs +++ b/DuckDB.NET.Data/Mapping/DuckDBClassMap.cs @@ -1,6 +1,7 @@ +using DuckDB.NET.Native; using System; using System.Collections.Generic; -using System.Linq.Expressions; +using System.Numerics; namespace DuckDB.NET.Data.Mapping; @@ -10,39 +11,26 @@ namespace DuckDB.NET.Data.Mapping; /// The type to map public abstract class DuckDBClassMap { - private readonly List> propertyMappings = new(); - /// /// Gets the property mappings defined for this class map. /// - internal IReadOnlyList> PropertyMappings => propertyMappings; + internal List> PropertyMappings { get; } = new(8); /// /// Maps a property to the next column in sequence. /// /// The property type - /// Expression to select the property - /// The current map instance for fluent configuration - protected void Map(Expression> propertyExpression) + /// Function to get the property value + protected void Map(Func getter) { - if (propertyExpression.Body is not MemberExpression memberExpression) - { - throw new ArgumentException("Expression must be a member expression", nameof(propertyExpression)); - } - - var propertyName = memberExpression.Member.Name; - var propertyType = typeof(TProperty); - var getter = propertyExpression.Compile(); - - var mapping = new PropertyMapping + var mapping = new PropertyMapping { - PropertyName = propertyName, - PropertyType = propertyType, - Getter = obj => getter(obj), + PropertyType = typeof(TProperty), + Getter = getter, MappingType = PropertyMappingType.Property }; - propertyMappings.Add(mapping); + PropertyMappings.Add(mapping); } /// @@ -50,15 +38,13 @@ protected void Map(Expression> propertyExpression) /// protected void DefaultValue() { - var mapping = new PropertyMapping + var mapping = new DefaultValueMapping { - PropertyName = "", PropertyType = typeof(object), - Getter = _ => null, MappingType = PropertyMappingType.Default }; - propertyMappings.Add(mapping); + PropertyMappings.Add(mapping); } /// @@ -66,21 +52,16 @@ protected void DefaultValue() /// protected void NullValue() { - var mapping = new PropertyMapping + var mapping = new NullValueMapping { - PropertyName = "", PropertyType = typeof(object), - Getter = _ => null, MappingType = PropertyMappingType.Null }; - propertyMappings.Add(mapping); + PropertyMappings.Add(mapping); } } -/// -/// Represents the type of mapping. -/// internal enum PropertyMappingType { Property, @@ -88,13 +69,81 @@ internal enum PropertyMappingType Null } -/// -/// Represents a mapping between a property and a column. -/// -internal class PropertyMapping +internal interface IPropertyMapping +{ + Type PropertyType { get; } + PropertyMappingType MappingType { get; } + IDuckDBAppenderRow AppendToRow(IDuckDBAppenderRow row, T record); +} + +internal sealed class PropertyMapping : IPropertyMapping { - public string PropertyName { get; set; } = string.Empty; public Type PropertyType { get; set; } = typeof(object); - public Func Getter { get; set; } = _ => null; + public Func Getter { get; set; } = _ => default!; public PropertyMappingType MappingType { get; set; } + + public IDuckDBAppenderRow AppendToRow(IDuckDBAppenderRow row, T record) + { + var value = Getter(record); + + if (value is null) + { + return row.AppendNullValue(); + } + + return value switch + { + // Reference types + string v => row.AppendValue(v), + + // Value types + bool v => row.AppendValue(v), + sbyte v => row.AppendValue(v), + short v => row.AppendValue(v), + int v => row.AppendValue(v), + long v => row.AppendValue(v), + byte v => row.AppendValue(v), + ushort v => row.AppendValue(v), + uint v => row.AppendValue(v), + ulong v => row.AppendValue(v), + float v => row.AppendValue(v), + double v => row.AppendValue(v), + decimal v => row.AppendValue(v), + DateTime v => row.AppendValue(v), + DateTimeOffset v => row.AppendValue(v), + TimeSpan v => row.AppendValue(v), + Guid v => row.AppendValue(v), + BigInteger v => row.AppendValue(v), + DuckDBDateOnly v => row.AppendValue(v), + DuckDBTimeOnly v => row.AppendValue(v), +#if NET6_0_OR_GREATER + DateOnly v => row.AppendValue(v), + TimeOnly v => row.AppendValue(v), +#endif + + _ => throw new NotSupportedException($"Type {typeof(TProperty).Name} is not supported for appending") + }; + } +} + +internal sealed class DefaultValueMapping : IPropertyMapping +{ + public Type PropertyType { get; set; } = typeof(object); + public PropertyMappingType MappingType { get; set; } + + public IDuckDBAppenderRow AppendToRow(IDuckDBAppenderRow row, T record) + { + return row.AppendDefault(); + } +} + +internal sealed class NullValueMapping : IPropertyMapping +{ + public Type PropertyType { get; set; } = typeof(object); + public PropertyMappingType MappingType { get; set; } + + public IDuckDBAppenderRow AppendToRow(IDuckDBAppenderRow row, T record) + { + return row.AppendNullValue(); + } }