From dde9366a98365fff874f3142851865d9830fe4b2 Mon Sep 17 00:00:00 2001 From: James Abbott Date: Mon, 9 Jun 2025 22:13:03 +0100 Subject: [PATCH 1/8] feat: bump dependencies to ef9 --- ...ityFrameworkCore.Spanner.Benchmarks.csproj | 2 +- ...meworkCore.Spanner.IntegrationTests.csproj | 6 +- .../QueryTests.cs | 2 +- ...EntityFrameworkCore.Spanner.Samples.csproj | 2 +- ...d.EntityFrameworkCore.Spanner.Tests.csproj | 6 +- .../MigrationMockServerTests.cs | 17 +- .../MockSpannerServer.cs | 76 +++++++- .../SpannerServiceCollectionExtensions.cs | 1 + ...e.Cloud.EntityFrameworkCore.Spanner.csproj | 20 +- .../Internal/SpannerHistoryRepository.cs | 177 ++++++++++++++++- .../SpannerMigrationCommandExecutor.cs | 180 ++++++++++++++++-- .../Internal/SpannerMigrationDatabaseLock.cs | 50 +++++ .../Internal/SpannerContainsExpression.cs | 23 +++ .../Internal/SpannerDateExtractExpression.cs | 13 ++ .../SpannerDateTimeMethodTranslator.cs | 2 +- .../Internal/SpannerIntervalExpression.cs | 10 + .../SpannerParameterBasedSqlProcessor.cs | 6 +- ...pannerParameterBasedSqlProcessorFactory.cs | 5 +- .../Internal/SpannerSqlExpressionFactory.cs | 2 +- .../SpannerSqlNullabilityProcessor.cs | 4 +- .../SpannerSqlTranslatingExpressionVisitor.cs | 86 +++++++++ ...rSqlTranslatingExpressionVisitorFactory.cs | 55 ++++++ .../SpannerTimestampExtractExpression.cs | 12 ++ 23 files changed, 696 insertions(+), 61 deletions(-) create mode 100644 Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerMigrationDatabaseLock.cs create mode 100644 Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerSqlTranslatingExpressionVisitor.cs create mode 100644 Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerSqlTranslatingExpressionVisitorFactory.cs diff --git a/Google.Cloud.EntityFrameworkCore.Spanner.Benchmarks/Google.Cloud.EntityFrameworkCore.Spanner.Benchmarks.csproj b/Google.Cloud.EntityFrameworkCore.Spanner.Benchmarks/Google.Cloud.EntityFrameworkCore.Spanner.Benchmarks.csproj index 71675f10..6f4e265b 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner.Benchmarks/Google.Cloud.EntityFrameworkCore.Spanner.Benchmarks.csproj +++ b/Google.Cloud.EntityFrameworkCore.Spanner.Benchmarks/Google.Cloud.EntityFrameworkCore.Spanner.Benchmarks.csproj @@ -1,7 +1,7 @@  - net8.0 + net9.0 diff --git a/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests.csproj b/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests.csproj index f3ffc261..a7ecba9e 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests.csproj +++ b/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests.csproj @@ -1,7 +1,7 @@  - net8.0 + net9.0 false @@ -19,8 +19,8 @@ - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests/QueryTests.cs b/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests/QueryTests.cs index 2aeb847d..54c808e0 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests/QueryTests.cs +++ b/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests/QueryTests.cs @@ -692,7 +692,7 @@ public async Task CanUseStringFormat() var formattedName = await db.Singers .Where(s => new long[] { singerId }.Contains(s.SingerId)) - .Select(s => string.Format("String without formatting")) + .Select(s => string.Format("String without formatting", Array.Empty())) .FirstOrDefaultAsync(); Assert.Equal("String without formatting", formattedName); diff --git a/Google.Cloud.EntityFrameworkCore.Spanner.Samples/Google.Cloud.EntityFrameworkCore.Spanner.Samples.csproj b/Google.Cloud.EntityFrameworkCore.Spanner.Samples/Google.Cloud.EntityFrameworkCore.Spanner.Samples.csproj index abcd06a7..9cf905fe 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner.Samples/Google.Cloud.EntityFrameworkCore.Spanner.Samples.csproj +++ b/Google.Cloud.EntityFrameworkCore.Spanner.Samples/Google.Cloud.EntityFrameworkCore.Spanner.Samples.csproj @@ -1,7 +1,7 @@  - net8.0 + net9.0 SampleRunner Exe diff --git a/Google.Cloud.EntityFrameworkCore.Spanner.Tests/Google.Cloud.EntityFrameworkCore.Spanner.Tests.csproj b/Google.Cloud.EntityFrameworkCore.Spanner.Tests/Google.Cloud.EntityFrameworkCore.Spanner.Tests.csproj index 3fe42443..0b14f6e1 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner.Tests/Google.Cloud.EntityFrameworkCore.Spanner.Tests.csproj +++ b/Google.Cloud.EntityFrameworkCore.Spanner.Tests/Google.Cloud.EntityFrameworkCore.Spanner.Tests.csproj @@ -1,7 +1,7 @@  - net8.0 + net9.0 false @@ -18,11 +18,11 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Google.Cloud.EntityFrameworkCore.Spanner.Tests/MigrationTests/MigrationMockServerTests.cs b/Google.Cloud.EntityFrameworkCore.Spanner.Tests/MigrationTests/MigrationMockServerTests.cs index 001f07aa..ef699eb8 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner.Tests/MigrationTests/MigrationMockServerTests.cs +++ b/Google.Cloud.EntityFrameworkCore.Spanner.Tests/MigrationTests/MigrationMockServerTests.cs @@ -17,6 +17,7 @@ using Microsoft.EntityFrameworkCore.Migrations; using System; using Xunit; +using V1 = Google.Cloud.Spanner.V1; namespace Google.Cloud.EntityFrameworkCore.Spanner.Tests.MigrationTests { @@ -43,6 +44,16 @@ public void TestMigrateUsesDdlBatch() var version = typeof(Migration).Assembly.GetName().Version ?? new Version(); var formattedVersion = $"{version.Major}.{version.Minor}.{version.Build}"; _fixture.SpannerMock.AddOrUpdateStatementResult("SELECT 1", StatementResult.CreateException(MockSpannerService.CreateDatabaseNotFoundException("d1"))); + _fixture.SpannerMock.AddOrUpdateStatementResult( + "SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_catalog = '' and table_schema = '' and table_name = '''__EFMigrationsLock''')", + StatementResult.CreateSelect1ResultSet() + ); + // Add mock result for the INSERT OR IGNORE migration lock statement - this has a dynamic timestamp so we'll use a pattern + // The SQL will be something like: INSERT OR IGNORE INTO "__EFMigrationsLock"("Id", "Timestamp") VALUES(1, '2025-06-09 20:16:59.350519+00:00'); SELECT changes(); + _fixture.SpannerMock.AddOrUpdatePatternResult( + "INSERT OR IGNORE INTO \"__EFMigrationsLock\"(\"Id\", \"Timestamp\") VALUES(1, '*');\nSELECT changes();", + StatementResult.CreateSingleColumnResultSet(new V1.Type { Code = V1.TypeCode.Int64 }, "changes", 1L) + ); _fixture.SpannerMock.AddOrUpdateStatementResult( $"INSERT INTO `EFMigrationsHistory` (`MigrationId`, `ProductVersion`)\nVALUES ('''20210309110233_Initial''', '''{formattedVersion}''')", StatementResult.CreateUpdateCount(1) @@ -51,6 +62,10 @@ public void TestMigrateUsesDdlBatch() $"INSERT INTO `EFMigrationsHistory` (`MigrationId`, `ProductVersion`)\nVALUES ('''20210830_V2''', '''{formattedVersion}''')", StatementResult.CreateUpdateCount(1) ); + _fixture.SpannerMock.AddOrUpdateStatementResult( + "DELETE FROM \"__EFMigrationsLock\";", + StatementResult.CreateUpdateCount(1) + ); using var db = new MockMigrationSampleDbContext(ConnectionString); db.Database.Migrate(); @@ -62,7 +77,7 @@ public void TestMigrateUsesDdlBatch() var update = request as UpdateDatabaseDdlRequest; Assert.NotNull(update); Assert.Collection(update.Statements, - sql => Assert.StartsWith("CREATE TABLE `EFMigrationsHistory`", sql) + sql => Assert.StartsWith("CREATE TABLE IF NOT EXISTS `EFMigrationsHistory`", sql) ); }, // Each migration will be executed as a separate DDL batch. diff --git a/Google.Cloud.EntityFrameworkCore.Spanner.Tests/MockSpannerServer.cs b/Google.Cloud.EntityFrameworkCore.Spanner.Tests/MockSpannerServer.cs index c3a5885e..01c6f2f6 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner.Tests/MockSpannerServer.cs +++ b/Google.Cloud.EntityFrameworkCore.Spanner.Tests/MockSpannerServer.cs @@ -314,6 +314,7 @@ public void Dispose() private readonly object _lock = new(); private readonly ConcurrentDictionary _results = new(); + private readonly ConcurrentDictionary _patternResults = new(); private ConcurrentQueue _requests = new(); private ConcurrentQueue _contexts = new(); private ConcurrentQueue _headers = new (); @@ -334,6 +335,74 @@ public void AddOrUpdateStatementResult(string sql, StatementResult result) ); } + public void AddOrUpdatePatternResult(string sqlPattern, StatementResult result) + { + _patternResults.AddOrUpdate(sqlPattern.Trim(), + result, + (_, _) => result + ); + } + + private StatementResult FindStatementResult(string sql) + { + // First try exact match + if (_results.TryGetValue(sql.Trim(), out StatementResult result)) + { + return result; + } + + // Then try pattern matching + foreach (var pattern in _patternResults.Keys) + { + if (IsWildcardMatch(sql.Trim(), pattern)) + { + return _patternResults[pattern]; + } + } + + return null; + } + + private static bool IsWildcardMatch(string text, string pattern) + { + // Simple wildcard matching supporting * as wildcard + if (pattern == "*") return true; + + var parts = pattern.Split('*'); + if (parts.Length == 1) + { + // No wildcards, exact match + return text.Equals(pattern, StringComparison.OrdinalIgnoreCase); + } + + int currentIndex = 0; + for (int i = 0; i < parts.Length; i++) + { + var part = parts[i]; + if (string.IsNullOrEmpty(part)) continue; + + int foundIndex = text.IndexOf(part, currentIndex, StringComparison.OrdinalIgnoreCase); + if (foundIndex == -1) return false; + + // First part must match from the beginning + if (i == 0 && foundIndex != 0) return false; + + currentIndex = foundIndex + part.Length; + } + + // Last part must match at the end (unless pattern ends with *) + if (!pattern.EndsWith("*") && parts.Length > 1) + { + var lastPart = parts[parts.Length - 1]; + if (!string.IsNullOrEmpty(lastPart) && !text.EndsWith(lastPart, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + } + + return true; + } + public void AddOrUpdateExecutionTime(string method, ExecutionTime executionTime) { _executionTimes.AddOrUpdate(method, @@ -370,6 +439,7 @@ public void Reset() _headers = new ConcurrentQueue(); _executionTimes.Clear(); _results.Clear(); + _patternResults.Clear(); _abortedTransactions.Clear(); _abortNextStatement = false; } @@ -592,7 +662,8 @@ public override Task ExecuteBatchDml(ExecuteBatchDmlReq { break; } - if (_results.TryGetValue(statement.Sql.Trim(), out StatementResult result)) + var result = FindStatementResult(statement.Sql); + if (result != null) { switch (result.Type) { @@ -652,7 +723,8 @@ public override async Task ExecuteStreamingSql(ExecuteSqlRequest request, IServe { returnTransaction = transaction; } - if (_results.TryGetValue(request.Sql.Trim(), out StatementResult result)) + var result = FindStatementResult(request.Sql); + if (result != null) { switch (result.Type) { diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Extensions/SpannerServiceCollectionExtensions.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Extensions/SpannerServiceCollectionExtensions.cs index 593de7f9..0b9ffc28 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner/Extensions/SpannerServiceCollectionExtensions.cs +++ b/Google.Cloud.EntityFrameworkCore.Spanner/Extensions/SpannerServiceCollectionExtensions.cs @@ -61,6 +61,7 @@ public static IServiceCollection AddEntityFrameworkSpanner(this IServiceCollecti .TryAdd() .TryAdd() .TryAdd() + .TryAdd() .TryAdd(p => p.GetService()) .TryAdd() .TryAdd() diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Google.Cloud.EntityFrameworkCore.Spanner.csproj b/Google.Cloud.EntityFrameworkCore.Spanner/Google.Cloud.EntityFrameworkCore.Spanner.csproj index 8ecc3180..662eb260 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner/Google.Cloud.EntityFrameworkCore.Spanner.csproj +++ b/Google.Cloud.EntityFrameworkCore.Spanner/Google.Cloud.EntityFrameworkCore.Spanner.csproj @@ -2,7 +2,7 @@ Google Cloud Spanner database provider for Entity Framework Core. - net8.0 + net9.0 Google.Cloud.EntityFrameworkCore.Spanner Google.Cloud.EntityFrameworkCore.Spanner latest @@ -19,23 +19,23 @@ - - - - - - - + + + + + + + True - + True - + diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerHistoryRepository.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerHistoryRepository.cs index 0312c0cc..45066abe 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerHistoryRepository.cs +++ b/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerHistoryRepository.cs @@ -12,11 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage; using System; using System.Text; +using System.Threading; +using System.Threading.Tasks; namespace Google.Cloud.EntityFrameworkCore.Spanner.Migrations.Internal { @@ -25,6 +28,8 @@ namespace Google.Cloud.EntityFrameworkCore.Spanner.Migrations.Internal /// public class SpannerHistoryRepository : HistoryRepository { + private static readonly TimeSpan _retryDelay = TimeSpan.FromSeconds(1); + /// /// The default name for the Migrations history table. /// @@ -46,6 +51,16 @@ public SpannerHistoryRepository(HistoryRepositoryDependencies dependencies) protected override HistoryRepositoryDependencies Dependencies { get; } + private string CreateExistsSql(string tableName) + { + var stringTypeMapping = Dependencies.TypeMappingSource.GetMapping(typeof(string)); + + var builder = new StringBuilder(); + builder.Append("SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_catalog = '' and table_schema = '' and table_name = ") + .Append($"{stringTypeMapping.GenerateSqlLiteral(Dependencies.SqlGenerationHelper.DelimitIdentifier(tableName, TableSchema))})"); + builder.Replace("`", ""); + return builder.ToString(); + } /// /// This is internal functionality and not intended for public use. /// @@ -53,27 +68,27 @@ protected override string ExistsSql { get { - var stringTypeMapping = Dependencies.TypeMappingSource.GetMapping(typeof(string)); - - var builder = new StringBuilder(); - builder.Append("SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_catalog = '' and table_schema = '' and table_name = ") - .Append($"{stringTypeMapping.GenerateSqlLiteral(Dependencies.SqlGenerationHelper.DelimitIdentifier(TableName, TableSchema))})"); - builder.Replace("`", ""); - return builder.ToString(); + return CreateExistsSql(TableName); } } /// /// This is internal functionality and not intended for public use. /// - protected override bool InterpretExistsResult(object value) => (bool)value; + protected override bool InterpretExistsResult(object value) => value switch { + long longValue => longValue != 0, + bool boolValue => boolValue, + _ => throw new ArgumentException( + $"Unexpected type for EXISTS result: {value.GetType().Name}. Expected long or bool.", nameof(value)) + }; /// /// This is internal functionality and not intended for public use. /// public override string GetCreateIfNotExistsScript() { - throw new NotSupportedException("Cloud Spanner does not support CREATE IF NOT EXISTS style commands."); + var script = GetCreateScript(); + return script.Insert(script.IndexOf("CREATE TABLE", StringComparison.Ordinal) + 12, " IF NOT EXISTS"); } /// @@ -99,5 +114,149 @@ public override string GetEndIfScript() .Append("") .AppendLine(SqlGenerationHelper.StatementTerminator) .ToString(); + + /// + /// The name of the table that will serve as a database-wide lock for migrations. + /// + protected virtual string LockTableName { get; } = "__EFMigrationsLock"; + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public override LockReleaseBehavior LockReleaseBehavior => LockReleaseBehavior.Explicit; + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public override IMigrationsDatabaseLock AcquireDatabaseLock() + { + Dependencies.MigrationsLogger.AcquiringMigrationLock(); + + if (!InterpretExistsResult( + Dependencies.RawSqlCommandBuilder.Build(CreateExistsSql(LockTableName)) + .ExecuteScalar(CreateRelationalCommandParameters()))) + { + CreateLockTableCommand().ExecuteNonQuery(CreateRelationalCommandParameters()); + } + + var retryDelay = _retryDelay; + while (true) + { + long insertCount = 0; + var dbLock = CreateMigrationDatabaseLock(); + var results = CreateInsertLockCommand(DateTimeOffset.UtcNow) + .ExecuteReader(CreateRelationalCommandParameters()); + + if (results.Read()) + { + if (results.DbDataReader.FieldCount == 0 || results.DbDataReader.IsDBNull(0)) + { + throw new InvalidOperationException("Failed to acquire migration lock."); + } + + insertCount = results.DbDataReader.GetInt64(0); + } + if ((long)insertCount! == 1) + { + return dbLock; + } + + Thread.Sleep(retryDelay); + if (retryDelay < TimeSpan.FromMinutes(1)) + { + retryDelay = retryDelay.Add(retryDelay); + } + } + } + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public override async Task AcquireDatabaseLockAsync( + CancellationToken cancellationToken = default) + { + Dependencies.MigrationsLogger.AcquiringMigrationLock(); + + if (!InterpretExistsResult( + await Dependencies.RawSqlCommandBuilder.Build(CreateExistsSql(LockTableName)) + .ExecuteScalarAsync(CreateRelationalCommandParameters(), cancellationToken).ConfigureAwait(false))) + { + await CreateLockTableCommand().ExecuteNonQueryAsync(CreateRelationalCommandParameters(), cancellationToken) + .ConfigureAwait(false); + } + + var retryDelay = _retryDelay; + while (true) + { + var dbLock = CreateMigrationDatabaseLock(); + var insertCount = await CreateInsertLockCommand(DateTimeOffset.UtcNow) + .ExecuteScalarAsync(CreateRelationalCommandParameters(), cancellationToken) + .ConfigureAwait(false); + if ((long)insertCount! == 1) + { + return dbLock; + } + + await Task.Delay(_retryDelay, cancellationToken).ConfigureAwait(true); + if (retryDelay < TimeSpan.FromMinutes(1)) + { + retryDelay = retryDelay.Add(retryDelay); + } + } + } + + private IRelationalCommand CreateLockTableCommand() + => Dependencies.RawSqlCommandBuilder.Build( + $""" +CREATE TABLE IF NOT EXISTS "{LockTableName}" ( + "Id" INTEGER NOT NULL CONSTRAINT "PK_{LockTableName}" PRIMARY KEY, + "Timestamp" TEXT NOT NULL +); +"""); + + private IRelationalCommand CreateInsertLockCommand(DateTimeOffset timestamp) + { + var timestampLiteral = Dependencies.TypeMappingSource.GetMapping(typeof(DateTimeOffset)).GenerateSqlLiteral(timestamp); + + return Dependencies.RawSqlCommandBuilder.Build( + $""" +INSERT OR IGNORE INTO "{LockTableName}"("Id", "Timestamp") VALUES(1, {timestampLiteral}); +SELECT changes(); +"""); + } + + private IRelationalCommand CreateDeleteLockCommand(int? id = null) + { + var sql = $""" +DELETE FROM "{LockTableName}" +"""; + if (id != null) + { + sql += $""" WHERE "Id" = {id}"""; + } + + sql += ";"; + return Dependencies.RawSqlCommandBuilder.Build(sql); + } + + private SpannerMigrationDatabaseLock CreateMigrationDatabaseLock() + => new(CreateDeleteLockCommand(), CreateRelationalCommandParameters(), this); + + private RelationalCommandParameterObject CreateRelationalCommandParameters() + => new( + Dependencies.Connection, + null, + null, + Dependencies.CurrentContext.Context, + Dependencies.CommandLogger, CommandSource.Migrations); } } diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerMigrationCommandExecutor.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerMigrationCommandExecutor.cs index 88cc1926..88c44920 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerMigrationCommandExecutor.cs +++ b/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerMigrationCommandExecutor.cs @@ -15,53 +15,193 @@ using Google.Api.Gax; using Google.Cloud.EntityFrameworkCore.Spanner.Storage.Internal; using Google.Cloud.Spanner.Data; +using Microsoft.EntityFrameworkCore.Diagnostics; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Storage; +using System; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; +using System.Transactions; namespace Google.Cloud.EntityFrameworkCore.Spanner.Migrations.Internal { - internal class SpannerMigrationCommandExecutor : IMigrationCommandExecutor + internal class SpannerMigrationCommandExecutor(IExecutionStrategy executionStrategy) : IMigrationCommandExecutor { public void ExecuteNonQuery(IEnumerable migrationCommands, IRelationalConnection connection) { - ExecuteNonQueryAsync(migrationCommands, connection).WaitWithUnwrappedExceptions(); + ExecuteNonQuery(migrationCommands.ToList(), connection, new MigrationExecutionState(), commitTransaction: true); } + public int ExecuteNonQuery(IReadOnlyList migrationCommands, IRelationalConnection connection, MigrationExecutionState executionState, bool commitTransaction, System.Data.IsolationLevel? isolationLevel = null) + { + GaxPreconditions.CheckArgument(connection is SpannerRelationalConnection, nameof(connection), "Can only be used with Spanner connections"); + + var inUserTransaction = connection.CurrentTransaction is not null && executionState.Transaction == null; + if (inUserTransaction + && (migrationCommands.Any(x => x.TransactionSuppressed) || executionStrategy.RetriesOnFailure)) + { + throw new NotSupportedException("Cannot execute transaction suppressed migration commands in user transaction."); + } + + using var transactionScope = new TransactionScope(TransactionScopeOption.Suppress, TransactionScopeAsyncFlowOption.Enabled); + + return executionStrategy.Execute( + (migrationCommands, connection, inUserTransaction, executionState, commitTransaction, isolationLevel), + static (_, s) => ExecuteInternal( + s.migrationCommands, + s.connection, + s.executionState, + beginTransaction: !s.inUserTransaction, + commitTransaction: !s.inUserTransaction && s.commitTransaction, + s.isolationLevel), + verifySucceeded: null); + } + + public async Task ExecuteNonQueryAsync(IEnumerable migrationCommands, IRelationalConnection connection, CancellationToken cancellationToken = default) + { + await ExecuteNonQueryAsync(migrationCommands.ToList(), connection, new MigrationExecutionState(), commitTransaction: true, System.Data.IsolationLevel.Unspecified, cancellationToken).ConfigureAwait(false); + } + + public async Task ExecuteNonQueryAsync(IReadOnlyList migrationCommands, IRelationalConnection connection, MigrationExecutionState executionState, bool commitTransaction, System.Data.IsolationLevel? isolationLevel = null, CancellationToken cancellationToken = default) { GaxPreconditions.CheckArgument(connection is SpannerRelationalConnection, nameof(connection), "Can only be used with Spanner connections"); - var statements = migrationCommands.Select(x => x.CommandText).ToArray(); - if (statements.Length == 0) + + var inUserTransaction = connection.CurrentTransaction is not null && executionState.Transaction == null; + if (inUserTransaction + && (migrationCommands.Any(x => x.TransactionSuppressed) || executionStrategy.RetriesOnFailure)) + { + throw new NotSupportedException("Cannot execute transaction suppressed migration commands in user transaction."); + } + + using var transactionScope = new TransactionScope(TransactionScopeOption.Suppress, TransactionScopeAsyncFlowOption.Enabled); + + return await executionStrategy.ExecuteAsync( + (migrationCommands, connection, inUserTransaction, executionState, commitTransaction, isolationLevel), + static (_, s, ct) => ExecuteInternalAsync( + s.migrationCommands, + s.connection, + s.executionState, + beginTransaction: !s.inUserTransaction, + commitTransaction: !s.inUserTransaction && s.commitTransaction, + s.isolationLevel, + ct), + verifySucceeded: null, + cancellationToken).ConfigureAwait(false); + } + + private static int ExecuteInternal( + IReadOnlyList migrationCommands, + IRelationalConnection connection, + MigrationExecutionState executionState, + bool beginTransaction, + bool commitTransaction, + System.Data.IsolationLevel? isolationLevel) + { + var result = 0; + var connectionOpened = connection.Open(); + + try { - return; + var statements = migrationCommands.Select(x => x.CommandText).ToArray(); + if (statements.Length == 0) + { + return result; + } + + var ddlStatements = statements.Where(IsDdlStatement).ToArray(); + var otherStatements = statements.Where(x => !IsDdlStatement(x)).ToArray(); + var spannerConnection = ((SpannerRelationalConnection)connection).DbConnection as SpannerRetriableConnection; + + if (ddlStatements.Any()) + { + var cmd = spannerConnection.CreateDdlCommand(ddlStatements[0], ddlStatements.Skip(1).ToArray()); + result += cmd.ExecuteNonQuery(); + } + + if (otherStatements.Any()) + { + using var transaction = spannerConnection.BeginTransaction(); + var cmd = spannerConnection.CreateBatchDmlCommand(); + cmd.Transaction = transaction; + foreach (var statement in otherStatements) + { + cmd.Add(statement); + } + // Batch DML returns IReadOnlyList, so sum the update counts + var updateCounts = cmd.ExecuteNonQuery(); + result += (int)updateCounts.Sum(); + transaction.Commit(); + } } - var ddlStatements = statements.Where(IsDdlStatement).ToArray(); - var otherStatements = statements.Where(x => !IsDdlStatement(x)); - var spannerConnection = ((SpannerRelationalConnection) connection).DbConnection as SpannerRetriableConnection; - if (ddlStatements.Any()) + catch { - var cmd = spannerConnection.CreateDdlCommand(ddlStatements[0], ddlStatements.Skip(1).ToArray()); - await cmd.ExecuteNonQueryAsync(cancellationToken); + connection.Close(); + throw; } - if (otherStatements.Any()) + + connection.Close(); + return result; + } + + private static async Task ExecuteInternalAsync( + IReadOnlyList migrationCommands, + IRelationalConnection connection, + MigrationExecutionState executionState, + bool beginTransaction, + bool commitTransaction, + System.Data.IsolationLevel? isolationLevel, + CancellationToken cancellationToken) + { + var result = 0; + var connectionOpened = await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + + try { - using var transaction = await spannerConnection.BeginTransactionAsync(cancellationToken); - var cmd = spannerConnection.CreateBatchDmlCommand(); - cmd.Transaction = transaction; - foreach (var statement in otherStatements) + var statements = migrationCommands.Select(x => x.CommandText).ToArray(); + if (statements.Length == 0) { - cmd.Add(statement); + return result; } - await cmd.ExecuteNonQueryAsync(cancellationToken); - await transaction.CommitAsync(cancellationToken); + + var ddlStatements = statements.Where(IsDdlStatement).ToArray(); + var otherStatements = statements.Where(x => !IsDdlStatement(x)).ToArray(); + var spannerConnection = ((SpannerRelationalConnection)connection).DbConnection as SpannerRetriableConnection; + + if (ddlStatements.Any()) + { + var cmd = spannerConnection.CreateDdlCommand(ddlStatements[0], ddlStatements.Skip(1).ToArray()); + result += await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + + if (otherStatements.Any()) + { + using var transaction = await spannerConnection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + var cmd = spannerConnection.CreateBatchDmlCommand(); + cmd.Transaction = transaction; + foreach (var statement in otherStatements) + { + cmd.Add(statement); + } + // Batch DML returns IReadOnlyList, so sum the update counts + var updateCounts = await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + result += (int)updateCounts.Sum(); + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + } + } + catch + { + await connection.CloseAsync().ConfigureAwait(false); + throw; } + + await connection.CloseAsync().ConfigureAwait(false); + return result; } - private bool IsDdlStatement(string statement) + private static bool IsDdlStatement(string statement) { return SpannerCommandTextBuilder.FromCommandText(statement).SpannerCommandType == SpannerCommandType.Ddl; } diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerMigrationDatabaseLock.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerMigrationDatabaseLock.cs new file mode 100644 index 00000000..dd630368 --- /dev/null +++ b/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerMigrationDatabaseLock.cs @@ -0,0 +1,50 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage; + +namespace Google.Cloud.EntityFrameworkCore.Spanner.Migrations.Internal; + +/// +/// This is an internal API that supports the Entity Framework Core infrastructure and not subject to +/// the same compatibility standards as public APIs. It may be changed or removed without notice in +/// any release. You should only use it directly in your code with extreme caution and knowing that +/// doing so can result in application failures when updating to a new Entity Framework Core release. +/// +public class SpannerMigrationDatabaseLock( + IRelationalCommand releaseLockCommand, + RelationalCommandParameterObject relationalCommandParameters, + IHistoryRepository historyRepository, + CancellationToken cancellationToken = default) + : IMigrationsDatabaseLock +{ + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public virtual IHistoryRepository HistoryRepository => historyRepository; + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public void Dispose() + => releaseLockCommand.ExecuteNonQuery(relationalCommandParameters); + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public async ValueTask DisposeAsync() + => await releaseLockCommand.ExecuteNonQueryAsync(relationalCommandParameters, cancellationToken).ConfigureAwait(false); +} \ No newline at end of file diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerContainsExpression.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerContainsExpression.cs index 2d7fbe0e..92090120 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerContainsExpression.cs +++ b/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerContainsExpression.cs @@ -16,7 +16,10 @@ using System; using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; using System.Linq.Expressions; +using System.Reflection; using Microsoft.EntityFrameworkCore.Query; using Microsoft.EntityFrameworkCore.Query.SqlExpressions; using Microsoft.EntityFrameworkCore.Storage; @@ -28,6 +31,8 @@ namespace Google.Cloud.EntityFrameworkCore.Spanner.Query.Internal; /// public class SpannerContainsExpression : SqlExpression { + private static ConstructorInfo? _quotingConstructorWithValues; + public SqlExpression Values { get; } public SqlExpression Item { get; } public virtual bool IsNegated { get; } @@ -52,6 +57,24 @@ protected override Expression VisitChildren(ExpressionVisitor visitor) return Update(item, values); } + + /// + public override Expression Quote() +#pragma warning disable EF9100 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + => this switch + { + { Values: not null } => New( + _quotingConstructorWithValues ??= typeof(SpannerContainsExpression).GetConstructor( + [typeof(SqlExpression), typeof(SqlExpression), typeof(bool), typeof(RelationalTypeMapping)])!, + Item.Quote(), + Values.Quote(), + Constant(IsNegated), + RelationalExpressionQuotingUtilities.QuoteTypeMapping(TypeMapping)), + + _ => throw new UnreachableException() + }; +#pragma warning restore EF9100 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + protected override void Print(ExpressionPrinter expressionPrinter) { expressionPrinter.Visit(Item); diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerDateExtractExpression.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerDateExtractExpression.cs index 9347b0d4..d443aadd 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerDateExtractExpression.cs +++ b/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerDateExtractExpression.cs @@ -67,5 +67,18 @@ public override int GetHashCode() hash *= 31 + _value.GetHashCode(); return hash; } + + public override Expression Quote() + { +#pragma warning disable EF9100 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + return New( + typeof(SpannerDateExtractExpression).GetConstructor( + [typeof(SqlExpression), typeof(SqlExpression), typeof(string), typeof(SpannerDateExtractExpression)])!, + _fromFragment.Quote(), + _value.Quote(), + Constant(_dateTimePartName), + RelationalExpressionQuotingUtilities.QuoteTypeMapping(TypeMapping)); +#pragma warning restore EF9100 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + } } } diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerDateTimeMethodTranslator.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerDateTimeMethodTranslator.cs index 29c9610a..b3440716 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerDateTimeMethodTranslator.cs +++ b/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerDateTimeMethodTranslator.cs @@ -144,7 +144,7 @@ private SqlExpression GetFirstArgumentAsInt64(IReadOnlyList argum "CAST", new[] { value, _sqlExpressionFactory.Fragment("INT64") }, true, - new []{true}, + new []{true, true}, typeof(long))); } if (multiplier != 1L) diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerIntervalExpression.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerIntervalExpression.cs index e985d1aa..fd5a2b1a 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerIntervalExpression.cs +++ b/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerIntervalExpression.cs @@ -70,5 +70,15 @@ public override int GetHashCode() hash *= _intervalNameFragment.GetHashCode(); return hash; } + + public override Expression Quote() + { + return New( + typeof(SpannerIntervalExpression).GetConstructor( + [typeof(ISqlExpressionFactory), typeof(SqlExpression), typeof(string)])!, + Constant(_sqlExpressionFactory), + _value.Quote(), + Constant(_intervalName)); + } } } diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerParameterBasedSqlProcessor.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerParameterBasedSqlProcessor.cs index e59d45cc..539080b1 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerParameterBasedSqlProcessor.cs +++ b/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerParameterBasedSqlProcessor.cs @@ -26,8 +26,8 @@ public class SpannerParameterBasedSqlProcessor : RelationalParameterBasedSqlProc /// /// Only for internal use. /// - public SpannerParameterBasedSqlProcessor([NotNull] RelationalParameterBasedSqlProcessorDependencies dependencies, bool useRelationalNulls) : - base(dependencies, useRelationalNulls) + public SpannerParameterBasedSqlProcessor([NotNull] RelationalParameterBasedSqlProcessorDependencies dependencies, RelationalParameterBasedSqlProcessorParameters parameters) : + base(dependencies, parameters) { } @@ -36,6 +36,6 @@ protected override Expression ProcessSqlNullability( Expression selectExpression, [ItemCanBeNull] IReadOnlyDictionary parametersValues, out bool canCache) - => new SpannerSqlNullabilityProcessor(Dependencies, UseRelationalNulls).Process( + => new SpannerSqlNullabilityProcessor(Dependencies, Parameters).Process( selectExpression, parametersValues, out canCache); } \ No newline at end of file diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerParameterBasedSqlProcessorFactory.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerParameterBasedSqlProcessorFactory.cs index 3c8c00bf..48b6bcd1 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerParameterBasedSqlProcessorFactory.cs +++ b/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerParameterBasedSqlProcessorFactory.cs @@ -26,7 +26,6 @@ public SpannerParameterBasedSqlProcessorFactory(RelationalParameterBasedSqlProce protected virtual RelationalParameterBasedSqlProcessorDependencies Dependencies { get; } - /// - public virtual RelationalParameterBasedSqlProcessor Create(bool useRelationalNulls) - => new SpannerParameterBasedSqlProcessor(Dependencies, useRelationalNulls); + public RelationalParameterBasedSqlProcessor Create(RelationalParameterBasedSqlProcessorParameters parameters) + => new SpannerParameterBasedSqlProcessor(Dependencies, parameters); } diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerSqlExpressionFactory.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerSqlExpressionFactory.cs index 91b9e868..196d359b 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerSqlExpressionFactory.cs +++ b/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerSqlExpressionFactory.cs @@ -30,7 +30,7 @@ public SpannerSqlExpressionFactory(SqlExpressionFactoryDependencies dependencies _boolTypeMapping = dependencies.TypeMappingSource.FindMapping(typeof(bool), dependencies.Model)!; } - public override InExpression In(SqlExpression item, SqlParameterExpression valuesParameter) + public override SqlExpression In(SqlExpression item, SqlParameterExpression valuesParameter) { var parametersTypeMapping = Dependencies.TypeMappingSource.FindMapping(valuesParameter.Type); if (parametersTypeMapping != null) diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerSqlNullabilityProcessor.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerSqlNullabilityProcessor.cs index c4ac4663..38cfa8dc 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerSqlNullabilityProcessor.cs +++ b/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerSqlNullabilityProcessor.cs @@ -24,8 +24,8 @@ public class SpannerSqlNullabilityProcessor : SqlNullabilityProcessor /// /// Only for internal use. /// - public SpannerSqlNullabilityProcessor([NotNull] RelationalParameterBasedSqlProcessorDependencies dependencies, bool useRelationalNulls) : - base(dependencies, useRelationalNulls) + public SpannerSqlNullabilityProcessor([NotNull] RelationalParameterBasedSqlProcessorDependencies dependencies, RelationalParameterBasedSqlProcessorParameters parameters) : + base(dependencies, parameters) { } diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerSqlTranslatingExpressionVisitor.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerSqlTranslatingExpressionVisitor.cs new file mode 100644 index 00000000..b61c1765 --- /dev/null +++ b/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerSqlTranslatingExpressionVisitor.cs @@ -0,0 +1,86 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.EntityFrameworkCore.Query; +using Microsoft.EntityFrameworkCore.Query.SqlExpressions; + +namespace Google.Cloud.EntityFrameworkCore.Spanner.Query.Internal +{ + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + internal class SpannerSqlTranslatingExpressionVisitor : RelationalSqlTranslatingExpressionVisitor + { + private readonly ISqlExpressionFactory _sqlExpressionFactory; + + public SpannerSqlTranslatingExpressionVisitor( + RelationalSqlTranslatingExpressionVisitorDependencies dependencies, + QueryCompilationContext queryCompilationContext, + QueryableMethodTranslatingExpressionVisitor queryableMethodTranslatingExpressionVisitor) + : base(dependencies, queryCompilationContext, queryableMethodTranslatingExpressionVisitor) + { + _sqlExpressionFactory = dependencies.SqlExpressionFactory; + } + + /// + /// Generates a SQL GREATEST expression for Cloud Spanner. + /// Cloud Spanner supports the GREATEST function with multiple arguments. + /// + public override SqlExpression? GenerateGreatest(IReadOnlyList expressions, System.Type resultType) + { + if (expressions.Count < 2) + { + return null; + } + + var resultTypeMapping = Microsoft.EntityFrameworkCore.Query.ExpressionExtensions.InferTypeMapping(expressions); + + return _sqlExpressionFactory.Function( + "GREATEST", + expressions, + nullable: true, + argumentsPropagateNullability: Enumerable.Repeat(true, expressions.Count), + resultType, + resultTypeMapping); + } + + /// + /// Generates a SQL LEAST expression for Cloud Spanner. + /// Cloud Spanner supports the LEAST function with multiple arguments. + /// + public override SqlExpression? GenerateLeast(IReadOnlyList expressions, System.Type resultType) + { + if (expressions.Count < 2) + { + return null; + } + + var resultTypeMapping = Microsoft.EntityFrameworkCore.Query.ExpressionExtensions.InferTypeMapping(expressions); + + return _sqlExpressionFactory.Function( + "LEAST", + expressions, + nullable: true, + argumentsPropagateNullability: Enumerable.Repeat(true, expressions.Count), + resultType, + resultTypeMapping); + } + } +} diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerSqlTranslatingExpressionVisitorFactory.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerSqlTranslatingExpressionVisitorFactory.cs new file mode 100644 index 00000000..af8f6e5e --- /dev/null +++ b/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerSqlTranslatingExpressionVisitorFactory.cs @@ -0,0 +1,55 @@ +// Copyright 2020, Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Microsoft.EntityFrameworkCore.Query; + +namespace Google.Cloud.EntityFrameworkCore.Spanner.Query.Internal +{ + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public class SpannerSqlTranslatingExpressionVisitorFactory : IRelationalSqlTranslatingExpressionVisitorFactory + { + private readonly RelationalSqlTranslatingExpressionVisitorDependencies _dependencies; + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public SpannerSqlTranslatingExpressionVisitorFactory( + RelationalSqlTranslatingExpressionVisitorDependencies dependencies) + { + _dependencies = dependencies; + } + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public virtual RelationalSqlTranslatingExpressionVisitor Create( + QueryCompilationContext queryCompilationContext, + QueryableMethodTranslatingExpressionVisitor queryableMethodTranslatingExpressionVisitor) + => new SpannerSqlTranslatingExpressionVisitor( + _dependencies, + queryCompilationContext, + queryableMethodTranslatingExpressionVisitor); + } +} diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerTimestampExtractExpression.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerTimestampExtractExpression.cs index f83feeb2..07cf2019 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerTimestampExtractExpression.cs +++ b/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerTimestampExtractExpression.cs @@ -70,5 +70,17 @@ public override int GetHashCode() hash *= _timezoneFragment.GetHashCode(); return hash; } + + public override Expression Quote() + { +#pragma warning disable EF9100 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + return New( + typeof(SpannerTimestampExtractExpression).GetConstructor( + [typeof(SqlExpression), typeof(SqlExpression), typeof(string), typeof(SpannerTimestampExtractExpression)])!, + _value.Quote(), + Constant(_dateTimePartName), + RelationalExpressionQuotingUtilities.QuoteTypeMapping(TypeMapping)); +#pragma warning restore EF9100 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + } } } From 0de4c50571548c06e105b8cf73c9f912b0497f97 Mon Sep 17 00:00:00 2001 From: James Abbott Date: Tue, 10 Jun 2025 14:24:26 +0100 Subject: [PATCH 2/8] ci: bump workflow tasks to net 9.0.x --- .github/workflows/build-and-test.yml | 2 +- .github/workflows/integration-tests-on-emulator.yml | 2 +- .github/workflows/integration-tests-on-production.yml | 2 +- .github/workflows/samples.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 93270532..3b34617e 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -15,7 +15,7 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v4 with: - dotnet-version: 8.0.x + dotnet-version: 9.0.x - name: Restore dependencies run: dotnet restore - name: Build diff --git a/.github/workflows/integration-tests-on-emulator.yml b/.github/workflows/integration-tests-on-emulator.yml index 967328f1..ea0511bf 100644 --- a/.github/workflows/integration-tests-on-emulator.yml +++ b/.github/workflows/integration-tests-on-emulator.yml @@ -22,7 +22,7 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v4 with: - dotnet-version: 8.0.x + dotnet-version: 9.0.x - name: Restore dependencies run: dotnet restore - name: Build diff --git a/.github/workflows/integration-tests-on-production.yml b/.github/workflows/integration-tests-on-production.yml index cd0b5cf1..ed6859f3 100644 --- a/.github/workflows/integration-tests-on-production.yml +++ b/.github/workflows/integration-tests-on-production.yml @@ -27,7 +27,7 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v4 with: - dotnet-version: 8.0.x + dotnet-version: 9.0.x - id: 'auth' uses: 'google-github-actions/auth@v2' with: diff --git a/.github/workflows/samples.yml b/.github/workflows/samples.yml index 4200d490..498884ed 100644 --- a/.github/workflows/samples.yml +++ b/.github/workflows/samples.yml @@ -13,7 +13,7 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v4 with: - dotnet-version: 8.0.x + dotnet-version: 9.0.x - name: Restore dependencies run: dotnet restore - name: Build From 810a4383bc8f452ce4a3eacc3772e99962ccb91d Mon Sep 17 00:00:00 2001 From: James Abbott Date: Tue, 10 Jun 2025 14:51:05 +0100 Subject: [PATCH 3/8] ci: make the EFMigrationsLock table name spanner compliant, and add timeouts to MigrationDatabaseLock. --- .../MigrationMockServerTests.cs | 7 ++-- .../Internal/SpannerHistoryRepository.cs | 37 +++++++++++++------ 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/Google.Cloud.EntityFrameworkCore.Spanner.Tests/MigrationTests/MigrationMockServerTests.cs b/Google.Cloud.EntityFrameworkCore.Spanner.Tests/MigrationTests/MigrationMockServerTests.cs index ef699eb8..d14ac2fe 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner.Tests/MigrationTests/MigrationMockServerTests.cs +++ b/Google.Cloud.EntityFrameworkCore.Spanner.Tests/MigrationTests/MigrationMockServerTests.cs @@ -45,13 +45,12 @@ public void TestMigrateUsesDdlBatch() var formattedVersion = $"{version.Major}.{version.Minor}.{version.Build}"; _fixture.SpannerMock.AddOrUpdateStatementResult("SELECT 1", StatementResult.CreateException(MockSpannerService.CreateDatabaseNotFoundException("d1"))); _fixture.SpannerMock.AddOrUpdateStatementResult( - "SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_catalog = '' and table_schema = '' and table_name = '''__EFMigrationsLock''')", + "SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_catalog = '' and table_schema = '' and table_name = '''EFMigrationsLock''')", StatementResult.CreateSelect1ResultSet() ); // Add mock result for the INSERT OR IGNORE migration lock statement - this has a dynamic timestamp so we'll use a pattern - // The SQL will be something like: INSERT OR IGNORE INTO "__EFMigrationsLock"("Id", "Timestamp") VALUES(1, '2025-06-09 20:16:59.350519+00:00'); SELECT changes(); _fixture.SpannerMock.AddOrUpdatePatternResult( - "INSERT OR IGNORE INTO \"__EFMigrationsLock\"(\"Id\", \"Timestamp\") VALUES(1, '*');\nSELECT changes();", + "INSERT OR IGNORE INTO \"EFMigrationsLock\"(\"Id\", \"Timestamp\") VALUES(1, '*');\nSELECT changes();", StatementResult.CreateSingleColumnResultSet(new V1.Type { Code = V1.TypeCode.Int64 }, "changes", 1L) ); _fixture.SpannerMock.AddOrUpdateStatementResult( @@ -63,7 +62,7 @@ public void TestMigrateUsesDdlBatch() StatementResult.CreateUpdateCount(1) ); _fixture.SpannerMock.AddOrUpdateStatementResult( - "DELETE FROM \"__EFMigrationsLock\";", + "DELETE FROM \"EFMigrationsLock\";", StatementResult.CreateUpdateCount(1) ); using var db = new MockMigrationSampleDbContext(ConnectionString); diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerHistoryRepository.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerHistoryRepository.cs index 45066abe..7e415b7f 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerHistoryRepository.cs +++ b/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerHistoryRepository.cs @@ -118,7 +118,7 @@ public override string GetEndIfScript() /// /// The name of the table that will serve as a database-wide lock for migrations. /// - protected virtual string LockTableName { get; } = "__EFMigrationsLock"; + protected virtual string LockTableName { get; } = "EFMigrationsLock"; /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to @@ -136,6 +136,7 @@ public override string GetEndIfScript() /// public override IMigrationsDatabaseLock AcquireDatabaseLock() { + // TODO: Merge async and sync methods to avoid code duplication. Dependencies.MigrationsLogger.AcquiringMigrationLock(); if (!InterpretExistsResult( @@ -145,8 +146,11 @@ public override IMigrationsDatabaseLock AcquireDatabaseLock() CreateLockTableCommand().ExecuteNonQuery(CreateRelationalCommandParameters()); } + // TODO: Inject with settings for timeout + var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + var retryDelay = _retryDelay; - while (true) + while (!timeout.IsCancellationRequested) { long insertCount = 0; var dbLock = CreateMigrationDatabaseLock(); @@ -173,6 +177,8 @@ public override IMigrationsDatabaseLock AcquireDatabaseLock() retryDelay = retryDelay.Add(retryDelay); } } + + throw new TimeoutException("Failed to acquire migration lock within the specified timeout."); } /// @@ -194,8 +200,11 @@ await CreateLockTableCommand().ExecuteNonQueryAsync(CreateRelationalCommandParam .ConfigureAwait(false); } + // TODO: Inject with settings for timeout + var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + var retryDelay = _retryDelay; - while (true) + while (!timeout.IsCancellationRequested) { var dbLock = CreateMigrationDatabaseLock(); var insertCount = await CreateInsertLockCommand(DateTimeOffset.UtcNow) @@ -212,15 +221,17 @@ await CreateLockTableCommand().ExecuteNonQueryAsync(CreateRelationalCommandParam retryDelay = retryDelay.Add(retryDelay); } } + + throw new TimeoutException("Failed to acquire migration lock within the specified timeout."); } private IRelationalCommand CreateLockTableCommand() => Dependencies.RawSqlCommandBuilder.Build( $""" -CREATE TABLE IF NOT EXISTS "{LockTableName}" ( - "Id" INTEGER NOT NULL CONSTRAINT "PK_{LockTableName}" PRIMARY KEY, - "Timestamp" TEXT NOT NULL -); +CREATE TABLE IF NOT EXISTS {LockTableName} ( + Id INT64 NOT NULL, + Timestamp STRING(256) NOT NULL +) PRIMARY KEY (Id) """); private IRelationalCommand CreateInsertLockCommand(DateTimeOffset timestamp) @@ -229,19 +240,23 @@ private IRelationalCommand CreateInsertLockCommand(DateTimeOffset timestamp) return Dependencies.RawSqlCommandBuilder.Build( $""" -INSERT OR IGNORE INTO "{LockTableName}"("Id", "Timestamp") VALUES(1, {timestampLiteral}); -SELECT changes(); +INSERT OR IGNORE INTO {LockTableName}(Id, Timestamp) VALUES(1, {timestampLiteral}) +THEN RETURN 1 """); } private IRelationalCommand CreateDeleteLockCommand(int? id = null) { var sql = $""" -DELETE FROM "{LockTableName}" +DELETE FROM {LockTableName} """; if (id != null) { - sql += $""" WHERE "Id" = {id}"""; + sql += $""" WHERE Id = {id}"""; + } + else + { + sql += " WHERE 1 = 1"; } sql += ";"; From 2f82900d416b1df16d88cccfb8361120f1955aff Mon Sep 17 00:00:00 2001 From: James Abbott Date: Tue, 10 Jun 2025 15:18:16 +0100 Subject: [PATCH 4/8] fix: respect the commit/begin transaction flags from execution state --- .../SpannerMigrationCommandExecutor.cs | 89 +++++-------------- 1 file changed, 22 insertions(+), 67 deletions(-) diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerMigrationCommandExecutor.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerMigrationCommandExecutor.cs index 88c44920..aa531a9c 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerMigrationCommandExecutor.cs +++ b/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerMigrationCommandExecutor.cs @@ -45,21 +45,21 @@ public int ExecuteNonQuery(IReadOnlyList migrationCommands, IR throw new NotSupportedException("Cannot execute transaction suppressed migration commands in user transaction."); } + var cancellationToken = CancellationToken.None; using var transactionScope = new TransactionScope(TransactionScopeOption.Suppress, TransactionScopeAsyncFlowOption.Enabled); - return executionStrategy.Execute( - (migrationCommands, connection, inUserTransaction, executionState, commitTransaction, isolationLevel), - static (_, s) => ExecuteInternal( + return executionStrategy.ExecuteAsync( + (migrationCommands, connection, inUserTransaction, commitTransaction, isolationLevel), + static (_, s, ct) => ExecuteInternalAsync( s.migrationCommands, s.connection, - s.executionState, beginTransaction: !s.inUserTransaction, commitTransaction: !s.inUserTransaction && s.commitTransaction, - s.isolationLevel), - verifySucceeded: null); + s.isolationLevel, + ct), + verifySucceeded: null, cancellationToken).ResultWithUnwrappedExceptions(); } - public async Task ExecuteNonQueryAsync(IEnumerable migrationCommands, IRelationalConnection connection, CancellationToken cancellationToken = default) { await ExecuteNonQueryAsync(migrationCommands.ToList(), connection, new MigrationExecutionState(), commitTransaction: true, System.Data.IsolationLevel.Unspecified, cancellationToken).ConfigureAwait(false); @@ -79,11 +79,10 @@ public async Task ExecuteNonQueryAsync(IReadOnlyList migr using var transactionScope = new TransactionScope(TransactionScopeOption.Suppress, TransactionScopeAsyncFlowOption.Enabled); return await executionStrategy.ExecuteAsync( - (migrationCommands, connection, inUserTransaction, executionState, commitTransaction, isolationLevel), + (migrationCommands, connection, inUserTransaction, commitTransaction, isolationLevel), static (_, s, ct) => ExecuteInternalAsync( s.migrationCommands, s.connection, - s.executionState, beginTransaction: !s.inUserTransaction, commitTransaction: !s.inUserTransaction && s.commitTransaction, s.isolationLevel, @@ -92,64 +91,9 @@ public async Task ExecuteNonQueryAsync(IReadOnlyList migr cancellationToken).ConfigureAwait(false); } - private static int ExecuteInternal( - IReadOnlyList migrationCommands, - IRelationalConnection connection, - MigrationExecutionState executionState, - bool beginTransaction, - bool commitTransaction, - System.Data.IsolationLevel? isolationLevel) - { - var result = 0; - var connectionOpened = connection.Open(); - - try - { - var statements = migrationCommands.Select(x => x.CommandText).ToArray(); - if (statements.Length == 0) - { - return result; - } - - var ddlStatements = statements.Where(IsDdlStatement).ToArray(); - var otherStatements = statements.Where(x => !IsDdlStatement(x)).ToArray(); - var spannerConnection = ((SpannerRelationalConnection)connection).DbConnection as SpannerRetriableConnection; - - if (ddlStatements.Any()) - { - var cmd = spannerConnection.CreateDdlCommand(ddlStatements[0], ddlStatements.Skip(1).ToArray()); - result += cmd.ExecuteNonQuery(); - } - - if (otherStatements.Any()) - { - using var transaction = spannerConnection.BeginTransaction(); - var cmd = spannerConnection.CreateBatchDmlCommand(); - cmd.Transaction = transaction; - foreach (var statement in otherStatements) - { - cmd.Add(statement); - } - // Batch DML returns IReadOnlyList, so sum the update counts - var updateCounts = cmd.ExecuteNonQuery(); - result += (int)updateCounts.Sum(); - transaction.Commit(); - } - } - catch - { - connection.Close(); - throw; - } - - connection.Close(); - return result; - } - private static async Task ExecuteInternalAsync( IReadOnlyList migrationCommands, IRelationalConnection connection, - MigrationExecutionState executionState, bool beginTransaction, bool commitTransaction, System.Data.IsolationLevel? isolationLevel, @@ -178,9 +122,15 @@ private static async Task ExecuteInternalAsync( if (otherStatements.Any()) { - using var transaction = await spannerConnection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + using var transaction = beginTransaction ? await spannerConnection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false) : null; + var cmd = spannerConnection.CreateBatchDmlCommand(); - cmd.Transaction = transaction; + + if (transaction != null) + { + cmd.Transaction = transaction; + } + foreach (var statement in otherStatements) { cmd.Add(statement); @@ -188,7 +138,12 @@ private static async Task ExecuteInternalAsync( // Batch DML returns IReadOnlyList, so sum the update counts var updateCounts = await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); result += (int)updateCounts.Sum(); - await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + + if (commitTransaction && transaction != null) + { + // Commit the transaction if required + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + } } } catch From 7868ba9fa3b2e0fad9b15ad4969b4d4c72c0d8d8 Mon Sep 17 00:00:00 2001 From: James Abbott Date: Tue, 10 Jun 2025 15:49:40 +0100 Subject: [PATCH 5/8] fix: respect transaction state from ef migration manager. --- .../TestTableSplittingTest.cs | 2 +- .../SpannerMigrationCommandExecutor.cs | 183 ++++++++++++++---- 2 files changed, 145 insertions(+), 40 deletions(-) diff --git a/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests/TableSplittingTests/TestTableSplittingTest.cs b/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests/TableSplittingTests/TestTableSplittingTest.cs index a82a4fe7..dfd5aef1 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests/TableSplittingTests/TestTableSplittingTest.cs +++ b/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests/TableSplittingTests/TestTableSplittingTest.cs @@ -31,7 +31,7 @@ public async Task ShouldGenerateOneTable() var cmd = connection.CreateSelectCommand( "SELECT COUNT(*) " + "FROM INFORMATION_SCHEMA.TABLES " + - "WHERE TABLE_CATALOG='' AND TABLE_SCHEMA='' AND TABLE_NAME != 'EFMigrationsHistory'"); + "WHERE TABLE_CATALOG='' AND TABLE_SCHEMA='' AND TABLE_NAME NOT IN('EFMigrationsHistory', 'EFMigrationsLock')"); using var reader = await cmd.ExecuteReaderAsync(); Assert.True(await reader.ReadAsync()); diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerMigrationCommandExecutor.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerMigrationCommandExecutor.cs index aa531a9c..757783ae 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerMigrationCommandExecutor.cs +++ b/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerMigrationCommandExecutor.cs @@ -45,19 +45,18 @@ public int ExecuteNonQuery(IReadOnlyList migrationCommands, IR throw new NotSupportedException("Cannot execute transaction suppressed migration commands in user transaction."); } - var cancellationToken = CancellationToken.None; using var transactionScope = new TransactionScope(TransactionScopeOption.Suppress, TransactionScopeAsyncFlowOption.Enabled); - return executionStrategy.ExecuteAsync( - (migrationCommands, connection, inUserTransaction, commitTransaction, isolationLevel), - static (_, s, ct) => ExecuteInternalAsync( + return executionStrategy.Execute( + (migrationCommands, connection, executionState, inUserTransaction, commitTransaction, isolationLevel), + static (_, s) => Execute( s.migrationCommands, s.connection, + s.executionState, beginTransaction: !s.inUserTransaction, commitTransaction: !s.inUserTransaction && s.commitTransaction, - s.isolationLevel, - ct), - verifySucceeded: null, cancellationToken).ResultWithUnwrappedExceptions(); + s.isolationLevel), + verifySucceeded: null); } public async Task ExecuteNonQueryAsync(IEnumerable migrationCommands, IRelationalConnection connection, CancellationToken cancellationToken = default) @@ -79,10 +78,11 @@ public async Task ExecuteNonQueryAsync(IReadOnlyList migr using var transactionScope = new TransactionScope(TransactionScopeOption.Suppress, TransactionScopeAsyncFlowOption.Enabled); return await executionStrategy.ExecuteAsync( - (migrationCommands, connection, inUserTransaction, commitTransaction, isolationLevel), - static (_, s, ct) => ExecuteInternalAsync( + (migrationCommands, connection, executionState, inUserTransaction, commitTransaction, isolationLevel), + static (_, s, ct) => ExecuteAsync( s.migrationCommands, s.connection, + s.executionState, beginTransaction: !s.inUserTransaction, commitTransaction: !s.inUserTransaction && s.commitTransaction, s.isolationLevel, @@ -91,63 +91,168 @@ public async Task ExecuteNonQueryAsync(IReadOnlyList migr cancellationToken).ConfigureAwait(false); } - private static async Task ExecuteInternalAsync( + private static int Execute( IReadOnlyList migrationCommands, IRelationalConnection connection, + MigrationExecutionState executionState, bool beginTransaction, bool commitTransaction, - System.Data.IsolationLevel? isolationLevel, - CancellationToken cancellationToken) + System.Data.IsolationLevel? isolationLevel) { var result = 0; - var connectionOpened = await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + var connectionOpened = connection.Open(); try { - var statements = migrationCommands.Select(x => x.CommandText).ToArray(); - if (statements.Length == 0) + for (var i = executionState.LastCommittedCommandIndex; i < migrationCommands.Count; i++) { - return result; + var command = migrationCommands[i]; + if (executionState.Transaction == null + && !command.TransactionSuppressed + && beginTransaction) + { + executionState.Transaction = isolationLevel == null + ? connection.BeginTransaction() + : connection.BeginTransaction(isolationLevel.Value); + if (executionState.DatabaseLock != null) + { + executionState.DatabaseLock = executionState.DatabaseLock.ReacquireIfNeeded( + connectionOpened, transactionRestarted: true); + connectionOpened = false; + } + } + + if (executionState.Transaction != null + && command.TransactionSuppressed) + { + executionState.Transaction.Commit(); + executionState.Transaction.Dispose(); + executionState.Transaction = null; + executionState.LastCommittedCommandIndex = i; + executionState.AnyOperationPerformed = true; + + if (executionState.DatabaseLock != null) + { + executionState.DatabaseLock = executionState.DatabaseLock.ReacquireIfNeeded( + connectionOpened, transactionRestarted: null); + connectionOpened = false; + } + } + + result = command.ExecuteNonQuery(connection); + + if (executionState.Transaction == null) + { + executionState.LastCommittedCommandIndex = i + 1; + executionState.AnyOperationPerformed = true; + } } - - var ddlStatements = statements.Where(IsDdlStatement).ToArray(); - var otherStatements = statements.Where(x => !IsDdlStatement(x)).ToArray(); - var spannerConnection = ((SpannerRelationalConnection)connection).DbConnection as SpannerRetriableConnection; - - if (ddlStatements.Any()) + + if (commitTransaction + && executionState.Transaction != null) { - var cmd = spannerConnection.CreateDdlCommand(ddlStatements[0], ddlStatements.Skip(1).ToArray()); - result += await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + executionState.Transaction.Commit(); + executionState.Transaction.Dispose(); + executionState.Transaction = null; } - - if (otherStatements.Any()) - { - using var transaction = beginTransaction ? await spannerConnection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false) : null; + } + catch + { + executionState.Transaction?.Dispose(); + executionState.Transaction = null; + connection.Close(); + throw; + } + + connection.Close(); + return result; + } - var cmd = spannerConnection.CreateBatchDmlCommand(); + private static async Task ExecuteAsync( + IReadOnlyList migrationCommands, + IRelationalConnection connection, + MigrationExecutionState executionState, + bool beginTransaction, + bool commitTransaction, + System.Data.IsolationLevel? isolationLevel, + CancellationToken cancellationToken) + { + var result = 0; + var connectionOpened = await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + var spannerConnection = ((SpannerRelationalConnection)connection).DbConnection as SpannerRetriableConnection; - if (transaction != null) + try + { + for (var i = executionState.LastCommittedCommandIndex; i < migrationCommands.Count; i++) + { + var lockReacquired = false; + var command = migrationCommands[i]; + if (executionState.Transaction == null + && !command.TransactionSuppressed + && beginTransaction) { - cmd.Transaction = transaction; + executionState.Transaction = await (isolationLevel == null + ? connection.BeginTransactionAsync(cancellationToken) + : connection.BeginTransactionAsync(isolationLevel.Value, cancellationToken)) + .ConfigureAwait(false); + + if (executionState.DatabaseLock != null) + { + executionState.DatabaseLock = await executionState.DatabaseLock.ReacquireIfNeededAsync( + connectionOpened, transactionRestarted: true, cancellationToken) + .ConfigureAwait(false); + lockReacquired = true; + } } - foreach (var statement in otherStatements) + if (executionState.Transaction != null + && command.TransactionSuppressed) { - cmd.Add(statement); + await executionState.Transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + await executionState.Transaction.DisposeAsync().ConfigureAwait(false); + executionState.Transaction = null; + executionState.LastCommittedCommandIndex = i; + executionState.AnyOperationPerformed = true; + + if (executionState.DatabaseLock != null + && !lockReacquired) + { + executionState.DatabaseLock = await executionState.DatabaseLock.ReacquireIfNeededAsync( + connectionOpened, transactionRestarted: null, cancellationToken) + .ConfigureAwait(false); + } } - // Batch DML returns IReadOnlyList, so sum the update counts - var updateCounts = await cmd.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); - result += (int)updateCounts.Sum(); - if (commitTransaction && transaction != null) + var spannerCommand = IsDdlStatement(command.CommandText) ? + spannerConnection.CreateDdlCommand(command.CommandText) + : spannerConnection.CreateDmlCommand(command.CommandText); + + // spannerCommand.Transaction = executionState.Transaction/ + result = await spannerCommand.ExecuteNonQueryAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); + + if (executionState.Transaction == null) { - // Commit the transaction if required - await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + executionState.LastCommittedCommandIndex = i + 1; + executionState.AnyOperationPerformed = true; } } + + if (commitTransaction + && executionState.Transaction != null) + { + await executionState.Transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + await executionState.Transaction.DisposeAsync().ConfigureAwait(false); + executionState.Transaction = null; + } } catch { + if (executionState.Transaction != null) + { + await executionState.Transaction.DisposeAsync().ConfigureAwait(false); + executionState.Transaction = null; + } await connection.CloseAsync().ConfigureAwait(false); throw; } From 9a6cdf537f552858c5fbcbcc1063f048a30d0f9b Mon Sep 17 00:00:00 2001 From: James Abbott Date: Tue, 10 Jun 2025 16:07:56 +0100 Subject: [PATCH 6/8] ci: fix test assertions for literal escaped tables --- .../MigrationTests/MigrationMockServerTests.cs | 4 ++-- .../Migrations/Internal/SpannerHistoryRepository.cs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Google.Cloud.EntityFrameworkCore.Spanner.Tests/MigrationTests/MigrationMockServerTests.cs b/Google.Cloud.EntityFrameworkCore.Spanner.Tests/MigrationTests/MigrationMockServerTests.cs index d14ac2fe..c35b7215 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner.Tests/MigrationTests/MigrationMockServerTests.cs +++ b/Google.Cloud.EntityFrameworkCore.Spanner.Tests/MigrationTests/MigrationMockServerTests.cs @@ -50,7 +50,7 @@ public void TestMigrateUsesDdlBatch() ); // Add mock result for the INSERT OR IGNORE migration lock statement - this has a dynamic timestamp so we'll use a pattern _fixture.SpannerMock.AddOrUpdatePatternResult( - "INSERT OR IGNORE INTO \"EFMigrationsLock\"(\"Id\", \"Timestamp\") VALUES(1, '*');\nSELECT changes();", + "INSERT OR IGNORE INTO `EFMigrationsLock`(`Id`, `Timestamp`) VALUES(1, '*')\nTHEN RETURN 1", StatementResult.CreateSingleColumnResultSet(new V1.Type { Code = V1.TypeCode.Int64 }, "changes", 1L) ); _fixture.SpannerMock.AddOrUpdateStatementResult( @@ -62,7 +62,7 @@ public void TestMigrateUsesDdlBatch() StatementResult.CreateUpdateCount(1) ); _fixture.SpannerMock.AddOrUpdateStatementResult( - "DELETE FROM \"EFMigrationsLock\";", + "DELETE FROM `EFMigrationsLock` WHERE 1 = 1;", StatementResult.CreateUpdateCount(1) ); using var db = new MockMigrationSampleDbContext(ConnectionString); diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerHistoryRepository.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerHistoryRepository.cs index 7e415b7f..e4fdc45f 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerHistoryRepository.cs +++ b/Google.Cloud.EntityFrameworkCore.Spanner/Migrations/Internal/SpannerHistoryRepository.cs @@ -240,7 +240,7 @@ private IRelationalCommand CreateInsertLockCommand(DateTimeOffset timestamp) return Dependencies.RawSqlCommandBuilder.Build( $""" -INSERT OR IGNORE INTO {LockTableName}(Id, Timestamp) VALUES(1, {timestampLiteral}) +INSERT OR IGNORE INTO `{LockTableName}`(`Id`, `Timestamp`) VALUES(1, {timestampLiteral}) THEN RETURN 1 """); } @@ -248,7 +248,7 @@ THEN RETURN 1 private IRelationalCommand CreateDeleteLockCommand(int? id = null) { var sql = $""" -DELETE FROM {LockTableName} +DELETE FROM `{LockTableName}` """; if (id != null) { From 7f756337f2f0c5f6a77c2b5a3f6e0cd386483528 Mon Sep 17 00:00:00 2001 From: James Abbott Date: Wed, 11 Jun 2025 09:26:18 +0100 Subject: [PATCH 7/8] fix: resolve build warnings and obsolete members --- .../SpannerEFCoreBenchmarks.cs | 2 +- .../Query/Internal/SpannerSqlTranslatingExpressionVisitor.cs | 4 ++-- .../Storage/Internal/SpannerRetriableConnection.cs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Google.Cloud.EntityFrameworkCore.Spanner.Benchmarks/SpannerEFCoreBenchmarks.cs b/Google.Cloud.EntityFrameworkCore.Spanner.Benchmarks/SpannerEFCoreBenchmarks.cs index 1b2ed56e..22a9f6e2 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner.Benchmarks/SpannerEFCoreBenchmarks.cs +++ b/Google.Cloud.EntityFrameworkCore.Spanner.Benchmarks/SpannerEFCoreBenchmarks.cs @@ -381,7 +381,7 @@ public List SelectMultipleSingersEF() public List SelectMultipleSingersInReadOnlyTransactionSpanner() { using var connection = CreateConnection(); - using var transaction = connection.BeginReadOnlyTransaction(); + using var transaction = connection.BeginTransaction(SpannerTransactionCreationOptions.ReadOnly, new SpannerTransactionOptions()); using var command = connection.CreateSelectCommand("SELECT * FROM Singers ORDER BY LastName"); command.Transaction = transaction; using var reader = command.ExecuteReader(); diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerSqlTranslatingExpressionVisitor.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerSqlTranslatingExpressionVisitor.cs index b61c1765..7c5ebac8 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerSqlTranslatingExpressionVisitor.cs +++ b/Google.Cloud.EntityFrameworkCore.Spanner/Query/Internal/SpannerSqlTranslatingExpressionVisitor.cs @@ -43,7 +43,7 @@ public SpannerSqlTranslatingExpressionVisitor( /// Generates a SQL GREATEST expression for Cloud Spanner. /// Cloud Spanner supports the GREATEST function with multiple arguments. /// - public override SqlExpression? GenerateGreatest(IReadOnlyList expressions, System.Type resultType) + public override SqlExpression GenerateGreatest(IReadOnlyList expressions, System.Type resultType) { if (expressions.Count < 2) { @@ -65,7 +65,7 @@ public SpannerSqlTranslatingExpressionVisitor( /// Generates a SQL LEAST expression for Cloud Spanner. /// Cloud Spanner supports the LEAST function with multiple arguments. /// - public override SqlExpression? GenerateLeast(IReadOnlyList expressions, System.Type resultType) + public override SqlExpression GenerateLeast(IReadOnlyList expressions, System.Type resultType) { if (expressions.Count < 2) { diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Storage/Internal/SpannerRetriableConnection.cs b/Google.Cloud.EntityFrameworkCore.Spanner/Storage/Internal/SpannerRetriableConnection.cs index 71c8462e..09d775e8 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner/Storage/Internal/SpannerRetriableConnection.cs +++ b/Google.Cloud.EntityFrameworkCore.Spanner/Storage/Internal/SpannerRetriableConnection.cs @@ -83,7 +83,7 @@ protected override void Dispose(bool disposing) /// The read timestamp to use for the read-only transaction. /// A new read-only transaction with the specified public SpannerReadOnlyTransaction BeginReadOnlyTransaction(TimestampBound timestampBound) => - new SpannerReadOnlyTransaction(this, SpannerConnection.BeginReadOnlyTransaction(timestampBound)); + new SpannerReadOnlyTransaction(this, SpannerConnection.BeginTransaction(SpannerTransactionCreationOptions.ForTimestampBoundReadOnly(timestampBound), new SpannerTransactionOptions())); /// /// Begins a read-only transaction with @@ -99,7 +99,7 @@ public Task BeginReadOnlyTransactionAsync(Cancellati /// A cancellation token to monitor for the asynchronous operation. /// A new read-only transaction with the specified public async Task BeginReadOnlyTransactionAsync(TimestampBound timestampBound, CancellationToken cancellationToken = default) => - new SpannerReadOnlyTransaction(this, await SpannerConnection.BeginReadOnlyTransactionAsync(timestampBound, cancellationToken)); + new SpannerReadOnlyTransaction(this, await SpannerConnection.BeginTransactionAsync(SpannerTransactionCreationOptions.ForTimestampBoundReadOnly(timestampBound), new SpannerTransactionOptions(), cancellationToken)); /// /// Begins a new read/write transaction on the connection. The transaction will automatically be From 72ce9465695cbe163fb38948ad4de0f2c027f25a Mon Sep 17 00:00:00 2001 From: James Abbott Date: Tue, 29 Jul 2025 12:34:44 +0100 Subject: [PATCH 8/8] chore: bump net sdk packages --- ...EntityFrameworkCore.Spanner.IntegrationTests.csproj | 4 ++-- ...le.Cloud.EntityFrameworkCore.Spanner.Samples.csproj | 3 +-- ...ogle.Cloud.EntityFrameworkCore.Spanner.Tests.csproj | 4 ++-- .../Google.Cloud.EntityFrameworkCore.Spanner.csproj | 10 +++++----- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests.csproj b/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests.csproj index 4462d4aa..e6c86360 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests.csproj +++ b/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests/Google.Cloud.EntityFrameworkCore.Spanner.IntegrationTests.csproj @@ -19,8 +19,8 @@ - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Google.Cloud.EntityFrameworkCore.Spanner.Samples/Google.Cloud.EntityFrameworkCore.Spanner.Samples.csproj b/Google.Cloud.EntityFrameworkCore.Spanner.Samples/Google.Cloud.EntityFrameworkCore.Spanner.Samples.csproj index 3d6e7522..2552aa52 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner.Samples/Google.Cloud.EntityFrameworkCore.Spanner.Samples.csproj +++ b/Google.Cloud.EntityFrameworkCore.Spanner.Samples/Google.Cloud.EntityFrameworkCore.Spanner.Samples.csproj @@ -4,7 +4,6 @@ net9.0 SampleRunner Exe - net8.0 @@ -25,7 +24,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Google.Cloud.EntityFrameworkCore.Spanner.Tests/Google.Cloud.EntityFrameworkCore.Spanner.Tests.csproj b/Google.Cloud.EntityFrameworkCore.Spanner.Tests/Google.Cloud.EntityFrameworkCore.Spanner.Tests.csproj index 45596ed3..adbeac6d 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner.Tests/Google.Cloud.EntityFrameworkCore.Spanner.Tests.csproj +++ b/Google.Cloud.EntityFrameworkCore.Spanner.Tests/Google.Cloud.EntityFrameworkCore.Spanner.Tests.csproj @@ -18,11 +18,11 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Google.Cloud.EntityFrameworkCore.Spanner/Google.Cloud.EntityFrameworkCore.Spanner.csproj b/Google.Cloud.EntityFrameworkCore.Spanner/Google.Cloud.EntityFrameworkCore.Spanner.csproj index 46f30d91..94b48fd2 100644 --- a/Google.Cloud.EntityFrameworkCore.Spanner/Google.Cloud.EntityFrameworkCore.Spanner.csproj +++ b/Google.Cloud.EntityFrameworkCore.Spanner/Google.Cloud.EntityFrameworkCore.Spanner.csproj @@ -19,13 +19,13 @@ - + - - - + + + - +