From faf4903e24a4262de232be5503fecb31fe1feab8 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 10 Dec 2025 04:31:04 -0800 Subject: [PATCH] feat: Enable full object checksum validation for multi-chunk resumable uploads --- .../Upload/ResumableUploadTest.MultiChunk.cs | 110 ++++++++++++++++ .../Upload/ResumableUploadTest.Utilities.cs | 10 ++ Src/Support/Google.Apis/Google.Apis.csproj | 2 + .../Google.Apis/Upload/ResumableUpload.cs | 118 ++++++++++++++++++ 4 files changed, 240 insertions(+) diff --git a/Src/Support/Google.Apis.Tests/Apis/Upload/ResumableUploadTest.MultiChunk.cs b/Src/Support/Google.Apis.Tests/Apis/Upload/ResumableUploadTest.MultiChunk.cs index ef71117f9bf..df5ff0785cb 100644 --- a/Src/Support/Google.Apis.Tests/Apis/Upload/ResumableUploadTest.MultiChunk.cs +++ b/Src/Support/Google.Apis.Tests/Apis/Upload/ResumableUploadTest.MultiChunk.cs @@ -500,6 +500,103 @@ public void TestUploadInPartialServer( Assert.Equal(UploadStatus.Completed, progress.Status); } } +#if NET6_0_OR_GREATER + /// + /// Verifies that a resumable upload completes successfully and validates crc32c checksum across different chunking configurations + /// and stream types of known size. + /// + [Theory, CombinatorialData] + public void TestChunkedUpload_CompletesAndVerifiesChecksumInPartialServer( + [CombinatorialValues(true, false)] bool knownSize, + [CombinatorialValues(80, 150)] int partialSize, + [CombinatorialValues(100, 200)] int chunkSize) + { + var actualChunkSize = Math.Min(partialSize, chunkSize); + var expectedCallCount = (UploadLength + actualChunkSize - 1) / actualChunkSize + 1; + using (var server = new MultiChunkPartialServer(_server, partialSize)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var content = knownSize ? new MemoryStream(UploadTestBytes) : new UnknownSizeMemoryStream(UploadTestBytes); + var uploader = new TestResumableUpload(service, "MultiChunk", "POST", content, "text/plain", actualChunkSize); + var progress = uploader.Upload(); + Assert.Equal(expectedCallCount, server.Requests.Count); + var request = server.Requests.Last(); + + if (knownSize) + { + Assert.Contains(ResumableUpload.GoogleHashHeader, request.Headers.AllKeys); + var headerValue = request.Headers[ResumableUpload.GoogleHashHeader]; + var expectedBase64Hash = CalculateCrc32c(UploadTestBytes); + Assert.Equal($"crc32c={expectedBase64Hash}", headerValue); + } + else + { + Assert.DoesNotContain(ResumableUpload.GoogleHashHeader, request.Headers.AllKeys); + } + + Assert.Equal(UploadTestBytes, server.Bytes); + Assert.Equal(UploadStatus.Completed, progress.Status); + } + } + + /// + /// Verifies that the session initiation succeeds when a matching CRC32C checksum is provided. + /// + [Fact] + public async Task TestInitiateSessionWithMatchingCrc32c() + { + var content = new MemoryStream(UploadTestBytes); + using (var server = new MultiChunkServer(_server)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var chunkSize = 1024; + var expectedBase64Hash = CalculateCrc32c(UploadTestBytes); + var body = new TestResumableUploadWithCrc32c { Crc32c = expectedBase64Hash }; + var uploader = new TestResumableUpload(service, "MultiChunk", "POST", content, "text/plain", chunkSize) + { + Body = body + }; + + await uploader.InitiateSessionAsync(); + // Assert that the initiation request was sent + Assert.Single(server.Requests); + var initiationRequest = server.Requests.First(); + + // Assert that the X-Goog-Hash header was sent with the correct CRC32C + Assert.Contains(ResumableUpload.GoogleHashHeader, initiationRequest.Headers.AllKeys); + Assert.Equal($"crc32c={expectedBase64Hash}", initiationRequest.Headers[ResumableUpload.GoogleHashHeader]); + Assert.True(uploader.Crc32cHeaderSentInInitiation); + } + } + + /// + /// Verifies that the session initiation fails when a mismatched CRC32C checksum is provided. + /// + [Fact] + public async Task TestInitiateSessionWithMismatchingCrc32c() + { + var content = new MemoryStream(UploadTestBytes); + using (var server = new MultiChunkServer(_server)) + using (var service = new MockClientService(server.HttpPrefix)) + { + var chunkSize = 1024; + var expectedBase64Hash = CalculateCrc32c(UploadTestBytes); + var body = new TestResumableUploadWithCrc32c { Crc32c = "incorrectHash" }; + + var uploader = new TestResumableUpload(service, "MultiChunk", "POST", content, "text/plain", chunkSize) + { + Body = body + }; + + var exception = await Assert.ThrowsAsync(() => uploader.InitiateSessionAsync()); + Assert.Equal($"The calculated CRC32C of the stream ({expectedBase64Hash}) does not match the CRC32C provided in the object metadata ({body.Crc32c}).", exception.Message); + + // Assert that no request was sent to the server + Assert.Empty(server.Requests); + Assert.False(uploader.Crc32cHeaderSentInInitiation); + } + } +#endif /// /// Server that expects an initial call with path and query parameters. @@ -697,5 +794,18 @@ public void TestUploadWithRequestAndResponseBody( Assert.Equal(1, reponseReceivedCount); } } + +#if NET6_0_OR_GREATER + /// + /// Calculates the CRC32C checksum for the specified data. + /// + private string CalculateCrc32c(byte[] uploadTestBytes) + { + var crc32 = new System.IO.Hashing.Crc32(); + crc32.Append(uploadTestBytes); + byte[] hashBytes = crc32.GetCurrentHash(); + return Convert.ToBase64String(hashBytes); + } +#endif } } diff --git a/Src/Support/Google.Apis.Tests/Apis/Upload/ResumableUploadTest.Utilities.cs b/Src/Support/Google.Apis.Tests/Apis/Upload/ResumableUploadTest.Utilities.cs index f735237f11c..3b95a43d26e 100644 --- a/Src/Support/Google.Apis.Tests/Apis/Upload/ResumableUploadTest.Utilities.cs +++ b/Src/Support/Google.Apis.Tests/Apis/Upload/ResumableUploadTest.Utilities.cs @@ -92,5 +92,15 @@ internal TestLogger(ITestOutputHelper outputHelper) internal void WriteLine(string message, [CallerMemberName] string caller = null) => _outputHelper.WriteLine($"Test {_id:0000} at {DateTime.UtcNow:HH:mm:ss.fff}: {caller} - {message}"); } + +#if NET6_0_OR_GREATER + /// + /// Resumable upload class that allows Crc32c to be set for testing. + /// + private class TestResumableUploadWithCrc32c + { + public string Crc32c { get; set; } + } +#endif } } diff --git a/Src/Support/Google.Apis/Google.Apis.csproj b/Src/Support/Google.Apis/Google.Apis.csproj index 56d5967b5cc..327f346a293 100644 --- a/Src/Support/Google.Apis/Google.Apis.csproj +++ b/Src/Support/Google.Apis/Google.Apis.csproj @@ -22,6 +22,8 @@ The library supports service requests, media upload and download, etc. + + diff --git a/Src/Support/Google.Apis/Upload/ResumableUpload.cs b/Src/Support/Google.Apis/Upload/ResumableUpload.cs index 9dd3f75bfa1..f75335654f0 100644 --- a/Src/Support/Google.Apis/Upload/ResumableUpload.cs +++ b/Src/Support/Google.Apis/Upload/ResumableUpload.cs @@ -72,6 +72,19 @@ public abstract class ResumableUpload /// The x-goog-api-client header value used for resumable uploads initiated without any options or an HttpClient. /// private static readonly string DefaultGoogleApiClientHeader = new VersionHeaderBuilder().AppendDotNetEnvironment().AppendAssemblyVersion("gdcl", typeof(ResumableUpload)).ToString(); + + /// + /// The header name used to specify the crc32c hash of the stream for validation. + /// + internal const string GoogleHashHeader = "X-Goog-Hash"; + +#if NET6_0_OR_GREATER + /// + /// The size of the buffer (in bytes) used when reading the stream to calculate the CRC32C checksum. + /// Set to 80 KB (81920 bytes) to balance memory usage and I/O performance. + /// + private const int Crc32cCalculationBufferSize = 81920; +#endif #endregion // Constants #region Construction @@ -142,6 +155,12 @@ public InitiatedResumableUpload(Uri uploadUri, Stream contentStream, ResumableUp /// internal ConfigurableHttpClient HttpClient { get; } + /// + /// Gets or sets a value indicating whether the CRC32C checksum header was included + /// during the initiation of the resumable upload session. + /// + public bool Crc32cHeaderSentInInitiation { get; internal set; } + /// Gets or sets the stream to upload. public Stream ContentStream { get; } @@ -641,6 +660,24 @@ protected async Task SendNextChunkAsync(Stream stream, CancellationToken c BytesClientSent = BytesServerReceived + contentLength; Logger.Debug("MediaUpload[{0}] - Sending bytes={1}-{2}", UploadUri, BytesServerReceived, BytesClientSent - 1); + bool isFinalChunk = BytesClientSent == StreamLength; + if (isFinalChunk && !Crc32cHeaderSentInInitiation) + { + string trailingCrc32c = await CalculateCrc32cAsync(ContentStream, cancellationToken).ConfigureAwait(false); + if (trailingCrc32c != null) + { + request.Headers.TryAddWithoutValidation(GoogleHashHeader, $"crc32c={trailingCrc32c}"); + } + } + else + { + // Explicitly ensure the header is NOT present for non-final chunks + // (Just in case it was added by a default modifier elsewhere) + if (request.Headers.Contains(GoogleHashHeader)) + { + request.Headers.Remove(GoogleHashHeader); + } + } // We can't assume that the server actually received all the data. It almost always does, // but just occasionally it'll return a 308 that makes us resend a chunk. We need to @@ -664,6 +701,54 @@ protected async Task SendNextChunkAsync(Stream stream, CancellationToken c return completed; } +#if NET6_0_OR_GREATER + /// + /// Calculates the CRC32C hash of the entire stream and returns it as a Base64 string. + /// + internal async Task CalculateCrc32cAsync(Stream stream, CancellationToken cancellationToken) + { + if (stream == null || !stream.CanSeek) + { + return null; + } + + long originalPosition = stream.Position; + try + { + stream.Position = 0; + + var crc32c = new System.IO.Hashing.Crc32(); + byte[] buffer = System.Buffers.ArrayPool.Shared.Rent(Crc32cCalculationBufferSize); + try + { + int bytesRead; + while ((bytesRead = await stream.ReadAsync(buffer, 0, buffer.Length, cancellationToken).ConfigureAwait(false)) > 0) + { + crc32c.Append(new ReadOnlySpan(buffer, 0, bytesRead)); + } + } + finally + { + System.Buffers.ArrayPool.Shared.Return(buffer); + } + + byte[] hashBytes = crc32c.GetCurrentHash(); + return Convert.ToBase64String(hashBytes); + } + finally + { + // Restore the stream position. Important so the upload continues correctly. + stream.Position = originalPosition; + } + } +#else + /// + /// Calculates the CRC32C hash of the entire stream and returns it as a Base64 string. + /// + internal Task CalculateCrc32cAsync(Stream stream, CancellationToken cancellationToken) => + Task.FromResult(null); +#endif + /// Handles a media upload HTTP response. /// True if the entire media has been completely uploaded. private async Task HandleResponse(HttpResponseMessage response) @@ -1063,6 +1148,10 @@ public class ResumableUpload : ResumableUpload /// The uploadType parameter value for resumable uploads. private const string Resumable = "resumable"; + /// + /// The identifier string for the CRC32C hash. + private const string Crc32c = "Crc32c"; + #endregion // Constants #region Construction @@ -1133,6 +1222,29 @@ protected ResumableUpload(IClientService service, string path, string httpMethod public override async Task InitiateSessionAsync(CancellationToken cancellationToken = default(CancellationToken)) { HttpRequestMessage request = CreateInitializeRequest(); + Crc32cHeaderSentInInitiation = false; + if (Body != null) + { + var metadataHash = GetPropertyFromGenericBody(Body, Crc32c); + if (!string.IsNullOrEmpty(metadataHash)) + { + string calculatedHash = await CalculateCrc32cAsync(ContentStream, cancellationToken).ConfigureAwait(false); + if (calculatedHash != null) + { + if (metadataHash != calculatedHash) + { + throw new InvalidOperationException( + $"The calculated CRC32C of the stream ({calculatedHash}) does not match the CRC32C provided in the object metadata ({metadataHash})."); + } + request.Headers.TryAddWithoutValidation(GoogleHashHeader, $"crc32c={calculatedHash}"); + Crc32cHeaderSentInInitiation = true; + } + else + { + ApplicationContext.Logger.ForType>().Warning("A CRC32C hash was provided in the request body, but the stream is not seekable. The hash will not be validated on the client, and will not be sent to the server."); + } + } + } Options?.ModifySessionInitiationRequest?.Invoke(request); var response = await Service.HttpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); @@ -1211,6 +1323,12 @@ private void SetAllPropertyValues(RequestBuilder requestBuilder) } } + /// + /// Retrieves the value of a specific property from a generic object using reflection. + /// + internal string GetPropertyFromGenericBody(TRequest body, string propertyName) => + body?.GetType().GetProperty(propertyName)?.GetValue(body) as string; + #endregion Upload Implementation }