diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/UploadObjectTest.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/UploadObjectTest.cs index b1cb95bb2fba..972848f0ab79 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/UploadObjectTest.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/UploadObjectTest.cs @@ -24,6 +24,7 @@ using System.Linq; using System.Net; using System.Net.Http; +using System.Text; using System.Threading; using System.Threading.Tasks; using Xunit; @@ -308,9 +309,8 @@ public void UploadObject_InvalidHash_ThrowOnly() var name = IdGenerator.FromGuid(); var bucket = _fixture.MultiVersionBucket; var options = new UploadObjectOptions { UploadValidationMode = UploadValidationMode.ThrowOnly }; - Assert.Throws(() => client.UploadObject(bucket, name, null, stream, options)); - // We don't delete the object, so it's still present. - ValidateData(bucket, name, new MemoryStream(interceptor.UploadedBytes)); + var exception = Assert.Throws(() => client.UploadObject(bucket, name, null, stream, options)); + Assert.Equal(HttpStatusCode.BadRequest, exception.HttpStatusCode); } [Fact] @@ -323,28 +323,12 @@ public void UploadObject_InvalidHash_DeleteAndThrow() var name = IdGenerator.FromGuid(); var bucket = _fixture.MultiVersionBucket; var options = new UploadObjectOptions { UploadValidationMode = UploadValidationMode.DeleteAndThrow }; - Assert.Throws(() => client.UploadObject(bucket, name, null, stream, options)); + var exception = Assert.Throws(() => client.UploadObject(bucket, name, null, stream, options)); + Assert.Equal(HttpStatusCode.BadRequest, exception.HttpStatusCode); var notFound = Assert.Throws(() => _fixture.Client.GetObject(bucket, name)); Assert.Equal(HttpStatusCode.NotFound, notFound.HttpStatusCode); } - [Fact] - public void UploadObject_InvalidHash_DeleteAndThrow_DeleteFails() - { - var client = StorageClient.Create(); - var interceptor = new BreakUploadInterceptor(); - client.Service.HttpClient.MessageHandler.AddExecuteInterceptor(interceptor); - client.Service.HttpClient.MessageHandler.AddExecuteInterceptor(new BreakDeleteInterceptor()); - var stream = GenerateData(50); - var name = IdGenerator.FromGuid(); - var bucket = _fixture.MultiVersionBucket; - var options = new UploadObjectOptions { UploadValidationMode = UploadValidationMode.DeleteAndThrow }; - var ex = Assert.Throws(() => client.UploadObject(bucket, name, null, stream, options)); - Assert.NotNull(ex.AdditionalFailures); - // The deletion failed, so the uploaded object still exists. - ValidateData(bucket, name, new MemoryStream(interceptor.UploadedBytes)); - } - [Fact] public async Task UploadObjectAsync_InvalidHash_None() { @@ -371,9 +355,8 @@ public async Task UploadObjectAsync_InvalidHash_ThrowOnly() var name = IdGenerator.FromGuid(); var bucket = _fixture.MultiVersionBucket; var options = new UploadObjectOptions { UploadValidationMode = UploadValidationMode.ThrowOnly }; - await Assert.ThrowsAsync(() => client.UploadObjectAsync(bucket, name, null, stream, options)); - // We don't delete the object, so it's still present. - ValidateData(bucket, name, new MemoryStream(interceptor.UploadedBytes)); + var exception = await Assert.ThrowsAsync(() => client.UploadObjectAsync(bucket, name, null, stream, options)); + Assert.Equal(HttpStatusCode.BadRequest, exception.HttpStatusCode); } [Fact] @@ -387,28 +370,12 @@ public async Task UploadObjectAsync_InvalidHash_DeleteAndThrow() var name = IdGenerator.FromGuid(); var bucket = _fixture.MultiVersionBucket; var options = new UploadObjectOptions { UploadValidationMode = UploadValidationMode.DeleteAndThrow }; - await Assert.ThrowsAsync(() => client.UploadObjectAsync(bucket, name, null, stream, options)); + var exception = await Assert.ThrowsAsync(() => client.UploadObjectAsync(bucket, name, null, stream, options)); + Assert.Equal(HttpStatusCode.BadRequest, exception.HttpStatusCode); var notFound = await Assert.ThrowsAsync(() => _fixture.Client.GetObjectAsync(bucket, name)); Assert.Equal(HttpStatusCode.NotFound, notFound.HttpStatusCode); } - [Fact] - public async Task UploadObjectAsync_InvalidHash_DeleteAndThrow_DeleteFails() - { - var client = StorageClient.Create(); - var interceptor = new BreakUploadInterceptor(); - client.Service.HttpClient.MessageHandler.AddExecuteInterceptor(interceptor); - client.Service.HttpClient.MessageHandler.AddExecuteInterceptor(new BreakDeleteInterceptor()); - var stream = GenerateData(50); - var name = IdGenerator.FromGuid(); - var bucket = _fixture.MultiVersionBucket; - var options = new UploadObjectOptions { UploadValidationMode = UploadValidationMode.DeleteAndThrow }; - var ex = await Assert.ThrowsAsync(() => client.UploadObjectAsync(bucket, name, null, stream, options)); - Assert.NotNull(ex.AdditionalFailures); - // The deletion failed, so the uploaded object still exists. - ValidateData(bucket, name, new MemoryStream(interceptor.UploadedBytes)); - } - [Fact] public async Task InitiateUploadSessionAsync_NegativeLength() { @@ -488,19 +455,50 @@ public async Task InterceptAsync(HttpRequestMessage request, CancellationToken c } } - private class BreakDeleteInterceptor : IHttpExecuteInterceptor + [Fact] + public void HashingStream_ShouldHandleRetries_WhenRestartedFromBeginning() { - public Task InterceptAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - // We only care about Delete requests - if (request.Method == HttpMethod.Delete) - { - // Ugly but effective hack: replace the generation URL parameter so that we add a leading 9, - // so the generation we try to delete is the wrong one. - request.RequestUri = new Uri(request.RequestUri.ToString().Replace("generation=", "generation=9")); - } - return Task.FromResult(0); - } + var data = Encoding.UTF8.GetBytes("The quick brown fox jumps over the lazy dog"); + var baseStream = new MemoryStream(data); + var hashingStream = new CustomMediaUpload.HashingStream(baseStream); + var buffer = new byte[data.Length]; + + hashingStream.Read(buffer, 0, 10); + var hashAfterPartial = hashingStream.GetBase64Hash(); + + // Simulate the Retry logic: Seek back to the beginning + hashingStream.Position = 0; + + hashingStream.Read(buffer, 0, data.Length); + var finalHash = hashingStream.GetBase64Hash(); + + var expectedHasher = new Crc32c(); + expectedHasher.UpdateHash(data, 0, data.Length); + var expectedHash = Convert.ToBase64String(expectedHasher.GetHash()); + Assert.Equal(expectedHash, finalHash); + } + + [Fact] + public void HashingStream_ShouldHandleRetries_WhenSeekingBackwardsToIntermediatePoint() + { + var data = Encoding.UTF8.GetBytes("The quick brown fox jumps over the lazy dog"); + var baseStream = new MemoryStream(data); + var hashingStream = new CustomMediaUpload.HashingStream(baseStream); + var buffer = new byte[data.Length]; + + hashingStream.Read(buffer, 0, 10); + var hashAfterPartial = hashingStream.GetBase64Hash(); + + // Simulate the Retry logic: Seek back to the intermediate point. + hashingStream.Position = 5; + + hashingStream.Read(buffer, 0, data.Length); + var finalHash = hashingStream.GetBase64Hash(); + + var expectedHasher = new Crc32c(); + expectedHasher.UpdateHash(data, 0, data.Length); + var expectedHash = Convert.ToBase64String(expectedHasher.GetHash()); + Assert.Equal(expectedHash, finalHash); } private Object GetExistingObject() diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.Tests/UploadObjectOptionsTest.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.Tests/UploadObjectOptionsTest.cs index d0ef1b0d9f93..1838a0aebca1 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.Tests/UploadObjectOptionsTest.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.Tests/UploadObjectOptionsTest.cs @@ -54,7 +54,7 @@ public void InvalidChunkSize(int chunkSize) [Fact] public void ModifyMediaUpload_DefaultOptions() { - var upload = new CustomMediaUpload(new DummyService(), null, "bucket", new MemoryStream(), null); + var upload = new CustomMediaUpload(new DummyService(), null, "bucket", new MemoryStream(), null, null); var options = new UploadObjectOptions(); options.ModifyMediaUpload(upload); Assert.Equal(ResumableUpload.DefaultChunkSize, upload.ChunkSize); @@ -71,7 +71,7 @@ public void ModifyMediaUpload_DefaultOptions() [Fact] public void ModifyMediaUpload_AllOptions_PositiveMatch() { - var upload = new CustomMediaUpload(new DummyService(), null, "bucket", new MemoryStream(), null); + var upload = new CustomMediaUpload(new DummyService(), null, "bucket", new MemoryStream(), null, null); var options = new UploadObjectOptions { ChunkSize = UploadObjectOptions.MinimumChunkSize * 3, @@ -96,7 +96,7 @@ public void ModifyMediaUpload_AllOptions_PositiveMatch() [Fact] public void ModifyMediaUpload_AllOptions_NegativeMatch() { - var upload = new CustomMediaUpload(new DummyService(), null, "bucket", new MemoryStream(), null); + var upload = new CustomMediaUpload(new DummyService(), null, "bucket", new MemoryStream(), null, null); var options = new UploadObjectOptions { ChunkSize = UploadObjectOptions.MinimumChunkSize * 3, @@ -117,7 +117,7 @@ public void ModifyMediaUpload_AllOptions_NegativeMatch() [Fact] public void ModifyMediaUpload_MatchNotMatchConflicts() { - var upload = new CustomMediaUpload(new DummyService(), null, "bucket", new MemoryStream(), null); + var upload = new CustomMediaUpload(new DummyService(), null, "bucket", new MemoryStream(), null, null); Assert.Throws(() => { var options = new UploadObjectOptions { IfGenerationMatch = 1L, IfGenerationNotMatch = 2L }; diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.Tests/UploadValidationExceptionTest.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.Tests/UploadValidationExceptionTest.cs index a3af1d16a424..32b7d52cab3e 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.Tests/UploadValidationExceptionTest.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.Tests/UploadValidationExceptionTest.cs @@ -1,4 +1,4 @@ -// Copyright 2017 Google Inc. All Rights Reserved. +// Copyright 2017 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ namespace Google.Cloud.Storage.V1.Tests public class UploadValidationExceptionTest { [Fact] + [Obsolete] public void Construction_NoAdditionalFailure() { var ex = new UploadValidationException("hash", new Object(), null); @@ -28,6 +29,7 @@ public void Construction_NoAdditionalFailure() } [Fact] + [Obsolete] public void Construction_WithAdditionalFailure() { var additional = new Exception(); @@ -36,6 +38,7 @@ public void Construction_WithAdditionalFailure() } [Fact] + [Obsolete] public void Construction_WithAdditionalFailure_Empty() { Assert.Throws(() => new UploadValidationException("hash", new Object(), new AggregateException("No inner exceptions"))); diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/CustomMediaUpload.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/CustomMediaUpload.cs index c91a50ef9265..0ede13e023a8 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/CustomMediaUpload.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/CustomMediaUpload.cs @@ -1,4 +1,4 @@ -// Copyright 2017 Google Inc. All Rights Reserved. +// Copyright 2017 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,12 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. +using Google.Apis.Http; using Google.Apis.Services; +using Google.Apis.Upload; using System; using System.IO; +using System.Linq; using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; using static Google.Apis.Storage.v1.ObjectsResource; -using Google.Apis.Upload; namespace Google.Cloud.Storage.V1 { @@ -26,12 +30,185 @@ namespace Google.Cloud.Storage.V1 /// internal sealed class CustomMediaUpload : InsertMediaUpload { + private readonly Crc32cHashInterceptor _interceptor; + private readonly IClientService _service; + private readonly HashingStream _hashingStream; + public CustomMediaUpload(IClientService service, Apis.Storage.v1.Data.Object body, string bucket, - Stream stream, string contentType) - : base(service, body, bucket, stream, contentType) + Stream stream, string contentType, UploadObjectOptions options) + : base(service, body, bucket, options?.UploadValidationMode != UploadValidationMode.None ? new HashingStream(stream) : stream, contentType) { + _service = service; + var validationMode = options?.UploadValidationMode ?? UploadObjectOptions.DefaultValidationMode; + if (validationMode != UploadValidationMode.None) + { + _hashingStream = ContentStream as HashingStream; + _interceptor = new Crc32cHashInterceptor(this, _hashingStream, _service); + _service?.HttpClient?.MessageHandler?.AddExecuteInterceptor(_interceptor); + } } internal new ResumableUploadOptions Options => base.Options; + + private sealed class Crc32cHashInterceptor : IHttpExecuteInterceptor + { + private const string GoogleHashHeader = "x-goog-hash"; + private readonly IClientService _service; + private readonly CustomMediaUpload _mediaUpload; + private Uri _uploadUri; + private readonly HashingStream _hashingStream; + + public Crc32cHashInterceptor(CustomMediaUpload mediaUpload, HashingStream hashingStream, IClientService service) + { + _hashingStream = hashingStream; + _service = service; + _mediaUpload = mediaUpload; + _mediaUpload.UploadSessionData += OnSessionData; + _mediaUpload.ProgressChanged += OnProgressChanged; + } + + public Task InterceptAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + if (_uploadUri == null || !_uploadUri.Equals(request.RequestUri)) + { + return Task.CompletedTask; + } + + if (request.Method == System.Net.Http.HttpMethod.Put && request.Content?.Headers.Contains("Content-Range") is true) + { + var rangeHeader = request.Content.Headers.GetValues("Content-Range").First(); + + if (IsFinalChunk(rangeHeader)) + { + var calculatedHash = _hashingStream.GetBase64Hash(); + request.Headers.TryAddWithoutValidation(GoogleHashHeader, $"crc32c={calculatedHash}"); + } + } + return Task.CompletedTask; + } + + private void OnSessionData(IUploadSessionData data) + { + _uploadUri = data.UploadUri; + _mediaUpload.UploadSessionData -= OnSessionData; + } + + private void OnProgressChanged(IUploadProgress progress) + { + if (progress.Status is UploadStatus.Completed or UploadStatus.Failed) + { + // Clean up when upload is finished. + _service?.HttpClient?.MessageHandler?.RemoveExecuteInterceptor(this); + _mediaUpload.ProgressChanged -= OnProgressChanged; + } + } + + private bool IsFinalChunk(string rangeHeader) + { + // Expected format: "bytes {start}-{end}/{total}" or "bytes */{total}" for the final request. + // We are interested in the final chunk of a known-size upload. + const string prefix = "bytes "; + if (!rangeHeader.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + ReadOnlySpan span = rangeHeader.AsSpan(prefix.Length); + int slashIndex = span.IndexOf('/'); + if (slashIndex == -1) + { + return false; + } + + var totalSpan = span.Slice(slashIndex + 1); + if (totalSpan.IsEmpty || totalSpan[0] == '*') + { + return false; + } + + if (!long.TryParse(totalSpan.ToString(), System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out long totalSize)) + { + return false; + } + + var rangeSpan = span.Slice(0, slashIndex); + int dashIndex = rangeSpan.IndexOf('-'); + if (dashIndex == -1) + { + return false; + } + + var endByteSpan = rangeSpan.Slice(dashIndex + 1); + if (!long.TryParse(endByteSpan.ToString(), System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out long endByte)) + { + return false; + } + + // If endByte is the last byte of the file, it's the final chunk. + return (endByte + 1) == totalSize; + } + } + + internal sealed class HashingStream : Stream + { + private readonly Stream _stream; + private readonly Crc32c _hasher; + private long _maxPositionHashed = 0; + + public HashingStream(Stream stream) + { + _stream = stream; + _hasher = new Crc32c(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + long startingPos = _stream.Position; + int bytesRead = _stream.Read(buffer, offset, count); + ProcessBytes(buffer, offset, bytesRead, startingPos); + return bytesRead; + } + + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + long startingPos = _stream.Position; + int bytesRead = await _stream.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + ProcessBytes(buffer, offset, bytesRead, startingPos); + return bytesRead; + } + + private void ProcessBytes(byte[] buffer, int offset, int bytesRead, long startingPos) + { + if (bytesRead <= 0) return; + + // Only hash bytes that are beyond the furthest point we've already hashed. + // This handles the rewind and re-read scenario during retries. + if (startingPos + bytesRead > _maxPositionHashed) + { + long newBytesStart = Math.Max(startingPos, _maxPositionHashed); + int actuallyNewCount = (int) ((startingPos + bytesRead) - newBytesStart); + int bufferOffset = offset + (int) (newBytesStart - startingPos); + + _hasher.UpdateHash(buffer, bufferOffset, actuallyNewCount); + _maxPositionHashed = startingPos + bytesRead; + } + } + + public override long Position + { + get => _stream.Position; + set => _stream.Position = value; + } + + public override long Seek(long offset, SeekOrigin origin) => _stream.Seek(offset, origin); + public string GetBase64Hash() => Convert.ToBase64String(_hasher.GetHash()); + public override bool CanRead => _stream.CanRead; + public override bool CanSeek => _stream.CanSeek; + public override bool CanWrite => _stream.CanWrite; + public override long Length => _stream.Length; + public override void Flush() => _stream.Flush(); + public override void SetLength(long value) => _stream.SetLength(value); + public override void Write(byte[] buffer, int offset, int count) => _stream.Write(buffer, offset, count); + } } } diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.UploadObject.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.UploadObject.cs index 84e7f457edb6..ada419658d39 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.UploadObject.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.UploadObject.cs @@ -48,7 +48,7 @@ public override ObjectsResource.InsertMediaUpload CreateObjectUploader( { ValidateObject(destination, nameof(destination)); GaxPreconditions.CheckNotNull(source, nameof(source)); - var mediaUpload = new CustomMediaUpload(Service, destination, destination.Bucket, source, destination.ContentType); + var mediaUpload = new CustomMediaUpload(Service, destination, destination.Bucket, source, destination.ContentType, options); options?.ModifyMediaUpload(mediaUpload); ApplyEncryptionKey(options?.EncryptionKey, options?.KmsKeyName, mediaUpload); return mediaUpload; @@ -195,7 +195,6 @@ internal Object Execute() { additionalFailures = new AggregateException(e); } - throw new UploadValidationException(hash, result, additionalFailures); } return result; } @@ -220,7 +219,6 @@ internal async Task ExecuteAsync(CancellationToken cancellationToken) { additionalFailures = new AggregateException(e); } - throw new UploadValidationException(hash, result, additionalFailures); } return result; } diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/UploadValidationException.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/UploadValidationException.cs index 4f7a9d674b81..61f04bb8cd35 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/UploadValidationException.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/UploadValidationException.cs @@ -1,4 +1,4 @@ -// Copyright 2017 Google Inc. All Rights Reserved. +// Copyright 2017 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -22,6 +22,8 @@ namespace Google.Cloud.Storage.V1 /// /// Exception thrown when an upload failed validation. /// + [Obsolete("This exception is no longer thrown. Server-side validation is now performed, " + + "resulting in a GoogleApiException with a 400 (Bad Request) status code on failure.")] public sealed class UploadValidationException : IOException { ///