feat(Storage): Enable full object checksum validation for resumable uploads

apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/UploadObjectTest.cs

@@ -24,6 +24,7 @@
 using System.Linq;
 using System.Net;
 using System.Net.Http;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Xunit;
@@ -308,9 +309,8 @@ public void UploadObject_InvalidHash_ThrowOnly()
             var name = IdGenerator.FromGuid();
             var bucket = _fixture.MultiVersionBucket;
             var options = new UploadObjectOptions { UploadValidationMode = UploadValidationMode.ThrowOnly };
-            Assert.Throws<UploadValidationException>(() => client.UploadObject(bucket, name, null, stream, options));
-            // We don't delete the object, so it's still present.
-            ValidateData(bucket, name, new MemoryStream(interceptor.UploadedBytes));
+            var exception = Assert.Throws<GoogleApiException>(() => client.UploadObject(bucket, name, null, stream, options));
+            Assert.Equal(HttpStatusCode.BadRequest, exception.HttpStatusCode);
         }
 
         [Fact]
@@ -323,28 +323,12 @@ public void UploadObject_InvalidHash_DeleteAndThrow()
             var name = IdGenerator.FromGuid();
             var bucket = _fixture.MultiVersionBucket;
             var options = new UploadObjectOptions { UploadValidationMode = UploadValidationMode.DeleteAndThrow };
-            Assert.Throws<UploadValidationException>(() => client.UploadObject(bucket, name, null, stream, options));
+            var exception = Assert.Throws<GoogleApiException>(() => client.UploadObject(bucket, name, null, stream, options));
+            Assert.Equal(HttpStatusCode.BadRequest, exception.HttpStatusCode);
             var notFound = Assert.Throws<GoogleApiException>(() => _fixture.Client.GetObject(bucket, name));
             Assert.Equal(HttpStatusCode.NotFound, notFound.HttpStatusCode);
         }
 
-        [Fact]
-        public void UploadObject_InvalidHash_DeleteAndThrow_DeleteFails()
-        {
-            var client = StorageClient.Create();
-            var interceptor = new BreakUploadInterceptor();
-            client.Service.HttpClient.MessageHandler.AddExecuteInterceptor(interceptor);
-            client.Service.HttpClient.MessageHandler.AddExecuteInterceptor(new BreakDeleteInterceptor());
-            var stream = GenerateData(50);
-            var name = IdGenerator.FromGuid();
-            var bucket = _fixture.MultiVersionBucket;
-            var options = new UploadObjectOptions { UploadValidationMode = UploadValidationMode.DeleteAndThrow };
-            var ex = Assert.Throws<UploadValidationException>(() => client.UploadObject(bucket, name, null, stream, options));
-            Assert.NotNull(ex.AdditionalFailures);
-            // The deletion failed, so the uploaded object still exists.
-            ValidateData(bucket, name, new MemoryStream(interceptor.UploadedBytes));
-        }
-
         [Fact]
         public async Task UploadObjectAsync_InvalidHash_None()
         {
@@ -371,9 +355,8 @@ public async Task UploadObjectAsync_InvalidHash_ThrowOnly()
             var name = IdGenerator.FromGuid();
             var bucket = _fixture.MultiVersionBucket;
             var options = new UploadObjectOptions { UploadValidationMode = UploadValidationMode.ThrowOnly };
-            await Assert.ThrowsAsync<UploadValidationException>(() => client.UploadObjectAsync(bucket, name, null, stream, options));
-            // We don't delete the object, so it's still present.
-            ValidateData(bucket, name, new MemoryStream(interceptor.UploadedBytes));
+            var exception = await Assert.ThrowsAsync<GoogleApiException>(() => client.UploadObjectAsync(bucket, name, null, stream, options));
+            Assert.Equal(HttpStatusCode.BadRequest, exception.HttpStatusCode);
         }
 
         [Fact]
@@ -387,28 +370,12 @@ public async Task UploadObjectAsync_InvalidHash_DeleteAndThrow()
             var name = IdGenerator.FromGuid();
             var bucket = _fixture.MultiVersionBucket;
             var options = new UploadObjectOptions { UploadValidationMode = UploadValidationMode.DeleteAndThrow };
-            await Assert.ThrowsAsync<UploadValidationException>(() => client.UploadObjectAsync(bucket, name, null, stream, options));
+            var exception = await Assert.ThrowsAsync<GoogleApiException>(() => client.UploadObjectAsync(bucket, name, null, stream, options));
+            Assert.Equal(HttpStatusCode.BadRequest, exception.HttpStatusCode);
             var notFound = await Assert.ThrowsAsync<GoogleApiException>(() => _fixture.Client.GetObjectAsync(bucket, name));
             Assert.Equal(HttpStatusCode.NotFound, notFound.HttpStatusCode);
         }
 
-        [Fact]
-        public async Task UploadObjectAsync_InvalidHash_DeleteAndThrow_DeleteFails()
-        {
-            var client = StorageClient.Create();
-            var interceptor = new BreakUploadInterceptor();
-            client.Service.HttpClient.MessageHandler.AddExecuteInterceptor(interceptor);
-            client.Service.HttpClient.MessageHandler.AddExecuteInterceptor(new BreakDeleteInterceptor());
-            var stream = GenerateData(50);
-            var name = IdGenerator.FromGuid();
-            var bucket = _fixture.MultiVersionBucket;
-            var options = new UploadObjectOptions { UploadValidationMode = UploadValidationMode.DeleteAndThrow };
-            var ex = await Assert.ThrowsAsync<UploadValidationException>(() => client.UploadObjectAsync(bucket, name, null, stream, options));
-            Assert.NotNull(ex.AdditionalFailures);
-            // The deletion failed, so the uploaded object still exists.
-            ValidateData(bucket, name, new MemoryStream(interceptor.UploadedBytes));
-        }
-
         [Fact]
         public async Task InitiateUploadSessionAsync_NegativeLength()
         {
@@ -488,19 +455,50 @@ public async Task InterceptAsync(HttpRequestMessage request, CancellationToken c
             }
         }
 
-        private class BreakDeleteInterceptor : IHttpExecuteInterceptor
+        [Fact]
+        public void HashingStream_ShouldHandleRetries_WhenRestartedFromBeginning()
         {
-            public Task InterceptAsync(HttpRequestMessage request, CancellationToken cancellationToken)
-            {
-                // We only care about Delete requests
-                if (request.Method == HttpMethod.Delete)
-                {
-                    // Ugly but effective hack: replace the generation URL parameter so that we add a leading 9,
-                    // so the generation we try to delete is the wrong one.
-                    request.RequestUri = new Uri(request.RequestUri.ToString().Replace("generation=", "generation=9"));
-                }
-                return Task.FromResult(0);
-            }
+            var data = Encoding.UTF8.GetBytes("The quick brown fox jumps over the lazy dog");
+            var baseStream = new MemoryStream(data);
+            var hashingStream = new CustomMediaUpload.HashingStream(baseStream);
+            var buffer = new byte[data.Length];
+
+            hashingStream.Read(buffer, 0, 10);
+            var hashAfterPartial = hashingStream.GetBase64Hash();
+
+            // Simulate the Retry logic: Seek back to the beginning
+            hashingStream.Position = 0;
+
+            hashingStream.Read(buffer, 0, data.Length);
+            var finalHash = hashingStream.GetBase64Hash();
+
+            var expectedHasher = new Crc32c();
+            expectedHasher.UpdateHash(data, 0, data.Length);
+            var expectedHash = Convert.ToBase64String(expectedHasher.GetHash());
+            Assert.Equal(expectedHash, finalHash);
+        }
+
+        [Fact]
+        public void HashingStream_ShouldHandleRetries_WhenSeekingBackwardsToIntermediatePoint()
+        {
+            var data = Encoding.UTF8.GetBytes("The quick brown fox jumps over the lazy dog");
+            var baseStream = new MemoryStream(data);
+            var hashingStream = new CustomMediaUpload.HashingStream(baseStream);
+            var buffer = new byte[data.Length];
+
+            hashingStream.Read(buffer, 0, 10);
+            var hashAfterPartial = hashingStream.GetBase64Hash();
+
+            // Simulate the Retry logic: Seek back to the intermediate point.
+            hashingStream.Position = 5;
+
+            hashingStream.Read(buffer, 0, data.Length);
+            var finalHash = hashingStream.GetBase64Hash();
+
+            var expectedHasher = new Crc32c();
+            expectedHasher.UpdateHash(data, 0, data.Length);
+            var expectedHash = Convert.ToBase64String(expectedHasher.GetHash());
+            Assert.Equal(expectedHash, finalHash);
         }
 
         private Object GetExistingObject()

apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.Tests/UploadObjectOptionsTest.cs

@@ -54,7 +54,7 @@ public void InvalidChunkSize(int chunkSize)
         [Fact]
         public void ModifyMediaUpload_DefaultOptions()
         {
-            var upload = new CustomMediaUpload(new DummyService(), null, "bucket", new MemoryStream(), null);
+            var upload = new CustomMediaUpload(new DummyService(), null, "bucket", new MemoryStream(), null, null);
             var options = new UploadObjectOptions();
             options.ModifyMediaUpload(upload);
             Assert.Equal(ResumableUpload<InsertMediaUpload>.DefaultChunkSize, upload.ChunkSize);
@@ -71,7 +71,7 @@ public void ModifyMediaUpload_DefaultOptions()
         [Fact]
         public void ModifyMediaUpload_AllOptions_PositiveMatch()
         {
-            var upload = new CustomMediaUpload(new DummyService(), null, "bucket", new MemoryStream(), null);
+            var upload = new CustomMediaUpload(new DummyService(), null, "bucket", new MemoryStream(), null, null);
             var options = new UploadObjectOptions
             {
                 ChunkSize = UploadObjectOptions.MinimumChunkSize * 3,
@@ -96,7 +96,7 @@ public void ModifyMediaUpload_AllOptions_PositiveMatch()
         [Fact]
         public void ModifyMediaUpload_AllOptions_NegativeMatch()
         {
-            var upload = new CustomMediaUpload(new DummyService(), null, "bucket", new MemoryStream(), null);
+            var upload = new CustomMediaUpload(new DummyService(), null, "bucket", new MemoryStream(), null, null);
             var options = new UploadObjectOptions
             {
                 ChunkSize = UploadObjectOptions.MinimumChunkSize * 3,
@@ -117,7 +117,7 @@ public void ModifyMediaUpload_AllOptions_NegativeMatch()
         [Fact]
         public void ModifyMediaUpload_MatchNotMatchConflicts()
         {
-            var upload = new CustomMediaUpload(new DummyService(), null, "bucket", new MemoryStream(), null);
+            var upload = new CustomMediaUpload(new DummyService(), null, "bucket", new MemoryStream(), null, null);
             Assert.Throws<ArgumentException>(() =>
             {
                 var options = new UploadObjectOptions { IfGenerationMatch = 1L, IfGenerationNotMatch = 2L };

apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.Tests/UploadValidationExceptionTest.cs

@@ -1,4 +1,4 @@
-// Copyright 2017 Google Inc. All Rights Reserved.
+// Copyright 2017 Google Inc. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -21,13 +21,15 @@ namespace Google.Cloud.Storage.V1.Tests
     public class UploadValidationExceptionTest
     {
         [Fact]
+        [Obsolete]
         public void Construction_NoAdditionalFailure()
         {
             var ex = new UploadValidationException("hash", new Object(), null);
             Assert.Null(ex.AdditionalFailures);
         }
 
         [Fact]
+        [Obsolete]
         public void Construction_WithAdditionalFailure()
         {
             var additional = new Exception();
@@ -36,6 +38,7 @@ public void Construction_WithAdditionalFailure()
         }
 
         [Fact]
+        [Obsolete]
         public void Construction_WithAdditionalFailure_Empty()
         {
             Assert.Throws<ArgumentException>(() => new UploadValidationException("hash", new Object(), new AggregateException("No inner exceptions")));