From c6802fd46295c4215048e108118fdd542a81eb1b Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Mon, 20 Oct 2025 02:36:22 -0700 Subject: [PATCH 01/12] fix: implement method in storage clien t to prevent directry traversal attack --- .../StorageClientImpl.cs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs index 43fe62836c84..589681c18ae9 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs @@ -19,6 +19,7 @@ using Google.Apis.Storage.v1; using Google.Apis.Storage.v1.Data; using System; +using System.IO; using System.Net.Http; using System.Text.RegularExpressions; using Object = Google.Apis.Storage.v1.Data.Object; @@ -119,6 +120,24 @@ internal static string ValidateBucketName(string bucket) return bucket; } + /// + /// Validates object download path to the base directory only. + /// This method ensures the object is downloaded securely and prevent directory traversal attack. + /// + private void ValidateObjectDownloadPath(Stream stream) + { + GaxPreconditions.CheckNotNull(stream, nameof(stream)); + string baseDir = Path.GetFullPath(AppDomain.CurrentDomain.BaseDirectory); + if (stream is FileStream fileStream) + { + string fullPath = Path.GetFullPath(fileStream.Name); + if (!fullPath.StartsWith(baseDir, StringComparison.OrdinalIgnoreCase)) + { + throw new ArgumentException("Path traversal is not allowed. File path is outside the designated directory"); + } + } + } + /// /// Validates that the given Bucket has a "somewhat valid" (no URI encoding required) bucket name. /// From 75e4a408b5585aef54cf8d0bc6df12e866454f38 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Mon, 20 Oct 2025 03:22:56 -0700 Subject: [PATCH 02/12] fix(storage): add test to prevent directory traversal attack --- .../DownloadObjectTest.cs | 14 ++++++++++++++ .../StorageClientImpl.DownloadObject.cs | 4 ++++ .../Google.Cloud.Storage.V1/StorageClientImpl.cs | 6 +++--- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/DownloadObjectTest.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/DownloadObjectTest.cs index 211cb903e2df..276959174d66 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/DownloadObjectTest.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/DownloadObjectTest.cs @@ -137,6 +137,20 @@ public void DownloadObjectFromInvalidBucket() Assert.Throws(() => _fixture.Client.DownloadObject("!!!", _fixture.LargeObject, new MemoryStream())); } + [Fact] + public void DownloadObjectWithRelativePathTraversal() + { + using var outputFile = File.OpenWrite("../DownloadTest.txt"); + Assert.Throws(() => _fixture.Client.DownloadObject(_fixture.ReadBucket, _fixture.SmallObject, outputFile)); + } + + [Fact] + public void DownloadObjectWithAbsolutePathTraversal() + { + using var outputFile = File.OpenWrite("/DownloadTest.txt"); + Assert.Throws(() => _fixture.Client.DownloadObject(_fixture.ReadBucket, _fixture.SmallObject, outputFile)); + } + [Fact] public void DownloadObjectWrongGeneration() { diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.DownloadObject.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.DownloadObject.cs index c70bcff309c2..9e8ec87c43c8 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.DownloadObject.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.DownloadObject.cs @@ -35,6 +35,7 @@ public override Object DownloadObject( DownloadObjectOptions options = null, IProgress progress = null) { + ValidateObjectDownloadPath(destination); var builder = CreateRequestBuilder(bucket, objectName); var metadata = new Object { Bucket = bucket, Name = objectName }; return DownloadObjectImpl(metadata, builder, destination, options, progress); @@ -49,6 +50,7 @@ public override Task DownloadObjectAsync( CancellationToken cancellationToken = default, IProgress progress = null) { + ValidateObjectDownloadPath(destination); var builder = CreateRequestBuilder(bucket, objectName); var metadata = new Object { Bucket = bucket, Name = objectName }; return DownloadObjectAsyncImpl(metadata, builder, destination, options, cancellationToken, progress); @@ -61,6 +63,7 @@ public override Object DownloadObject( DownloadObjectOptions options = null, IProgress progress = null) { + ValidateObjectDownloadPath(destination); var builder = CreateRequestBuilder(source); var metadata = new Object { Bucket = source.Bucket, Name = source.Name }; return DownloadObjectImpl(metadata, builder, destination, options, progress); @@ -74,6 +77,7 @@ public override Task DownloadObjectAsync( CancellationToken cancellationToken = default, IProgress progress = null) { + ValidateObjectDownloadPath(destination); var builder = CreateRequestBuilder(source); var metadata = new Object { Bucket = source.Bucket, Name = source.Name }; return DownloadObjectAsyncImpl(metadata, builder, destination, options, cancellationToken, progress); diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs index 589681c18ae9..dd394b0c3a17 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs @@ -122,9 +122,9 @@ internal static string ValidateBucketName(string bucket) /// /// Validates object download path to the base directory only. - /// This method ensures the object is downloaded securely and prevent directory traversal attack. + /// This method ensures that the object is downloaded securely and prevent directory absolute and relative traversal attack. /// - private void ValidateObjectDownloadPath(Stream stream) + internal void ValidateObjectDownloadPath(Stream stream) { GaxPreconditions.CheckNotNull(stream, nameof(stream)); string baseDir = Path.GetFullPath(AppDomain.CurrentDomain.BaseDirectory); @@ -133,7 +133,7 @@ private void ValidateObjectDownloadPath(Stream stream) string fullPath = Path.GetFullPath(fileStream.Name); if (!fullPath.StartsWith(baseDir, StringComparison.OrdinalIgnoreCase)) { - throw new ArgumentException("Path traversal is not allowed. File path is outside the designated directory"); + throw new ArgumentException("Path traversal is not allowed. File path provided is outside the base directory"); } } } From d007c1e1c16198b50636aa3fb14ead30cc00b81e Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Mon, 20 Oct 2025 11:09:11 +0000 Subject: [PATCH 03/12] fix: modify test to prevent directory traversal attack --- .../DownloadObjectTest.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/DownloadObjectTest.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/DownloadObjectTest.cs index 276959174d66..ff3499dc6ad4 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/DownloadObjectTest.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/DownloadObjectTest.cs @@ -147,8 +147,16 @@ public void DownloadObjectWithRelativePathTraversal() [Fact] public void DownloadObjectWithAbsolutePathTraversal() { - using var outputFile = File.OpenWrite("/DownloadTest.txt"); - Assert.Throws(() => _fixture.Client.DownloadObject(_fixture.ReadBucket, _fixture.SmallObject, outputFile)); + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + using var outputFile = File.OpenWrite("D:\\DownloadTest.txt"); + Assert.Throws(() => _fixture.Client.DownloadObject(_fixture.ReadBucket, _fixture.SmallObject, outputFile)); + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + using var outputFileTwo = File.OpenWrite("/usr/local/DownloadTest.txt"); + Assert.Throws(() => _fixture.Client.DownloadObject(_fixture.ReadBucket, _fixture.SmallObject, outputFileTwo)); + } } [Fact] From 232afb9cb3319f03c19a97229cc24c84a8623575 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Mon, 20 Oct 2025 11:40:27 +0000 Subject: [PATCH 04/12] fix: modify method description --- .../Google.Cloud.Storage.V1/StorageClientImpl.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs index dd394b0c3a17..80205d44b224 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs @@ -122,7 +122,7 @@ internal static string ValidateBucketName(string bucket) /// /// Validates object download path to the base directory only. - /// This method ensures that the object is downloaded securely and prevent directory absolute and relative traversal attack. + /// This method ensures that the object is downloaded securely and prevent directory traversal attack using absolute or relative path. /// internal void ValidateObjectDownloadPath(Stream stream) { From 888072394337cdb08e1a9a0e2a60576fcf1d8c52 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Mon, 20 Oct 2025 11:54:37 +0000 Subject: [PATCH 05/12] fix: modify test --- .../DownloadObjectTest.cs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/DownloadObjectTest.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/DownloadObjectTest.cs index ff3499dc6ad4..bd5745f62dd7 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/DownloadObjectTest.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/DownloadObjectTest.cs @@ -21,6 +21,7 @@ using System.Linq; using System.Net; using System.Net.Http.Headers; +using System.Runtime.InteropServices; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -149,13 +150,21 @@ public void DownloadObjectWithAbsolutePathTraversal() { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { - using var outputFile = File.OpenWrite("D:\\DownloadTest.txt"); - Assert.Throws(() => _fixture.Client.DownloadObject(_fixture.ReadBucket, _fixture.SmallObject, outputFile)); + string winDirPath = @"D:\"; + if (Directory.Exists(winDirPath)) + { + using var outputFile = File.OpenWrite($"{winDirPath}DownloadTest.txt"); + Assert.Throws(() => _fixture.Client.DownloadObject(_fixture.ReadBucket, _fixture.SmallObject, outputFile)); + } } else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { - using var outputFileTwo = File.OpenWrite("/usr/local/DownloadTest.txt"); - Assert.Throws(() => _fixture.Client.DownloadObject(_fixture.ReadBucket, _fixture.SmallObject, outputFileTwo)); + string unixDirPath = "/home/test/"; + if (Directory.Exists(unixDirPath)) + { + using var outputFile = File.OpenWrite($"{unixDirPath}DownloadTest.txt"); + Assert.Throws(() => _fixture.Client.DownloadObject(_fixture.ReadBucket, _fixture.SmallObject, outputFile)); + } } } From e1d18725dff702b90746f34e47c34e9e7d3ebed7 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 22 Oct 2025 04:03:29 -0700 Subject: [PATCH 06/12] fix(storage): add check to path to prevent direcory traversal --- .../Google.Cloud.Storage.V1/StorageClientImpl.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs index 80205d44b224..1f93777d19fa 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs @@ -128,6 +128,12 @@ internal void ValidateObjectDownloadPath(Stream stream) { GaxPreconditions.CheckNotNull(stream, nameof(stream)); string baseDir = Path.GetFullPath(AppDomain.CurrentDomain.BaseDirectory); + + // Normalize path with a trailing separator to prevent path traversal attack. + if (!baseDir.EndsWith(Path.DirectorySeparatorChar.ToString())) + { + baseDir += Path.DirectorySeparatorChar; + } if (stream is FileStream fileStream) { string fullPath = Path.GetFullPath(fileStream.Name); From 091e031fccb14c7634940d4aab4f6cb13722f0a8 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Fri, 24 Oct 2025 01:35:59 -0700 Subject: [PATCH 07/12] fix(storage): modify tests for directory traversal --- .../DownloadObjectTest.cs | 26 +++++-------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/DownloadObjectTest.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/DownloadObjectTest.cs index bd5745f62dd7..073b2165240c 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/DownloadObjectTest.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/DownloadObjectTest.cs @@ -141,31 +141,19 @@ public void DownloadObjectFromInvalidBucket() [Fact] public void DownloadObjectWithRelativePathTraversal() { - using var outputFile = File.OpenWrite("../DownloadTest.txt"); + var testFile = IdGenerator.FromGuid(prefix: "test"); + using var outputFile = File.OpenWrite($"../{testFile}"); Assert.Throws(() => _fixture.Client.DownloadObject(_fixture.ReadBucket, _fixture.SmallObject, outputFile)); } [Fact] public void DownloadObjectWithAbsolutePathTraversal() { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - string winDirPath = @"D:\"; - if (Directory.Exists(winDirPath)) - { - using var outputFile = File.OpenWrite($"{winDirPath}DownloadTest.txt"); - Assert.Throws(() => _fixture.Client.DownloadObject(_fixture.ReadBucket, _fixture.SmallObject, outputFile)); - } - } - else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) - { - string unixDirPath = "/home/test/"; - if (Directory.Exists(unixDirPath)) - { - using var outputFile = File.OpenWrite($"{unixDirPath}DownloadTest.txt"); - Assert.Throws(() => _fixture.Client.DownloadObject(_fixture.ReadBucket, _fixture.SmallObject, outputFile)); - } - } + var testFile = IdGenerator.FromGuid(prefix: "test"); + // Path.GetTempPath() is very likely to be outside the application's base directory. + string testFilePath = Path.Combine(Path.GetTempPath(), testFile); + using var outputFile = File.OpenWrite(testFilePath); + Assert.Throws(() => _fixture.Client.DownloadObject(_fixture.ReadBucket, _fixture.SmallObject, outputFile)); } [Fact] From 71964c7128fe86d73c85b5fc7946ea100a90974d Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Fri, 24 Oct 2025 02:01:46 -0700 Subject: [PATCH 08/12] fix(storage): add file deletion code in tests --- .../DownloadObjectTest.cs | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/DownloadObjectTest.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/DownloadObjectTest.cs index 073b2165240c..7bedb7cad55d 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/DownloadObjectTest.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/DownloadObjectTest.cs @@ -142,8 +142,19 @@ public void DownloadObjectFromInvalidBucket() public void DownloadObjectWithRelativePathTraversal() { var testFile = IdGenerator.FromGuid(prefix: "test"); - using var outputFile = File.OpenWrite($"../{testFile}"); - Assert.Throws(() => _fixture.Client.DownloadObject(_fixture.ReadBucket, _fixture.SmallObject, outputFile)); + string testFilePath = Path.Combine("../", testFile); + try + { + using var outputFile = File.OpenWrite(testFilePath); + Assert.Throws(() => _fixture.Client.DownloadObject(_fixture.ReadBucket, _fixture.SmallObject, outputFile)); + } + finally + { + if (File.Exists(testFilePath)) + { + File.Delete(testFilePath); + } + } } [Fact] @@ -152,8 +163,18 @@ public void DownloadObjectWithAbsolutePathTraversal() var testFile = IdGenerator.FromGuid(prefix: "test"); // Path.GetTempPath() is very likely to be outside the application's base directory. string testFilePath = Path.Combine(Path.GetTempPath(), testFile); - using var outputFile = File.OpenWrite(testFilePath); - Assert.Throws(() => _fixture.Client.DownloadObject(_fixture.ReadBucket, _fixture.SmallObject, outputFile)); + try + { + using var outputFile = File.OpenWrite(testFilePath); + Assert.Throws(() => _fixture.Client.DownloadObject(_fixture.ReadBucket, _fixture.SmallObject, outputFile)); + } + finally + { + if (File.Exists(testFilePath)) + { + File.Delete(testFilePath); + } + } } [Fact] From c030f7e1c8e1f531cd0536ef812ece13eaabf70b Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Mon, 27 Oct 2025 02:03:52 -0700 Subject: [PATCH 09/12] fix(storage): add cross-platform extention method to get rid of polymorphism abuse as suggetsed by gemini --- .../DownloadObjectTest.cs | 12 +- .../Google.Cloud.Storage.V1.csproj | 3 + .../StorageClientImpl.cs | 12 +- .../FileStreamExtensions.cs | 122 ++++++++++++++++++ 4 files changed, 142 insertions(+), 7 deletions(-) create mode 100644 tools/Google.Cloud.ClientTesting/FileStreamExtensions.cs diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/DownloadObjectTest.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/DownloadObjectTest.cs index 7bedb7cad55d..328d6e6db809 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/DownloadObjectTest.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/DownloadObjectTest.cs @@ -145,8 +145,10 @@ public void DownloadObjectWithRelativePathTraversal() string testFilePath = Path.Combine("../", testFile); try { - using var outputFile = File.OpenWrite(testFilePath); - Assert.Throws(() => _fixture.Client.DownloadObject(_fixture.ReadBucket, _fixture.SmallObject, outputFile)); + using var stream = File.OpenWrite(testFilePath); + // Create a new FileStream using only the handle. + using var fileStream = new FileStream(stream.SafeFileHandle, FileAccess.Write); + Assert.Throws(() => _fixture.Client.DownloadObject(_fixture.ReadBucket, _fixture.SmallObject, fileStream)); } finally { @@ -165,8 +167,10 @@ public void DownloadObjectWithAbsolutePathTraversal() string testFilePath = Path.Combine(Path.GetTempPath(), testFile); try { - using var outputFile = File.OpenWrite(testFilePath); - Assert.Throws(() => _fixture.Client.DownloadObject(_fixture.ReadBucket, _fixture.SmallObject, outputFile)); + using var stream = File.OpenWrite(testFilePath); + // Create a new FileStream using only the handle. + using var fileStream = new FileStream(stream.SafeFileHandle, FileAccess.Write); + Assert.Throws(() => _fixture.Client.DownloadObject(_fixture.ReadBucket, _fixture.SmallObject, fileStream)); } finally { diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.csproj b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.csproj index c1dac22ac237..7d570890423d 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.csproj +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.csproj @@ -12,6 +12,9 @@ + + + StorageClient.cs diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs index 1f93777d19fa..4ece5ec61cec 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs @@ -18,6 +18,7 @@ using Google.Apis.Requests; using Google.Apis.Storage.v1; using Google.Apis.Storage.v1.Data; +using Google.Cloud.ClientTesting; using System; using System.IO; using System.Net.Http; @@ -121,9 +122,13 @@ internal static string ValidateBucketName(string bucket) } /// - /// Validates object download path to the base directory only. - /// This method ensures that the object is downloaded securely and prevent directory traversal attack using absolute or relative path. + /// Validates the object download path to prevent traversal outside the base directory. /// + /// + /// This is a security measure to prevent directory traversal attacks. + /// It checks for malicious components (e.g., '..', or absolute paths) to ensure + /// the final resolved path is safely contained within the base directory. + /// internal void ValidateObjectDownloadPath(Stream stream) { GaxPreconditions.CheckNotNull(stream, nameof(stream)); @@ -136,7 +141,7 @@ internal void ValidateObjectDownloadPath(Stream stream) } if (stream is FileStream fileStream) { - string fullPath = Path.GetFullPath(fileStream.Name); + string fullPath = fileStream.GetActualPath(); if (!fullPath.StartsWith(baseDir, StringComparison.OrdinalIgnoreCase)) { throw new ArgumentException("Path traversal is not allowed. File path provided is outside the base directory"); @@ -228,5 +233,6 @@ internal EncryptionKey GetEffectiveEncryptionKey(EncryptionKey keyFromOptions, s private void MarkAsRetriable(StorageBaseServiceRequest request, RetryOptions options) => RetryHandler.MarkAsRetriable(request, options, _scheduler); + } } diff --git a/tools/Google.Cloud.ClientTesting/FileStreamExtensions.cs b/tools/Google.Cloud.ClientTesting/FileStreamExtensions.cs new file mode 100644 index 000000000000..31b0b073346c --- /dev/null +++ b/tools/Google.Cloud.ClientTesting/FileStreamExtensions.cs @@ -0,0 +1,122 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"): +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.ComponentModel; +using System.IO; +using System.Runtime.InteropServices; +using System.Text; +using Microsoft.Win32.SafeHandles; + +namespace Google.Cloud.ClientTesting +{ + public static class FileStreamExtensions + { + /// + /// Gets the actual, absolute path for the file associated with the FileStream handle. + /// + /// The FileStream to query. + /// The full, resolved path, or null if it cannot be determined. + public static string GetActualPath(this FileStream fileStream) + { + try + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return GetPathWindows(fileStream.SafeFileHandle); + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || + RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + return GetPathLinuxOrOsx(fileStream.SafeFileHandle); + } + } + catch (Exception) + { + // Handle or log errors, e.g., permission denied + } + + // Fallback or for unsupported platforms + return null; + } + + // --- Windows Implementation --- + + [DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Unicode)] + private static extern int GetFinalPathNameByHandle( + SafeFileHandle hFile, + [MarshalAs(UnmanagedType.LPWStr)] StringBuilder lpszFilePath, + int cchFilePath, + int dwFlags); + + private const int FILE_NAME_NORMALIZED = 0x0; + private const int VOLUME_NAME_DOS = 0x0; // Use this for "C:\" style paths + + private static string GetPathWindows(SafeFileHandle handle) + { + var builder = new StringBuilder(1024); // MAX_PATH is 260, but let's be safe + + // Use 0 for dwFlags to get the default (normalized) path. + // Or use VOLUME_NAME_DOS to ensure "C:\..." format. + int result = GetFinalPathNameByHandle(handle, builder, builder.Capacity, VOLUME_NAME_DOS); + + if (result == 0) + { + // Failed + throw new Win32Exception(Marshal.GetLastWin32Error()); + } + + // The result includes the terminating null, but StringBuilder doesn't. + // The path returned by GetFinalPathNameByHandle is prefixed with "\\?\" + // (the "long path" syntax). We need to remove this for normal use. + string path = builder.ToString(); + if (path.StartsWith(@"\\?\")) + { + return path.Substring(4); + } + + return path; + } + + // --- Linux & macOS Implementation --- + + [DllImport("libc", SetLastError = true, CharSet = CharSet.Ansi)] + private static extern int readlink( + string pathname, + StringBuilder buf, + int bufsiz); + + private static string GetPathLinuxOrOsx(SafeFileHandle handle) + { + // On Unix, the file descriptor (handle) is an integer. + int fd = handle.DangerousGetHandle().ToInt32(); + + // The path to the symlink for this process's file descriptor + string symlinkPath = $"/proc/self/fd/{fd}"; + + var builder = new StringBuilder(1024); + int result = readlink(symlinkPath, builder, builder.Capacity); + + if (result == -1) + { + // Failed + throw new Win32Exception(Marshal.GetLastWin32Error()); + } + + // readlink doesn't null-terminate the string in the buffer + return builder.ToString(0, result); + } + } +} From a827fd65d9d585620239b5c882366e84741d38d3 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Mon, 27 Oct 2025 02:20:03 -0700 Subject: [PATCH 10/12] fix(storage): remove blank line from storage client implementation --- .../Google.Cloud.Storage.V1/StorageClientImpl.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs index 4ece5ec61cec..d6a79f16ee3e 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs @@ -233,6 +233,5 @@ internal EncryptionKey GetEffectiveEncryptionKey(EncryptionKey keyFromOptions, s private void MarkAsRetriable(StorageBaseServiceRequest request, RetryOptions options) => RetryHandler.MarkAsRetriable(request, options, _scheduler); - } } From 7c1bed7e58d57af2106f8a09175b72fa2a6dd07b Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Mon, 27 Oct 2025 04:00:01 -0700 Subject: [PATCH 11/12] fix(storage): add comment in ValidateObjectDownloadPath method --- .../Google.Cloud.Storage.V1/StorageClientImpl.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs index d6a79f16ee3e..c37e7414c183 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.cs @@ -147,6 +147,8 @@ internal void ValidateObjectDownloadPath(Stream stream) throw new ArgumentException("Path traversal is not allowed. File path provided is outside the base directory"); } } + // For other stream types (e.g., MemoryStream, NetworkStream), path traversal is not applicable, + // as they do not represent local file system paths. No further validation is needed here. } /// From a1f2d59dfaf6a0efa2f0684a142cd5f5943ebfac Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 11 Dec 2025 07:06:14 -0800 Subject: [PATCH 12/12] feat(Storage): Add hashuploadvalidator in upload object --- .../UploadObjectTest.cs | 3 + .../HashValidatingUploaderTest.cs | 219 ++++++++++++++++++ .../CustomMediaUpload.cs | 4 +- .../Google.Cloud.Storage.V1.csproj | 2 +- .../HashValidationUploader.cs | 110 +++++++++ .../MetadataRecordingMediaDownloader.cs | 2 +- .../MetadataRecordingMediaUploader.cs | 90 +++++++ .../StorageClientImpl.UploadObject.cs | 5 +- 8 files changed, 430 insertions(+), 5 deletions(-) create mode 100644 apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.Tests/HashValidatingUploaderTest.cs create mode 100644 apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/HashValidationUploader.cs create mode 100644 apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/MetadataRecordingMediaUploader.cs diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/UploadObjectTest.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/UploadObjectTest.cs index 906b8137e749..af99356b770a 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/UploadObjectTest.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.IntegrationTests/UploadObjectTest.cs @@ -72,6 +72,7 @@ public void CorsUpload() [Fact] public void UploadWithObject() { + var client = StorageClient.Create(); var destination = new Object { Bucket = _fixture.MultiVersionBucket, @@ -81,6 +82,8 @@ public void UploadWithObject() Metadata = new Dictionary { { "x", "y" } } }; var source = GenerateData(100); + var interceptor = new BreakUploadInterceptor(); + client.Service.HttpClient.MessageHandler.AddExecuteInterceptor(interceptor); var result = _fixture.Client.UploadObject(destination, source); Assert.NotSame(destination, result); Assert.Equal(destination.Name, result.Name); diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.Tests/HashValidatingUploaderTest.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.Tests/HashValidatingUploaderTest.cs new file mode 100644 index 000000000000..2eb91e398173 --- /dev/null +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.Tests/HashValidatingUploaderTest.cs @@ -0,0 +1,219 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +using Google.Api; +using Google.Api.Gax; +using Google.Apis.Download; +using Google.Apis.Http; +using Google.Apis.Services; +using Google.Apis.Upload; +using Google.Cloud.ClientTesting; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using Xunit; + +namespace Google.Cloud.Storage.V1.Tests +{ + public class HashValidatingUploaderTest + { + private static readonly byte[] s_data = Enumerable.Range(0, 200000).Select(i => (byte) i).ToArray(); + private const string Md5Value = "md5=PL4+tNeevS58x7PmIPW6YA=="; + private const string Crc32Value = "crc32c=WfO2Ug=="; + + [Theory] + [InlineData(null)] + [InlineData(Md5Value)] + [InlineData(Md5Value + "," + Crc32Value)] + [InlineData(Crc32Value + "," + Md5Value)] + [InlineData(Crc32Value)] + public void Valid_ModeAlways(string headerValue) + { + HashValidatingUploader uploader = CreateUploader(request => CreateResponse(request, headerValue), UploadValidationMode.DeleteAndThrow); + AssertDownloadSucceeds(uploader); + } + + [Theory] + [InlineData("gzip", "gzip")] + [InlineData(null, "gzip")] + [InlineData("plain", null)] + [InlineData("plain", "gzip")] + [InlineData("gzip", "plain")] + [InlineData("gzip", null)] + [InlineData(null, null)] + [InlineData("plain", "plain")] + public void Valid_ModeAutomatic(string storedContentEncoding, string contentEncoding) + { + // This test doesn't try different hashes exhaustively; the way it treats the header + // is the same as with Always. + string headerValue = Md5Value + "," + Crc32Value; + HashValidatingUploader uploader = CreateUploader( + request => CreateResponse(request, headerValue, storedContentEncoding, contentEncoding), + UploadValidationMode.DeleteAndThrow); + AssertDownloadSucceeds(uploader); + } + + [Fact] + public void Invalid_ModeAlways() + { + HashValidatingUploader uploader = CreateUploader(request => CreateResponse(request, "crc32c=Bogus1=="), UploadValidationMode.DeleteAndThrow); + AssertDownloadFails(uploader); + } + + // - Mode is DownloadValidationMode.Automatic + // - Data is incorrect + // - And we're validating (so will throw) + [Theory] + [InlineData("gzip", "gzip")] + [InlineData(null, "gzip")] + [InlineData("plain", null)] + [InlineData("plain", "gzip")] + [InlineData(null, null)] + [InlineData("plain", "plain")] + public void InvalidData_ModeAutomatic_Validated(string storedContentEncoding, string contentEncoding) + { + HashValidatingUploader uploader = CreateUploader( + request => CreateResponse(request, "crc32c=Bogus1==", storedContentEncoding, contentEncoding), + UploadValidationMode.DeleteAndThrow); + AssertDownloadFails(uploader); + } + + // - Mode is DownloadValidationMode.Automatic + // - Data is incorrect + // - We're not validating because of the stored content encoding / content encoding (so will "succeed") + [Theory] + [InlineData("gzip", "plain")] + [InlineData("gzip", null)] + public void InvalidData_ModeAutomatic_NotValidated(string storedContentEncoding, string contentEncoding) + { + HashValidatingUploader downloader = CreateUploader( + request => CreateResponse(request, "crc32c=Bogus1==", storedContentEncoding, contentEncoding), + UploadValidationMode.ThrowOnly); + AssertDownloadSucceeds(downloader); + } + + [Fact] + public void Invalid_ModeNever() + { + HashValidatingUploader downloader = CreateUploader(request => CreateResponse(request, "crc32c=Bogus1=="), UploadValidationMode.DeleteAndThrow); + AssertDownloadSucceeds(downloader); + } + + private static void AssertDownloadSucceeds(HashValidatingUploader uploader) + { + for (int chunks = 1; chunks < 5; chunks++) + { + // Make sure it definitely fits... + uploader.ChunkSize = 1 * 1024 * 1024; + var stream = new MemoryStream(); + var status = uploader.Upload(); + if (status.Exception != null) + { + throw status.Exception; + } + Assert.Equal(UploadStatus.Completed, status.Status); + Assert.Equal(s_data, stream.ToArray()); + } + } + + private static void AssertDownloadFails(HashValidatingUploader uploader) + { + for (int chunks = 1; chunks < 5; chunks++) + { + // Make sure it definitely fits... + uploader.ChunkSize = (s_data.Length + 100) / chunks; + var stream = new MemoryStream(); + var status = uploader.Upload(); + Assert.Equal(UploadStatus.Failed, status.Status); + Assert.Contains("Incorrect hash", status.Exception.Message); + } + } + + private static HashValidatingUploader CreateUploader(Func handler, UploadValidationMode mode) + { + var service = new MockableService(handler); + var metadata = new Apis.Storage.v1.Data.Object(); + var bucket = new Apis.Storage.v1.Data.Bucket(); + string contentType = "application/json"; + var stream = new MemoryStream(); + return new HashValidatingUploader(service, metadata, bucket.Name, stream, contentType, mode); + } + + internal class MockableService : BaseClientService + { + internal MockableService(Func handler) + : base(GetInitializer(handler)) + { + } + + public override string BasePath => ""; + public override string BaseUri => ""; + public override IList Features => new List(); + public override string Name => "Mockable"; + + private static Initializer GetInitializer(Func handler) + { + var httpMessageHandler = new MockableMessageHandler(handler); + var configurableHandler = new ConfigurableMessageHandler(httpMessageHandler); + var clientFactory = new FakeHttpClientFactory(configurableHandler); + return new Initializer { HttpClientFactory = clientFactory }; + } + } + + private static HttpResponseMessage CreateResponse( + HttpRequestMessage request, string hashHeaderValue, + string storedContentEncoding = null, string contentEncoding = null) + { + HttpResponseMessage response = new HttpResponseMessage { Content = new ByteArrayContent(s_data) }; + if (hashHeaderValue != null) + { + response.Headers.Add(Crc32c.HashHeaderName, hashHeaderValue); + } + if (storedContentEncoding != null) + { + response.Headers.Add(HashValidatingUploader.StoredContentEncodingHeaderName, storedContentEncoding); + } + if (contentEncoding != null) + { + response.Content.Headers.ContentEncoding.Add(contentEncoding); + } + MaybeIntercept(request, response); + return response; + } + + // This is ugly - it's effectively mimicing the code in the REST support library for response stream + // interception. We basically want to use Google.Apis.Http.StreamInterceptionHandler, but that's (understandably) + // internal. Fake it for now. An alternative might be to use reflection, but that's brittle too. + private static void MaybeIntercept(HttpRequestMessage request, HttpResponseMessage response) + { + // HttpRequestMessage.Properties is obsolete, but that's what the support library uses, and + // HttpRequestMessage.Options isn't even available in .NET Framework. +#pragma warning disable CS0618 + var provider = request.Properties[ConfigurableMessageHandler.ResponseStreamInterceptorProviderKey] as Func; +#pragma warning restore CS0618 + if (provider == null) + { + return; + } + var interceptor = provider(response); + if (interceptor == null) + { + return; + } + interceptor(s_data, 0, s_data.Length); + return; + } + } +} diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/CustomMediaUpload.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/CustomMediaUpload.cs index c91a50ef9265..045d8d2be694 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/CustomMediaUpload.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/CustomMediaUpload.cs @@ -1,4 +1,4 @@ -// Copyright 2017 Google Inc. All Rights Reserved. +// Copyright 2017 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace Google.Cloud.Storage.V1 /// /// Upload subclass which allows us to modify headers, for customer-supplied encryption keys. /// - internal sealed class CustomMediaUpload : InsertMediaUpload + internal class CustomMediaUpload : InsertMediaUpload { public CustomMediaUpload(IClientService service, Apis.Storage.v1.Data.Object body, string bucket, Stream stream, string contentType) diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.csproj b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.csproj index 7d570890423d..71d98a4ad5d9 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.csproj +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.csproj @@ -10,7 +10,7 @@ - + diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/HashValidationUploader.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/HashValidationUploader.cs new file mode 100644 index 000000000000..07ba79ee2f3b --- /dev/null +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/HashValidationUploader.cs @@ -0,0 +1,110 @@ +// Copyright 2025 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Google.Apis.Download; +using Google.Apis.Http; +using Google.Apis.Services; +using Google.Apis.Storage.v1; +using Google.Apis.Storage.v1.Data; +using Google.Apis.Upload; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; + +namespace Google.Cloud.Storage.V1 +{ + /// + /// Subclass of which (potentially) validates the data it receives + /// against a CRC32c hash set in the header. + /// + internal sealed class HashValidatingUploader : ContentMetadataRecordingMediaUploader + { + internal const string StoredContentEncodingHeaderName = "x-goog-hash"; + + private readonly UploadValidationMode _mode; + + public System.Func ResponseStreamInterceptorProvider { get; } + + private string _crc32cHashBase64; + private Crc32c _hasher; + + /// Constructs a new uploader with the given client service. + internal HashValidatingUploader(IClientService service, Object body, string bucket, + Stream stream, string contentType, UploadValidationMode mode) + : base(service, body, bucket, stream, contentType) + { + _mode = mode; + ResponseStreamInterceptorProvider = CreateInterceptor; + } + + private StreamInterceptor CreateInterceptor(HttpResponseMessage response) + { + _crc32cHashBase64 = null; + _hasher = null; + + switch (_mode) + { + case UploadValidationMode.ThrowOnly: + return PrepareForHashing(); + case UploadValidationMode.None: + return null; + case UploadValidationMode.DeleteAndThrow: + bool decompressedByServer = + response.Headers.TryGetValues(StoredContentEncodingHeaderName, out var storedContentEncoding) && + storedContentEncoding.FirstOrDefault() == "gzip" && + response.Content?.Headers?.ContentEncoding?.FirstOrDefault() != "gzip"; + return decompressedByServer ? null : PrepareForHashing(); + default: + return null; + } + + StreamInterceptor PrepareForHashing() + { + IEnumerable values; + if (response.Headers.TryGetValues(Crc32c.HashHeaderName, out values)) + { + string prefix = Crc32c.HashName + "="; + foreach (var value in values.SelectMany(v => v.Split(','))) + { + if (value.StartsWith(prefix)) + { + _hasher = new Crc32c(); + _crc32cHashBase64 = value.Substring(prefix.Length); + return _hasher.UpdateHash; + } + } + } + // The mode indicates that we'd like to validate the hash, but we don't have one to validate. + // (We could potentially have a mode of "fail if there isn't a hash to validate", in the future, + // but for the moment we just ignore it.) + return null; + } + } + + //protected override void Upload() + //{ + // base.Upload(); + + // if (_crc32cHashBase64 != null) + // { + // string actualHash = System.Convert.ToBase64String(_hasher.GetHash()); + // if (actualHash != _crc32cHashBase64) + // { + // throw new IOException($"Incorrect hash: expected '{_crc32cHashBase64}' (base64), was '{actualHash}' (base64)"); + // } + // } + //} + } +} diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/MetadataRecordingMediaDownloader.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/MetadataRecordingMediaDownloader.cs index 42489b4daa89..bc3af7dd1651 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/MetadataRecordingMediaDownloader.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/MetadataRecordingMediaDownloader.cs @@ -1,4 +1,4 @@ -// Copyright 2022 Google LLC +// Copyright 2022 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/MetadataRecordingMediaUploader.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/MetadataRecordingMediaUploader.cs new file mode 100644 index 000000000000..8dcdbc2e5483 --- /dev/null +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/MetadataRecordingMediaUploader.cs @@ -0,0 +1,90 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Google.Apis.Download; +using Google.Apis.Services; +using Google.Apis.Storage.v1.Data; +using Google.Apis.Upload; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading; +using System.Threading.Tasks; +using static Google.Apis.Requests.BatchRequest; + +namespace Google.Cloud.Storage.V1; + +/// +/// CustomMediaUpload subclass which populates an instance +/// from the headers it sees on a successful response. Note: if this ever becomes public, +/// we should put more effort into naming it carefully. +/// +internal class ContentMetadataRecordingMediaUploader : CustomMediaUpload +{ + private const string ETagHeader = "ETag"; + private const string GenerationHeader = "X-Goog-Generation"; + private const string MetagenerationHeader = "X-Goog-Metageneration"; + private const string HashHeader = "X-Goog-Hash"; + private const string ContentTypeHeader = "Content-Type"; + + // The hashes are comma-separated... + private static readonly char[] HashToElementSplitter = new[] { ',' }; + // ... and each hash is a key=value pair + private static readonly char[] HashKeyValueSplitter = new[] { '=' }; + private readonly HttpResponseMessage _response = new HttpResponseMessage(); + + /// Constructs a new uploader with the given client service. + /// + /// + /// The client service. + /// + /// + + internal ContentMetadataRecordingMediaUploader(IClientService service, Apis.Storage.v1.Data.Object body, string bucket, + Stream stream, string contentType) + : base(service, body, bucket, stream, contentType) + { + Body = body; + ProcessMetadataHeaders(_response.Headers, _response.Content.Headers); + } + + + private void ProcessMetadataHeaders(HttpResponseHeaders headers, HttpContentHeaders contentHeaders) + { + Body.Generation = MaybeParse(GetFirstHeaderOrNull(GenerationHeader)); + Body.Metageneration = MaybeParse(GetFirstHeaderOrNull(MetagenerationHeader)); + Body.ETag = GetFirstHeaderOrNull(ETagHeader); + var hashes = GetFirstHeaderOrNull(HashHeader) ?? ""; + // The hash header returns multiple comma-separated hashes. + var hashesByKey = hashes.Split(HashToElementSplitter) + .Where(hash => hash.Contains('=')) + .Select(hash => hash.Split(HashKeyValueSplitter, 2)) + .ToDictionary(bits => bits[0], bits => bits[1]); + Body.Crc32c = hashesByKey.TryGetValue("crc32c", out string crc32c) ? crc32c : null; + Body.Md5Hash = hashesByKey.TryGetValue("md5", out string md5) ? md5 : null; + Body.ContentType = contentHeaders.ContentType?.ToString(); + + string GetFirstHeaderOrNull(string headerName) => + headers.TryGetValues(headerName, out var values) ? values.FirstOrDefault() : null; + + long? MaybeParse(string text) => + text is null || !long.TryParse(text, NumberStyles.None, CultureInfo.InvariantCulture, out var value) + ? (long?) null + : value; + } +} diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.UploadObject.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.UploadObject.cs index 84e7f457edb6..187be454a623 100644 --- a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.UploadObject.cs +++ b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.UploadObject.cs @@ -48,7 +48,10 @@ public override ObjectsResource.InsertMediaUpload CreateObjectUploader( { ValidateObject(destination, nameof(destination)); GaxPreconditions.CheckNotNull(source, nameof(source)); - var mediaUpload = new CustomMediaUpload(Service, destination, destination.Bucket, source, destination.ContentType); + UploadValidationMode mode = options?.UploadValidationMode ?? UploadValidationMode.DeleteAndThrow; + GaxPreconditions.CheckEnumValue(mode, nameof(UploadObjectOptions.UploadValidationMode)); + + var mediaUpload = new HashValidatingUploader(Service, destination, destination.Bucket, source, destination.ContentType, mode); options?.ModifyMediaUpload(mediaUpload); ApplyEncryptionKey(options?.EncryptionKey, options?.KmsKeyName, mediaUpload); return mediaUpload;