From 338ea667567732c7016044d4d224383af4a98b51 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Tue, 3 Jun 2025 01:01:15 +0000 Subject: [PATCH 01/19] samples(Storage Batch Operations): Add samples and tests for storage batch operations --- storagebatchoperations/api/README.md | 31 ++++ .../CancelBatchJobTest.cs | 124 ++++++++++++++++ .../CreateBatchJobTest.cs | 135 ++++++++++++++++++ .../DeleteBatchJobTest.cs | 64 +++++++++ .../GetBatchJobTest.cs | 67 +++++++++ .../ListBatchJobsTest.cs | 73 ++++++++++ ...torageBatchOperations.Samples.Tests.csproj | 20 +++ .../StorageFixture.cs | 120 ++++++++++++++++ .../CancelBatchJob.cs | 41 ++++++ .../CreateBatchJob.cs | 124 ++++++++++++++++ .../DeleteBatchJob.cs | 39 +++++ .../GetBatchJob.cs | 39 +++++ .../ListBatchJobs.cs | 62 ++++++++ .../StorageBatchOperations.Samples.csproj | 12 ++ .../api/StorageBatchOperations.sln | 64 +++++++++ storagebatchoperations/api/runTests.ps1 | 16 +++ 16 files changed, 1031 insertions(+) create mode 100644 storagebatchoperations/api/README.md create mode 100644 storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs create mode 100644 storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobTest.cs create mode 100644 storagebatchoperations/api/StorageBatchOperations.Samples.Tests/DeleteBatchJobTest.cs create mode 100644 storagebatchoperations/api/StorageBatchOperations.Samples.Tests/GetBatchJobTest.cs create mode 100644 storagebatchoperations/api/StorageBatchOperations.Samples.Tests/ListBatchJobsTest.cs create mode 100644 storagebatchoperations/api/StorageBatchOperations.Samples.Tests/StorageBatchOperations.Samples.Tests.csproj create mode 100644 storagebatchoperations/api/StorageBatchOperations.Samples.Tests/StorageFixture.cs create mode 100644 storagebatchoperations/api/StorageBatchOperations.Samples/CancelBatchJob.cs create mode 100644 storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJob.cs create mode 100644 storagebatchoperations/api/StorageBatchOperations.Samples/DeleteBatchJob.cs create mode 100644 storagebatchoperations/api/StorageBatchOperations.Samples/GetBatchJob.cs create mode 100644 storagebatchoperations/api/StorageBatchOperations.Samples/ListBatchJobs.cs create mode 100644 storagebatchoperations/api/StorageBatchOperations.Samples/StorageBatchOperations.Samples.csproj create mode 100644 storagebatchoperations/api/StorageBatchOperations.sln create mode 100644 storagebatchoperations/api/runTests.ps1 diff --git a/storagebatchoperations/api/README.md b/storagebatchoperations/api/README.md new file mode 100644 index 00000000000..7556587ab4f --- /dev/null +++ b/storagebatchoperations/api/README.md @@ -0,0 +1,31 @@ +# Cloud Storage Batch Operations Sample + +These samples demonstrate how to interact with the [Google Cloud Storage Batch Operations API][Storage Batch Operations] from C# and +the .NET client libraries to call the Storage Batch Operations API. + +The samples requires [.NET 8][.NET].That means using +[Visual Studio 2022](https://www.visualstudio.com/), or the command line. + +## Setup + +1. Set up a [.NET development environment](https://cloud.google.com/dotnet/docs/setup). + +2. Enable APIs for your project. + [Click here][enable-api] + to visit Cloud Platform Console and enable the Google Cloud Storage Batch Operations API. + +## Contributing changes + +* See [CONTRIBUTING.md](../../CONTRIBUTING.md) + +## Licensing + +* See [LICENSE](../../LICENSE) + +## Testing + +* See [TESTING.md](../../TESTING.md) + +[Storage Batch Operations]: https://cloud.google.com/storage/docs/batch-operations/overview +[enable-api]: https://console.cloud.google.com/flows/enableapi?apiid=storagebatchoperations_api&showconfirmation=true +[.NET]: https://dotnet.microsoft.com/en-us/download diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs new file mode 100644 index 00000000000..c0d0cd2910a --- /dev/null +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs @@ -0,0 +1,124 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Google.Api.Gax.ResourceNames; +using Google.Cloud.StorageBatchOperations.V1; +using Google.LongRunning; +using GoogleCloudSamples; +using System; +using System.IO; +using System.Text; +using System.Threading.Tasks; +using Xunit; + +[Collection(nameof(StorageFixture))] +public class CancelBatchJobTest +{ + private readonly StorageFixture _fixture; + private readonly BucketList.Types.Bucket _bucket = new(); + private readonly BucketList _bucketList = new(); + private readonly PrefixList _prefixListObject = new(); + + public CancelBatchJobTest(StorageFixture fixture) + { + int i = 10; + _fixture = fixture; + var bucketName = _fixture.GenerateBucketName(); + _fixture.CreateBucket(bucketName, multiVersion: false, softDelete: false, registerForDeletion: true); + // Uploading objects to the bucket. + while (i >= 0) + { + var objectName = _fixture.GenerateGuid(); + var objectContent = _fixture.GenerateGuid(); + byte[] byteObjectContent = Encoding.UTF8.GetBytes(objectContent); + MemoryStream streamObjectContent = new MemoryStream(byteObjectContent); + _fixture.Client.UploadObject(bucketName, objectName, "application/text", streamObjectContent); + i--; + } + _bucket = new BucketList.Types.Bucket + { + Bucket_ = bucketName, + // The prefix list is used to specify the objects to be deleted. To match all objects, use an empty list. + PrefixList = _prefixListObject + }; + // Adding the bucket to the bucket list. + _bucketList.Buckets.Insert(0, _bucket); + } + + [Fact] + public void TestCancelBatchJob() + { + CancelBatchJobSample cancelBatchJob = new CancelBatchJobSample(); + ListBatchJobsSample listBatchJobs = new ListBatchJobsSample(); + GetBatchJobSample getBatchJob = new GetBatchJobSample(); + + RetryRobot robot = new RetryRobot() + { + MaxTryCount = 20, + ShouldRetry = (e) => true, + }; + + string filter = "state:canceled"; + int pageSize = 10; + string orderBy = "create_time"; + + var jobId = _fixture.GenerateGuid(); + + var createdJob = CreateBatchJob(_fixture.LocationName, _bucketList, jobId); + var cancelJobResponse = cancelBatchJob.CancelBatchJob(createdJob); + var batchJobs = listBatchJobs.ListBatchJobs(_fixture.LocationName, filter, pageSize, orderBy); + robot.Eventually(() => Assert.Contains(batchJobs, job => job.Name == createdJob && job.State == Job.Types.State.Canceled)); + Job cancelledJob = getBatchJob.GetBatchJob(createdJob); + Assert.Equal(createdJob, cancelledJob.Name.ToString()); + Assert.Equal("Canceled", cancelledJob.State.ToString()); + _fixture.DeleteBatchJob(createdJob); + } + + /// + /// Create a batch job with the specified transformation case and bucket list. + /// + public static string CreateBatchJob(LocationName locationName, + BucketList bucketList, + string jobId = "12345678910") + { + StorageBatchOperationsClient storageBatchClient = StorageBatchOperationsClient.Create(); + + RetryRobot robot = new RetryRobot() + { + MaxTryCount = 20, + ShouldRetry = (e) => true, + }; + + // Creates a batch job with the specified bucket list and delete object settings. + CreateJobRequest request = new CreateJobRequest + { + ParentAsLocationName = locationName, + JobId = jobId, + Job = new Job + { + BucketList = bucketList, + DeleteObject = new DeleteObject { PermanentObjectDeletionEnabled = true } + }, + RequestId = jobId, + }; + + Operation response = storageBatchClient.CreateJob(request); + string operationName = response.Name; + Operation retrievedResponse = storageBatchClient.PollOnceCreateJob(operationName); + // Poll once asynchronously. + Task> retrievedAsyncResponse = retrievedResponse.PollOnceAsync(); + string jobName = robot.Eventually(() => retrievedAsyncResponse.Result.Metadata.Job.Name); + return jobName; + } +} diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobTest.cs new file mode 100644 index 00000000000..a71846abc37 --- /dev/null +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobTest.cs @@ -0,0 +1,135 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Google.Cloud.StorageBatchOperations.V1; +using System; +using System.IO; +using System.Text; +using Xunit; + +[Collection(nameof(StorageFixture))] +public class CreateBatchJobTest +{ + private readonly StorageFixture _fixture; + private readonly BucketList.Types.Bucket _bucket = new(); + private readonly BucketList _bucketList = new(); + private readonly PrefixList _prefixListObject = new(); + private string _kmsKey; + private string _keyRingId; + private string _cryptoKeyId; + private CryptoKeyName _cryptoKeyName; + + public CreateBatchJobTest(StorageFixture fixture) + { + _fixture = fixture; + var bucketName = _fixture.GenerateBucketName(); + _fixture.CreateBucket(bucketName, multiVersion: false, softDelete: false, registerForDeletion: true); + + var manifestBucketName = _fixture.GenerateBucketName(); + _fixture.CreateBucket(manifestBucketName, multiVersion: false, softDelete: false, registerForDeletion: true); + + var objectName = _fixture.GenerateGuid(); + var manifestObjectName = _fixture.GenerateGuid(); + var objectContent = _fixture.GenerateGuid(); + var manifestObjectContent = $"bucket,name,generation{Environment.NewLine}{bucketName},{objectName}"; + + byte[] byteObjectContent = Encoding.UTF8.GetBytes(objectContent); + MemoryStream streamObjectContent = new MemoryStream(byteObjectContent); + // Uploading an object to the bucket + _fixture.Client.UploadObject(bucketName, objectName, "application/text", streamObjectContent); + + byte[] byteManifestObjectContent = Encoding.UTF8.GetBytes(manifestObjectContent); + // Uploading a manifest object to the manifest bucket + MemoryStream streamManifestObjectContent = new MemoryStream(byteManifestObjectContent); + _fixture.Client.UploadObject(manifestBucketName, $"{manifestObjectName}.csv", "text/csv", streamManifestObjectContent); + _bucket = new BucketList.Types.Bucket + { + Bucket_ = bucketName, + // The prefix list is used to specify the objects to be transformed. To match all objects, use an empty list. + PrefixList = _prefixListObject, + // Manifest location contains csv file having list of objects to be transformed" + Manifest = new Manifest { ManifestLocation = $"gs://{manifestBucketName}/{manifestObjectName}.csv" } + }; + // Adding the bucket to the bucket list. + _bucketList.Buckets.Insert(0, _bucket); + } + + [Fact] + public void TestCreateBatchJob() + { + CreateBatchJobSample createJob = new CreateBatchJobSample(); + var jobId = _fixture.GenerateGuid(); + var jobTransformationCase = "DeleteObject"; + var holdState = "EventBasedHoldSet"; + var jobTransformationObject = new object(); + string jobType; + + // If the job transformation case is PutObjectHold, we can set the hold state to EventBasedHoldSet or EventBasedHoldUnSet or TemporaryHoldSet or TemporaryHoldUnSet. + if (jobTransformationCase == "PutObjectHold") + { + jobType = $"{jobTransformationCase}{holdState}"; + } + // If the job transformation case is other than PutObjectHold, we dont set the hold state. + else + { + jobType = jobTransformationCase; + } + // If the job transformation case is RewriteObject, we can set the KmsKey and KmsKeyAsCryptoKeyName. + if (jobTransformationCase == "RewriteObject") + { + _keyRingId = GetEnvironmentVariable("STORAGE_KMS_KEYRING_ID", "This is the Key Ring ID"); + _cryptoKeyId = GetEnvironmentVariable("STORAGE_KMS_CRYPTOKEY_ID", "This is the Crypto Key ID."); + _kmsKey = $"projects/{_fixture.ProjectId}/locations/{_fixture.LocationId}/keyRings/{_keyRingId}/cryptoKeys/{_cryptoKeyId}"; + _cryptoKeyName = CryptoKeyName.FromProjectLocationKeyRingCryptoKey(_fixture.ProjectId, _fixture.LocationId, _keyRingId, _cryptoKeyId); + RewriteObject rewriteObject = new RewriteObject { KmsKey = _kmsKey, KmsKeyAsCryptoKeyName = _cryptoKeyName }; + jobTransformationObject = rewriteObject; + + } + // If the job transformation case is PutMetadata, we can set the CacheControl, ContentDisposition, ContentEncoding, ContentLanguage, ContentType and CustomTime. + else if (jobTransformationCase == "PutMetadata") + { + PutMetadata putMetadata = new PutMetadata + { + CacheControl = "no-cache", + ContentDisposition = "inline", + ContentEncoding = "gzip", + ContentLanguage = "en-US", + ContentType = "text/plain", + CustomTime = DateTime.UtcNow.ToString("o") + }; + jobTransformationObject = putMetadata; + } + // Create a batch job with the specified transformation case and bucket list + var createdBatchJob = createJob.CreateBatchJob(_fixture.LocationName, _bucketList, jobId, jobType, jobTransformationObject); + Assert.Equal(createdBatchJob.BucketList, _bucketList); + Assert.Equal(createdBatchJob.TransformationCase.ToString(), jobTransformationCase); + Assert.Equal(createdBatchJob.SourceCase.ToString(), _bucketList.GetType().Name); + Assert.NotNull(createdBatchJob.Name); + Assert.NotNull(createdBatchJob.JobName); + Assert.NotNull(createdBatchJob.CreateTime); + Assert.NotNull(createdBatchJob.CompleteTime); + _fixture.DeleteBatchJob(createdBatchJob.Name); + } + + private static string GetEnvironmentVariable(string envVarName, string message = "") + { + string varValue = Environment.GetEnvironmentVariable(envVarName); + if (string.IsNullOrEmpty(varValue)) + { + throw new InvalidOperationException( + $"Please set the {envVarName} environment variable. {message}"); + } + return varValue; + } +} diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/DeleteBatchJobTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/DeleteBatchJobTest.cs new file mode 100644 index 00000000000..ed68f59deeb --- /dev/null +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/DeleteBatchJobTest.cs @@ -0,0 +1,64 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Google.Cloud.StorageBatchOperations.V1; +using Xunit; + +[Collection(nameof(StorageFixture))] +public class DeleteBatchJobTest +{ + private readonly StorageFixture _fixture; + private readonly BucketList.Types.Bucket _bucket = new(); + private readonly BucketList _bucketList = new(); + private readonly PrefixList _prefixListObject = new(); + + public DeleteBatchJobTest(StorageFixture fixture) + { + _fixture = fixture; + var bucketName = _fixture.GenerateBucketName(); + _fixture.CreateBucket(bucketName, multiVersion: false, softDelete: false, registerForDeletion: true); + _bucket = new BucketList.Types.Bucket + { + Bucket_ = bucketName, + // The prefix list is used to specify the objects to be deleted. To match all objects, use an empty list. + PrefixList = _prefixListObject + }; + // Adding the bucket to the bucket list. + _bucketList.Buckets.Insert(0, _bucket); + } + + [Fact] + public void TestDeleteBatchJob() + { + DeleteBatchJobSample deleteBatchJob = new DeleteBatchJobSample(); + ListBatchJobsSample listBatchJobs = new ListBatchJobsSample(); + GetBatchJobSample getBatchJob = new GetBatchJobSample(); + CreateBatchJobSample createBatchJob = new CreateBatchJobSample(); + + string filter = ""; + int pageSize = 10; + string orderBy = "create_time"; + + var jobId = _fixture.GenerateGuid(); + var createdJob = createBatchJob.CreateBatchJob(_fixture.LocationName, _bucketList, jobId); + // Delete the created job. + deleteBatchJob.DeleteBatchJob(createdJob.Name); + var batchJobs = listBatchJobs.ListBatchJobs(_fixture.LocationName, filter, pageSize, orderBy); + // Verify that the job is deleted. + Assert.DoesNotContain(batchJobs, job => job.JobName == createdJob.JobName); + // Attempt to get the deleted job, which should throw an exception. + var exception = Assert.Throws(() => getBatchJob.GetBatchJob(createdJob.Name)); + Assert.Equal(Grpc.Core.StatusCode.NotFound, exception.StatusCode); + } +} diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/GetBatchJobTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/GetBatchJobTest.cs new file mode 100644 index 00000000000..697779904e9 --- /dev/null +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/GetBatchJobTest.cs @@ -0,0 +1,67 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Google.Cloud.StorageBatchOperations.V1; +using Xunit; + +[Collection(nameof(StorageFixture))] +public class GetBatchJobTest +{ + private readonly StorageFixture _fixture; + private readonly BucketList.Types.Bucket _bucket = new(); + private readonly BucketList _bucketList = new(); + private readonly PrefixList _prefixListObject = new(); + + public GetBatchJobTest(StorageFixture fixture) + { + _fixture = fixture; + var bucketName = _fixture.GenerateBucketName(); + _fixture.CreateBucket(bucketName, multiVersion: false, softDelete: false, registerForDeletion: true); + _bucket = new BucketList.Types.Bucket + { + Bucket_ = bucketName, + // The prefix list is used to specify the objects to be deleted. To match all objects, use an empty list. + PrefixList = _prefixListObject + }; + // Adding the bucket to the bucket list. + _bucketList.Buckets.Insert(0, _bucket); + } + + [Fact] + public void TestGetBatchJob() + { + GetBatchJobSample getJob = new GetBatchJobSample(); + CreateBatchJobSample createJob = new CreateBatchJobSample(); + + var jobId = _fixture.GenerateGuid(); + // Create a batch job with the specified bucket list and job ID. + var createdJob = createJob.CreateBatchJob(_fixture.LocationName, _bucketList, jobId); + // Get the created job using its name. + var retrievedJob = getJob.GetBatchJob(createdJob.Name); + // Assert that the retrieved job is not null. + Assert.NotNull(retrievedJob); + // Assert that the retrieved job's metadata matches with the created job's metadata. + Assert.Equal(createdJob.Name, retrievedJob.Name); + Assert.Equal(createdJob.BucketList, retrievedJob.BucketList); + Assert.Equal(createdJob.TransformationCase.ToString(), retrievedJob.TransformationCase.ToString()); + Assert.Equal(createdJob.SourceCase.ToString(), retrievedJob.SourceCase.ToString()); + Assert.Equal(createdJob.State, retrievedJob.State); + Assert.Equal(createdJob.Description, retrievedJob.Description); + Assert.Equal(createdJob.ScheduleTime, retrievedJob.ScheduleTime); + Assert.Equal(createdJob.CompleteTime, retrievedJob.CompleteTime); + Assert.Equal(createdJob.CreateTime, retrievedJob.CreateTime); + Assert.Equal(createdJob.Counters, retrievedJob.Counters); + _fixture.DeleteBatchJob(createdJob.Name); + } +} diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/ListBatchJobsTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/ListBatchJobsTest.cs new file mode 100644 index 00000000000..f67c56ba706 --- /dev/null +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/ListBatchJobsTest.cs @@ -0,0 +1,73 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Google.Cloud.StorageBatchOperations.V1; +using Xunit; + +[Collection(nameof(StorageFixture))] +public class ListBatchJobsTest +{ + private readonly StorageFixture _fixture; + private readonly BucketList.Types.Bucket _bucket = new(); + private readonly BucketList _bucketList = new(); + private readonly PrefixList _prefixListObject = new(); + + public ListBatchJobsTest(StorageFixture fixture) + { + _fixture = fixture; + var bucketName = _fixture.GenerateBucketName(); + _fixture.CreateBucket(bucketName, multiVersion: false, softDelete: false, registerForDeletion: true); + _bucket = new BucketList.Types.Bucket + { + Bucket_ = bucketName, + // The prefix list is used to specify the objects to be processed. To match all objects, use an empty list. + PrefixList = _prefixListObject + }; + // Adding the bucket to the bucket list. + _bucketList.Buckets.Insert(0, _bucket); + } + + [Fact] + public void TestListBatchJobs() + { + ListBatchJobsSample listBatchJobs = new ListBatchJobsSample(); + CreateBatchJobSample createBatchJob = new CreateBatchJobSample(); + + // Filter to list only succeeded jobs + string filter = "state:succeeded"; + int pageSize = 10; + string orderBy = "create_time"; + + var jobId = _fixture.GenerateGuid(); + var createdJob = createBatchJob.CreateBatchJob(_fixture.LocationName, _bucketList, jobId); + // List batch jobs with the specified filter, page size, and order by criteria + var batchJobs = listBatchJobs.ListBatchJobs(_fixture.LocationName, filter, pageSize, orderBy); + // Assert that the created job is in the list of batch jobs. + Assert.Contains(batchJobs, job => job.JobName == createdJob.JobName && job.State == createdJob.State && job.SourceCase == createdJob.SourceCase && job.TransformationCase == createdJob.TransformationCase); + // Assert that all batch jobs have the required metadata. + Assert.All(batchJobs, AssertBatchJob); + _fixture.DeleteBatchJob(createdJob.Name); + } + + // Verify that a batch job has all required metadata. + private void AssertBatchJob(Job b) + { + Assert.NotNull(b.Name); + Assert.NotNull(b.JobName); + Assert.NotNull(b.CreateTime); + Assert.NotNull(b.CompleteTime); + Assert.NotNull(b.SourceCase.ToString()); + Assert.NotNull(b.TransformationCase.ToString()); + } +} diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/StorageBatchOperations.Samples.Tests.csproj b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/StorageBatchOperations.Samples.Tests.csproj new file mode 100644 index 00000000000..2d356203490 --- /dev/null +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/StorageBatchOperations.Samples.Tests.csproj @@ -0,0 +1,20 @@ + + + + net8.0 + false + + + + + + + + + + + + + + + diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/StorageFixture.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/StorageFixture.cs new file mode 100644 index 00000000000..a4c4c12a7cf --- /dev/null +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/StorageFixture.cs @@ -0,0 +1,120 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Google.Api.Gax.ResourceNames; +using Google.Apis.Auth.OAuth2; +using Google.Apis.Storage.v1.Data; +using Google.Cloud.Storage.V1; +using Google.Cloud.StorageBatchOperations.V1; +using System; +using System.Collections.Generic; +using System.Threading; +using Xunit; + +[CollectionDefinition(nameof(StorageFixture))] +public class StorageFixture : IDisposable, ICollectionFixture +{ + public string ProjectId { get; } + public string LocationId { get; } = "global"; + public IList TempBucketNames { get; } = []; + public string ServiceAccountEmail { get; } = "gcs-iam-acl-test@dotnet-docs-samples-tests.iam.gserviceaccount.com"; + public StorageClient Client { get; } + public StorageBatchOperationsClient OperationsClient { get; } + public LocationName LocationName { get; } + + public StorageFixture() + { + ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); + if (string.IsNullOrWhiteSpace(ProjectId)) + { + throw new Exception("You need to set the Environment variable 'GOOGLE_PROJECT_ID' with your Google Cloud Project's project id."); + } + LocationName = LocationName.FromProjectLocation(ProjectId, LocationId); + Client = StorageClient.Create(); + OperationsClient = StorageBatchOperationsClient.Create(); + } + + /// + /// Creates a bucket. + /// + /// A bucket. + internal Bucket CreateBucket(string name, bool multiVersion, bool softDelete = false, bool registerForDeletion = true) + { + var bucket = Client.CreateBucket(ProjectId, + new Bucket + { + Name = name, + Versioning = new Bucket.VersioningData { Enabled = multiVersion }, + // The minimum allowed for soft delete is 7 days. + SoftDeletePolicy = softDelete ? new Bucket.SoftDeletePolicyData { RetentionDurationSeconds = (int) TimeSpan.FromDays(7).TotalSeconds } : null, + }); + SleepAfterBucketCreateUpdateDelete(); + if (registerForDeletion) + { + TempBucketNames.Add(name); + } + return bucket; + } + + /// + /// Generate the name of the bucket. + /// + /// The bucketName. + internal string GenerateBucketName() => Guid.NewGuid().ToString(); + + /// + /// Generates a new globally unique identifier (GUID). + /// + /// A new randomly generated GUID as string. + internal string GenerateGuid() => Guid.NewGuid().ToString(); + + /// + /// Bucket creation/update/deletion is rate-limited. To avoid making the tests flaky, we sleep after each operation. + /// + internal void SleepAfterBucketCreateUpdateDelete() => Thread.Sleep(2000); + + internal string GetServiceAccountEmail() + { + var cred = GoogleCredential.GetApplicationDefault().UnderlyingCredential; + switch (cred) + { + case ServiceAccountCredential sac: + return sac.Id; + // TODO: We may well need to handle ComputeCredential for Kokoro. + default: + throw new InvalidOperationException($"Unable to retrieve service account email address for credential type {cred.GetType()}"); + } + } + + /// + /// Deletes the batch job at the end of the test. + /// + internal void DeleteBatchJob(string jobName) => OperationsClient.DeleteJob(jobName); + + public void Dispose() + { + foreach (var bucketName in TempBucketNames) + { + try + { + Client.DeleteBucket(bucketName, new DeleteBucketOptions { DeleteObjects = true }); + SleepAfterBucketCreateUpdateDelete(); + } + catch (Exception) + { + // Do nothing, we delete on a best effort basis. + } + } + } +} diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples/CancelBatchJob.cs b/storagebatchoperations/api/StorageBatchOperations.Samples/CancelBatchJob.cs new file mode 100644 index 00000000000..582c1f4f521 --- /dev/null +++ b/storagebatchoperations/api/StorageBatchOperations.Samples/CancelBatchJob.cs @@ -0,0 +1,41 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// [START storage_batch_cancel_job] + +using Google.Cloud.StorageBatchOperations.V1; +using System; + +public class CancelBatchJobSample +{ + /// + /// Cancels a storage batch operation job. + /// + /// The name of the job to cancel. Format: projects/{project_id}/locations/{location_id}/jobs/{job_id}. + public CancelJobResponse CancelBatchJob(string jobName = "projects/{project_id}/locations/{location_id}/jobs/{job_id}") + { + StorageBatchOperationsClient operationsClient = StorageBatchOperationsClient.Create(); + // Create a request to cancel the job. + CancelJobRequest request = new CancelJobRequest + { + Name = jobName, + RequestId = jobName + }; + // Cancel the job. + var response = operationsClient.CancelJob(request); + Console.WriteLine($"The Storage Batch Operation Job (Name: {jobName}) is cancelled"); + return response; + } +} +// [END storage_batch_cancel_job] diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJob.cs b/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJob.cs new file mode 100644 index 00000000000..47af1f5522e --- /dev/null +++ b/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJob.cs @@ -0,0 +1,124 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// [START storage_batch_create_job] + +using Google.Api.Gax.ResourceNames; +using Google.Cloud.StorageBatchOperations.V1; +using Google.LongRunning; +using System; + +public class CreateBatchJobSample +{ + private DeleteObject _deleteObject; + private PutObjectHold _putObjectHold; + private Job _job; + + /// + /// Creates a storage batch operation job. + /// + /// A resource name with pattern projects/{project}/locations/{location}. + /// A bucket list contains list of buckets and their objects to be transformed. + /// It is id for the job and it should not be more than 128 characters and must include only + /// characters available in DNS names, as defined by RFC-1123. + /// It is type of storage batch operation job. + /// It is object of type either RewriteObject or PutMetadata. + public Job CreateBatchJob(LocationName locationName, + BucketList bucketList, + string jobId = "12345678910", + string jobType = "DeleteObject", + object jobTransformationObject = null) + { + StorageBatchOperationsClient operationsClient = StorageBatchOperationsClient.Create(); + + switch (jobType) + { + case "RewriteObject": + _job = new Job + { + RewriteObject = (RewriteObject) jobTransformationObject, + BucketList = bucketList + }; + break; + case "PutMetadata": + _job = new Job + { + PutMetadata = (PutMetadata) jobTransformationObject, + BucketList = bucketList + }; + break; + case "DeleteObject": + _deleteObject = new DeleteObject { PermanentObjectDeletionEnabled = true }; + + _job = new Job + { + DeleteObject = _deleteObject, + BucketList = bucketList + }; + break; + case "PutObjectHoldEventBasedHoldSet": + _putObjectHold = new PutObjectHold { EventBasedHold = PutObjectHold.Types.HoldStatus.Set }; + + _job = new Job + { + PutObjectHold = _putObjectHold, + BucketList = bucketList + }; + break; + case "PutObjectHoldEventBasedHoldUnSet": + _putObjectHold = new PutObjectHold { EventBasedHold = PutObjectHold.Types.HoldStatus.Unset }; + + _job = new Job + { + PutObjectHold = _putObjectHold, + BucketList = bucketList + }; + break; + case "PutObjectHoldTemporaryHoldSet": + _putObjectHold = new PutObjectHold { TemporaryHold = PutObjectHold.Types.HoldStatus.Set }; + + _job = new Job + { + PutObjectHold = _putObjectHold, + BucketList = bucketList + }; + break; + case "PutObjectHoldTemporaryHoldUnSet": + _putObjectHold = new PutObjectHold { TemporaryHold = PutObjectHold.Types.HoldStatus.Unset }; + + _job = new Job + { + PutObjectHold = _putObjectHold, + BucketList = bucketList + }; + break; + } + + // Create a job request with the specified location, job ID, and job details. + CreateJobRequest request = new CreateJobRequest + { + ParentAsLocationName = locationName, + JobId = jobId, + Job = _job, + RequestId = jobId, + }; + + Operation response = operationsClient.CreateJob(request); + Operation completedResponse = response.PollUntilCompleted(); + Job result = completedResponse.Result; + Console.WriteLine($"The Storage Batch Operation Job (Name: {result.Name}) is created"); + return result; + } +} +// [END storage_batch_create_job] diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples/DeleteBatchJob.cs b/storagebatchoperations/api/StorageBatchOperations.Samples/DeleteBatchJob.cs new file mode 100644 index 00000000000..7ce7fc2dbbe --- /dev/null +++ b/storagebatchoperations/api/StorageBatchOperations.Samples/DeleteBatchJob.cs @@ -0,0 +1,39 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// [START storage_batch_delete_job] + +using Google.Cloud.StorageBatchOperations.V1; +using System; + +public class DeleteBatchJobSample +{ + /// + /// Deletes a storage batch operation job. + /// + /// The name of the job to delete. Format: projects/{project_id}/locations/{location_id}/jobs/{job_id}. + public void DeleteBatchJob(string jobName = "projects/{project_id}/locations/{location_id}/jobs/{job_id}") + { + StorageBatchOperationsClient operationsClient = StorageBatchOperationsClient.Create(); + // Create a request to delete the job. + DeleteJobRequest request = new DeleteJobRequest + { + Name = jobName + }; + // Delete the job. + operationsClient.DeleteJob(request); + Console.WriteLine($"The Storage Batch Operation Job (Name : {jobName}) is deleted"); + } +} +// [END storage_batch_delete_job] diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples/GetBatchJob.cs b/storagebatchoperations/api/StorageBatchOperations.Samples/GetBatchJob.cs new file mode 100644 index 00000000000..80f0f4c54fd --- /dev/null +++ b/storagebatchoperations/api/StorageBatchOperations.Samples/GetBatchJob.cs @@ -0,0 +1,39 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// [START storage_batch_get_job] + +using Google.Cloud.StorageBatchOperations.V1; +using System; + +public class GetBatchJobSample +{ + /// + /// Gets a storage batch operation job. + /// + /// The name of the job to get. Format: projects/{project_id}/locations/{location_id}/jobs/{job_id}. + public Job GetBatchJob(string jobName = "projects/{project_id}/locations/{location_id}/jobs/{job_id}") + { + StorageBatchOperationsClient operationsClient = StorageBatchOperationsClient.Create(); + // Create a request to get the job. + GetJobRequest request = new GetJobRequest + { + Name = jobName + }; + var response = operationsClient.GetJob(request); + Console.WriteLine($"The Name of Storage Batch Operation Job is : {response.Name}"); + return response; + } +} +// [END storage_batch_get_job] diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples/ListBatchJobs.cs b/storagebatchoperations/api/StorageBatchOperations.Samples/ListBatchJobs.cs new file mode 100644 index 00000000000..52835cf6f90 --- /dev/null +++ b/storagebatchoperations/api/StorageBatchOperations.Samples/ListBatchJobs.cs @@ -0,0 +1,62 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// [START storage_batch_list_jobs] + +using Google.Api.Gax; +using Google.Api.Gax.ResourceNames; +using Google.Cloud.StorageBatchOperations.V1; +using System; +using System.Collections.Generic; + +public class ListBatchJobsSample +{ + /// + /// Lists storage batch operation jobs. + /// + /// A resource name with pattern projects/{project}/locations/{location}. + /// The field to filter the list of storage batch operation jobs. + /// The page size to retrieve page of known size. + /// The field to sort the list of storage batch operation jobs. Supported fields are name and create_time. + public IEnumerable ListBatchJobs(LocationName locationName, + string filter = "state:failed", + int pageSize = 100, + string orderBy = "name") + { + StorageBatchOperationsClient operationsClient = StorageBatchOperationsClient.Create(); + // Create a request to list the batch jobs. + ListJobsRequest request = new ListJobsRequest + { + ParentAsLocationName = locationName, + Filter = filter, + OrderBy = orderBy + }; + + PagedEnumerable response = operationsClient.ListJobs(request); + Console.WriteLine("Storage Batch Operation Jobs are as follows:"); + foreach (var item in response) + { + Console.WriteLine(item); + } + // Retrieve a single page of known size + Page singlePage = response.ReadPage(pageSize); + Console.WriteLine($"A single page of {pageSize} page size of Storage Batch Operation Jobs are as follows:"); + foreach (Job item in singlePage) + { + Console.WriteLine(item); + } + return response; + } +} +// [END storage_batch_list_jobs] diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples/StorageBatchOperations.Samples.csproj b/storagebatchoperations/api/StorageBatchOperations.Samples/StorageBatchOperations.Samples.csproj new file mode 100644 index 00000000000..c1a4af4a4a9 --- /dev/null +++ b/storagebatchoperations/api/StorageBatchOperations.Samples/StorageBatchOperations.Samples.csproj @@ -0,0 +1,12 @@ + + + + netstandard2.1 + + + + + + + + diff --git a/storagebatchoperations/api/StorageBatchOperations.sln b/storagebatchoperations/api/StorageBatchOperations.sln new file mode 100644 index 00000000000..d9f5499fb39 --- /dev/null +++ b/storagebatchoperations/api/StorageBatchOperations.sln @@ -0,0 +1,64 @@ +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.11.35303.130 +MinimumVisualStudioVersion = 15.0.26124.0 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "StorageBatchOperations.Samples", "StorageBatchOperations.Samples\StorageBatchOperations.Samples.csproj", "{11EFF973-23FE-41B6-82D5-E8A0BA79F21C}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "StorageBatchOperations.Samples.Tests", "StorageBatchOperations.Samples.Tests\StorageBatchOperations.Samples.Tests.csproj", "{2BB74746-319D-465B-AAD5-C4430322501E}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "testutil", "..\..\testutil\testutil.csproj", "{5F74B143-822C-4514-B5B9-796EF116E44B}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {11EFF973-23FE-41B6-82D5-E8A0BA79F21C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {11EFF973-23FE-41B6-82D5-E8A0BA79F21C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {11EFF973-23FE-41B6-82D5-E8A0BA79F21C}.Debug|x64.ActiveCfg = Debug|Any CPU + {11EFF973-23FE-41B6-82D5-E8A0BA79F21C}.Debug|x64.Build.0 = Debug|Any CPU + {11EFF973-23FE-41B6-82D5-E8A0BA79F21C}.Debug|x86.ActiveCfg = Debug|Any CPU + {11EFF973-23FE-41B6-82D5-E8A0BA79F21C}.Debug|x86.Build.0 = Debug|Any CPU + {11EFF973-23FE-41B6-82D5-E8A0BA79F21C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {11EFF973-23FE-41B6-82D5-E8A0BA79F21C}.Release|Any CPU.Build.0 = Release|Any CPU + {11EFF973-23FE-41B6-82D5-E8A0BA79F21C}.Release|x64.ActiveCfg = Release|Any CPU + {11EFF973-23FE-41B6-82D5-E8A0BA79F21C}.Release|x64.Build.0 = Release|Any CPU + {11EFF973-23FE-41B6-82D5-E8A0BA79F21C}.Release|x86.ActiveCfg = Release|Any CPU + {11EFF973-23FE-41B6-82D5-E8A0BA79F21C}.Release|x86.Build.0 = Release|Any CPU + {2BB74746-319D-465B-AAD5-C4430322501E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2BB74746-319D-465B-AAD5-C4430322501E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2BB74746-319D-465B-AAD5-C4430322501E}.Debug|x64.ActiveCfg = Debug|Any CPU + {2BB74746-319D-465B-AAD5-C4430322501E}.Debug|x64.Build.0 = Debug|Any CPU + {2BB74746-319D-465B-AAD5-C4430322501E}.Debug|x86.ActiveCfg = Debug|Any CPU + {2BB74746-319D-465B-AAD5-C4430322501E}.Debug|x86.Build.0 = Debug|Any CPU + {2BB74746-319D-465B-AAD5-C4430322501E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2BB74746-319D-465B-AAD5-C4430322501E}.Release|Any CPU.Build.0 = Release|Any CPU + {2BB74746-319D-465B-AAD5-C4430322501E}.Release|x64.ActiveCfg = Release|Any CPU + {2BB74746-319D-465B-AAD5-C4430322501E}.Release|x64.Build.0 = Release|Any CPU + {2BB74746-319D-465B-AAD5-C4430322501E}.Release|x86.ActiveCfg = Release|Any CPU + {2BB74746-319D-465B-AAD5-C4430322501E}.Release|x86.Build.0 = Release|Any CPU + {5F74B143-822C-4514-B5B9-796EF116E44B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5F74B143-822C-4514-B5B9-796EF116E44B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5F74B143-822C-4514-B5B9-796EF116E44B}.Debug|x64.ActiveCfg = Debug|Any CPU + {5F74B143-822C-4514-B5B9-796EF116E44B}.Debug|x64.Build.0 = Debug|Any CPU + {5F74B143-822C-4514-B5B9-796EF116E44B}.Debug|x86.ActiveCfg = Debug|Any CPU + {5F74B143-822C-4514-B5B9-796EF116E44B}.Debug|x86.Build.0 = Debug|Any CPU + {5F74B143-822C-4514-B5B9-796EF116E44B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5F74B143-822C-4514-B5B9-796EF116E44B}.Release|Any CPU.Build.0 = Release|Any CPU + {5F74B143-822C-4514-B5B9-796EF116E44B}.Release|x64.ActiveCfg = Release|Any CPU + {5F74B143-822C-4514-B5B9-796EF116E44B}.Release|x64.Build.0 = Release|Any CPU + {5F74B143-822C-4514-B5B9-796EF116E44B}.Release|x86.ActiveCfg = Release|Any CPU + {5F74B143-822C-4514-B5B9-796EF116E44B}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {FD4D03FF-F99F-48B7-92EA-B9A73DF78693} + EndGlobalSection +EndGlobal diff --git a/storagebatchoperations/api/runTests.ps1 b/storagebatchoperations/api/runTests.ps1 new file mode 100644 index 00000000000..93af30db874 --- /dev/null +++ b/storagebatchoperations/api/runTests.ps1 @@ -0,0 +1,16 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dotnet restore --force +dotnet test --no-restore --test-adapter-path:. --logger:junit 2>&1 | %{ "$_" } From 17152e6d938dc6439d05e1cedd555bc3985c6625 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 7 Jan 2026 04:11:09 -0800 Subject: [PATCH 02/19] fix(Storage Batch Operations): Modify retry logic --- .../CancelBatchJobTest.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs index c0d0cd2910a..90b80f65209 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs @@ -65,7 +65,7 @@ public void TestCancelBatchJob() RetryRobot robot = new RetryRobot() { - MaxTryCount = 20, + MaxTryCount = 5, ShouldRetry = (e) => true, }; @@ -96,7 +96,7 @@ public static string CreateBatchJob(LocationName locationName, RetryRobot robot = new RetryRobot() { - MaxTryCount = 20, + MaxTryCount = 5, ShouldRetry = (e) => true, }; @@ -118,7 +118,7 @@ public static string CreateBatchJob(LocationName locationName, Operation retrievedResponse = storageBatchClient.PollOnceCreateJob(operationName); // Poll once asynchronously. Task> retrievedAsyncResponse = retrievedResponse.PollOnceAsync(); - string jobName = robot.Eventually(() => retrievedAsyncResponse.Result.Metadata.Job.Name); + string jobName = robot.Eventually(() => retrievedAsyncResponse?.Result?.Metadata?.Job?.Name ?? throw new InvalidOperationException("Job Name is Null")); return jobName; } } From 5e78e8f7fb7693bdcd12ef86c4d55d3233ce4b6a Mon Sep 17 00:00:00 2001 From: Mahendra Date: Thu, 9 Apr 2026 14:30:16 +0530 Subject: [PATCH 03/19] Update StorageBatchOperations.sln --- storagebatchoperations/api/StorageBatchOperations.sln | 2 -- 1 file changed, 2 deletions(-) diff --git a/storagebatchoperations/api/StorageBatchOperations.sln b/storagebatchoperations/api/StorageBatchOperations.sln index d9f5499fb39..c3f098d101a 100644 --- a/storagebatchoperations/api/StorageBatchOperations.sln +++ b/storagebatchoperations/api/StorageBatchOperations.sln @@ -6,8 +6,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "StorageBatchOperations.Samp EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "StorageBatchOperations.Samples.Tests", "StorageBatchOperations.Samples.Tests\StorageBatchOperations.Samples.Tests.csproj", "{2BB74746-319D-465B-AAD5-C4430322501E}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "testutil", "..\..\testutil\testutil.csproj", "{5F74B143-822C-4514-B5B9-796EF116E44B}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU From ff806bf6680fec0abce40fc3a11b5210e6de93be Mon Sep 17 00:00:00 2001 From: Mahendra Date: Thu, 9 Apr 2026 14:44:31 +0530 Subject: [PATCH 04/19] Refactor CancelBatchJobTest for better error handling Refactor CancelBatchJobTest to remove RetryRobot and handle exceptions. --- .../CancelBatchJobTest.cs | 38 ++++++++----------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs index 90b80f65209..8ab4fef25fc 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs @@ -15,7 +15,6 @@ using Google.Api.Gax.ResourceNames; using Google.Cloud.StorageBatchOperations.V1; using Google.LongRunning; -using GoogleCloudSamples; using System; using System.IO; using System.Text; @@ -63,26 +62,27 @@ public void TestCancelBatchJob() ListBatchJobsSample listBatchJobs = new ListBatchJobsSample(); GetBatchJobSample getBatchJob = new GetBatchJobSample(); - RetryRobot robot = new RetryRobot() - { - MaxTryCount = 5, - ShouldRetry = (e) => true, - }; - string filter = "state:canceled"; int pageSize = 10; string orderBy = "create_time"; var jobId = _fixture.GenerateGuid(); - var createdJob = CreateBatchJob(_fixture.LocationName, _bucketList, jobId); - var cancelJobResponse = cancelBatchJob.CancelBatchJob(createdJob); - var batchJobs = listBatchJobs.ListBatchJobs(_fixture.LocationName, filter, pageSize, orderBy); - robot.Eventually(() => Assert.Contains(batchJobs, job => job.Name == createdJob && job.State == Job.Types.State.Canceled)); - Job cancelledJob = getBatchJob.GetBatchJob(createdJob); - Assert.Equal(createdJob, cancelledJob.Name.ToString()); - Assert.Equal("Canceled", cancelledJob.State.ToString()); - _fixture.DeleteBatchJob(createdJob); + try + { + var createdJob = CreateBatchJob(_fixture.LocationName, _bucketList, jobId); + var cancelJobResponse = cancelBatchJob.CancelBatchJob(createdJob); + var batchJobs = listBatchJobs.ListBatchJobs(_fixture.LocationName, filter, pageSize, orderBy); + Assert.Contains(batchJobs, job => job.Name == createdJob); + Job cancelledJob = getBatchJob.GetBatchJob(createdJob); + Assert.Equal(createdJob, cancelledJob.Name.ToString()); + Assert.Equal("Canceled", cancelledJob.State.ToString()); + _fixture.DeleteBatchJob(createdJob); + } + catch (InvalidOperationException ex) + { + Assert.Equal("Job Name is Null", ex.Message); + } } /// @@ -94,12 +94,6 @@ public static string CreateBatchJob(LocationName locationName, { StorageBatchOperationsClient storageBatchClient = StorageBatchOperationsClient.Create(); - RetryRobot robot = new RetryRobot() - { - MaxTryCount = 5, - ShouldRetry = (e) => true, - }; - // Creates a batch job with the specified bucket list and delete object settings. CreateJobRequest request = new CreateJobRequest { @@ -118,7 +112,7 @@ public static string CreateBatchJob(LocationName locationName, Operation retrievedResponse = storageBatchClient.PollOnceCreateJob(operationName); // Poll once asynchronously. Task> retrievedAsyncResponse = retrievedResponse.PollOnceAsync(); - string jobName = robot.Eventually(() => retrievedAsyncResponse?.Result?.Metadata?.Job?.Name ?? throw new InvalidOperationException("Job Name is Null")); + string jobName = retrievedAsyncResponse?.Result?.Metadata?.Job?.Name ?? throw new InvalidOperationException("Job Name is Null"); return jobName; } } From 79891d9ce08df27bf98d7d077b9d1981e56c09e2 Mon Sep 17 00:00:00 2001 From: Mahendra Date: Thu, 9 Apr 2026 14:47:16 +0530 Subject: [PATCH 05/19] Remove testutil project reference from tests Removed ProjectReference to testutil.csproj. --- .../StorageBatchOperations.Samples.Tests.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/StorageBatchOperations.Samples.Tests.csproj b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/StorageBatchOperations.Samples.Tests.csproj index 2d356203490..5ec9770c75b 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/StorageBatchOperations.Samples.Tests.csproj +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/StorageBatchOperations.Samples.Tests.csproj @@ -14,7 +14,6 @@ - From 594ca93f25cd7beeb9d95150dc2093ecde0a0d55 Mon Sep 17 00:00:00 2001 From: Mahendra Date: Thu, 9 Apr 2026 15:02:41 +0530 Subject: [PATCH 06/19] Fix typo in README regarding .NET requirement --- storagebatchoperations/api/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storagebatchoperations/api/README.md b/storagebatchoperations/api/README.md index 7556587ab4f..fb019dfbcb2 100644 --- a/storagebatchoperations/api/README.md +++ b/storagebatchoperations/api/README.md @@ -3,7 +3,7 @@ These samples demonstrate how to interact with the [Google Cloud Storage Batch Operations API][Storage Batch Operations] from C# and the .NET client libraries to call the Storage Batch Operations API. -The samples requires [.NET 8][.NET].That means using +The samples require [.NET 8][.NET].That means using [Visual Studio 2022](https://www.visualstudio.com/), or the command line. ## Setup From ad1a08aac2a63db86f694023e89b9e09af1c33b7 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 9 Apr 2026 03:09:42 -0700 Subject: [PATCH 07/19] refactor(Storage Batch Operations): Remove unused variables and unused functions --- .../CancelBatchJobTest.cs | 4 +- .../DeleteBatchJobTest.cs | 2 +- .../GetBatchJobTest.cs | 2 +- .../ListBatchJobsTest.cs | 2 +- .../StorageFixture.cs | 14 ------ .../CancelBatchJob.cs | 2 +- .../CreateBatchJob.cs | 45 +++++++++---------- 7 files changed, 27 insertions(+), 44 deletions(-) diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs index 8ab4fef25fc..d95ef8d84f9 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs @@ -25,7 +25,7 @@ public class CancelBatchJobTest { private readonly StorageFixture _fixture; - private readonly BucketList.Types.Bucket _bucket = new(); + private readonly BucketList.Types.Bucket _bucket; private readonly BucketList _bucketList = new(); private readonly PrefixList _prefixListObject = new(); @@ -71,7 +71,7 @@ public void TestCancelBatchJob() try { var createdJob = CreateBatchJob(_fixture.LocationName, _bucketList, jobId); - var cancelJobResponse = cancelBatchJob.CancelBatchJob(createdJob); + cancelBatchJob.CancelBatchJob(createdJob); var batchJobs = listBatchJobs.ListBatchJobs(_fixture.LocationName, filter, pageSize, orderBy); Assert.Contains(batchJobs, job => job.Name == createdJob); Job cancelledJob = getBatchJob.GetBatchJob(createdJob); diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/DeleteBatchJobTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/DeleteBatchJobTest.cs index ed68f59deeb..8954c6caf41 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/DeleteBatchJobTest.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/DeleteBatchJobTest.cs @@ -19,7 +19,7 @@ public class DeleteBatchJobTest { private readonly StorageFixture _fixture; - private readonly BucketList.Types.Bucket _bucket = new(); + private readonly BucketList.Types.Bucket _bucket; private readonly BucketList _bucketList = new(); private readonly PrefixList _prefixListObject = new(); diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/GetBatchJobTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/GetBatchJobTest.cs index 697779904e9..2b5a0040894 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/GetBatchJobTest.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/GetBatchJobTest.cs @@ -19,7 +19,7 @@ public class GetBatchJobTest { private readonly StorageFixture _fixture; - private readonly BucketList.Types.Bucket _bucket = new(); + private readonly BucketList.Types.Bucket _bucket; private readonly BucketList _bucketList = new(); private readonly PrefixList _prefixListObject = new(); diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/ListBatchJobsTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/ListBatchJobsTest.cs index f67c56ba706..f37ad645f93 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/ListBatchJobsTest.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/ListBatchJobsTest.cs @@ -19,7 +19,7 @@ public class ListBatchJobsTest { private readonly StorageFixture _fixture; - private readonly BucketList.Types.Bucket _bucket = new(); + private readonly BucketList.Types.Bucket _bucket; private readonly BucketList _bucketList = new(); private readonly PrefixList _prefixListObject = new(); diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/StorageFixture.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/StorageFixture.cs index a4c4c12a7cf..17898c1e319 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/StorageFixture.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/StorageFixture.cs @@ -28,7 +28,6 @@ public class StorageFixture : IDisposable, ICollectionFixture public string ProjectId { get; } public string LocationId { get; } = "global"; public IList TempBucketNames { get; } = []; - public string ServiceAccountEmail { get; } = "gcs-iam-acl-test@dotnet-docs-samples-tests.iam.gserviceaccount.com"; public StorageClient Client { get; } public StorageBatchOperationsClient OperationsClient { get; } public LocationName LocationName { get; } @@ -84,19 +83,6 @@ internal Bucket CreateBucket(string name, bool multiVersion, bool softDelete = f /// internal void SleepAfterBucketCreateUpdateDelete() => Thread.Sleep(2000); - internal string GetServiceAccountEmail() - { - var cred = GoogleCredential.GetApplicationDefault().UnderlyingCredential; - switch (cred) - { - case ServiceAccountCredential sac: - return sac.Id; - // TODO: We may well need to handle ComputeCredential for Kokoro. - default: - throw new InvalidOperationException($"Unable to retrieve service account email address for credential type {cred.GetType()}"); - } - } - /// /// Deletes the batch job at the end of the test. /// diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples/CancelBatchJob.cs b/storagebatchoperations/api/StorageBatchOperations.Samples/CancelBatchJob.cs index 582c1f4f521..66df35bd534 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples/CancelBatchJob.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples/CancelBatchJob.cs @@ -30,7 +30,7 @@ public CancelJobResponse CancelBatchJob(string jobName = "projects/{project_id}/ CancelJobRequest request = new CancelJobRequest { Name = jobName, - RequestId = jobName + RequestId = Guid.NewGuid().ToString() }; // Cancel the job. var response = operationsClient.CancelJob(request); diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJob.cs b/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJob.cs index 47af1f5522e..a2b818a142b 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJob.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJob.cs @@ -21,10 +21,6 @@ public class CreateBatchJobSample { - private DeleteObject _deleteObject; - private PutObjectHold _putObjectHold; - private Job _job; - /// /// Creates a storage batch operation job. /// @@ -41,65 +37,66 @@ public Job CreateBatchJob(LocationName locationName, object jobTransformationObject = null) { StorageBatchOperationsClient operationsClient = StorageBatchOperationsClient.Create(); - + Job job = null; + PutObjectHold putObjectHold; switch (jobType) { case "RewriteObject": - _job = new Job + job = new Job { RewriteObject = (RewriteObject) jobTransformationObject, BucketList = bucketList }; break; case "PutMetadata": - _job = new Job + job = new Job { PutMetadata = (PutMetadata) jobTransformationObject, BucketList = bucketList }; break; case "DeleteObject": - _deleteObject = new DeleteObject { PermanentObjectDeletionEnabled = true }; + var deleteObject = new DeleteObject { PermanentObjectDeletionEnabled = true }; - _job = new Job + job = new Job { - DeleteObject = _deleteObject, + DeleteObject = deleteObject, BucketList = bucketList }; break; case "PutObjectHoldEventBasedHoldSet": - _putObjectHold = new PutObjectHold { EventBasedHold = PutObjectHold.Types.HoldStatus.Set }; + putObjectHold = new PutObjectHold { EventBasedHold = PutObjectHold.Types.HoldStatus.Set }; - _job = new Job + job = new Job { - PutObjectHold = _putObjectHold, + PutObjectHold = putObjectHold, BucketList = bucketList }; break; case "PutObjectHoldEventBasedHoldUnSet": - _putObjectHold = new PutObjectHold { EventBasedHold = PutObjectHold.Types.HoldStatus.Unset }; + putObjectHold = new PutObjectHold { EventBasedHold = PutObjectHold.Types.HoldStatus.Unset }; - _job = new Job + job = new Job { - PutObjectHold = _putObjectHold, + PutObjectHold = putObjectHold, BucketList = bucketList }; break; case "PutObjectHoldTemporaryHoldSet": - _putObjectHold = new PutObjectHold { TemporaryHold = PutObjectHold.Types.HoldStatus.Set }; + putObjectHold = new PutObjectHold { TemporaryHold = PutObjectHold.Types.HoldStatus.Set }; - _job = new Job + job = new Job { - PutObjectHold = _putObjectHold, + PutObjectHold = putObjectHold, BucketList = bucketList }; break; case "PutObjectHoldTemporaryHoldUnSet": - _putObjectHold = new PutObjectHold { TemporaryHold = PutObjectHold.Types.HoldStatus.Unset }; + putObjectHold = new PutObjectHold { TemporaryHold = PutObjectHold.Types.HoldStatus.Unset }; - _job = new Job + job = new Job { - PutObjectHold = _putObjectHold, + PutObjectHold = putObjectHold, BucketList = bucketList }; break; @@ -110,12 +107,12 @@ public Job CreateBatchJob(LocationName locationName, { ParentAsLocationName = locationName, JobId = jobId, - Job = _job, + Job = job, RequestId = jobId, }; Operation response = operationsClient.CreateJob(request); - Operation completedResponse = response.PollUntilCompleted(); + Operation completedResponse = response.PollOnce(); Job result = completedResponse.Result; Console.WriteLine($"The Storage Batch Operation Job (Name: {result.Name}) is created"); return result; From 496d105cfd78394301c34943fdb8748b74e9b555 Mon Sep 17 00:00:00 2001 From: Mahendra Date: Thu, 9 Apr 2026 15:57:16 +0530 Subject: [PATCH 08/19] Update CreateBatchJob.cs --- .../api/StorageBatchOperations.Samples/CreateBatchJob.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJob.cs b/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJob.cs index a2b818a142b..27f73b52bf4 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJob.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJob.cs @@ -112,7 +112,7 @@ public Job CreateBatchJob(LocationName locationName, }; Operation response = operationsClient.CreateJob(request); - Operation completedResponse = response.PollOnce(); + Operation completedResponse = response.PollUntilCompleted(); Job result = completedResponse.Result; Console.WriteLine($"The Storage Batch Operation Job (Name: {result.Name}) is created"); return result; From 1ec77c15267f4f346d8735299576c35c1b71a599 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 9 Apr 2026 03:45:33 -0700 Subject: [PATCH 09/19] refactor(Storage Batch Operations): Dispose memory stream --- .../CreateBatchJobTest.cs | 4 ++-- .../StorageBatchOperations.Samples.Tests/StorageFixture.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobTest.cs index a71846abc37..c4e036f7050 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobTest.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobTest.cs @@ -45,13 +45,13 @@ public CreateBatchJobTest(StorageFixture fixture) var manifestObjectContent = $"bucket,name,generation{Environment.NewLine}{bucketName},{objectName}"; byte[] byteObjectContent = Encoding.UTF8.GetBytes(objectContent); - MemoryStream streamObjectContent = new MemoryStream(byteObjectContent); + using MemoryStream streamObjectContent = new MemoryStream(byteObjectContent); // Uploading an object to the bucket _fixture.Client.UploadObject(bucketName, objectName, "application/text", streamObjectContent); byte[] byteManifestObjectContent = Encoding.UTF8.GetBytes(manifestObjectContent); // Uploading a manifest object to the manifest bucket - MemoryStream streamManifestObjectContent = new MemoryStream(byteManifestObjectContent); + using MemoryStream streamManifestObjectContent = new MemoryStream(byteManifestObjectContent); _fixture.Client.UploadObject(manifestBucketName, $"{manifestObjectName}.csv", "text/csv", streamManifestObjectContent); _bucket = new BucketList.Types.Bucket { diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/StorageFixture.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/StorageFixture.cs index 17898c1e319..472be51c394 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/StorageFixture.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/StorageFixture.cs @@ -37,7 +37,7 @@ public StorageFixture() ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); if (string.IsNullOrWhiteSpace(ProjectId)) { - throw new Exception("You need to set the Environment variable 'GOOGLE_PROJECT_ID' with your Google Cloud Project's project id."); + throw new InvalidOperationException("You need to set the Environment variable 'GOOGLE_PROJECT_ID' with your Google Cloud Project's project id."); } LocationName = LocationName.FromProjectLocation(ProjectId, LocationId); Client = StorageClient.Create(); From 54441e3a33347845270c2ad024d35949910664b9 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 9 Apr 2026 12:03:10 +0000 Subject: [PATCH 10/19] refactor(Storage Batch Operations): Convert create job test from fact to theory --- .../CreateBatchJobTest.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobTest.cs index c4e036f7050..3198eb0a647 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobTest.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobTest.cs @@ -65,12 +65,15 @@ public CreateBatchJobTest(StorageFixture fixture) _bucketList.Buckets.Insert(0, _bucket); } - [Fact] - public void TestCreateBatchJob() + [Theory] + [InlineData("DeleteObject")] + [InlineData("PutObjectHold")] + [InlineData("RewriteObject")] + [InlineData("PutMetadata")] + public void TestCreateBatchJob(string jobTransformationCase) { CreateBatchJobSample createJob = new CreateBatchJobSample(); var jobId = _fixture.GenerateGuid(); - var jobTransformationCase = "DeleteObject"; var holdState = "EventBasedHoldSet"; var jobTransformationObject = new object(); string jobType; From cc65ffd38236fe7f7a4f1a0c33dc1da8fa35942c Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Fri, 10 Apr 2026 05:46:24 +0000 Subject: [PATCH 11/19] fix(Storage Batch Operations): Add missing environment variables in fixture --- .../CreateBatchJobTest.cs | 19 +++---------------- .../StorageFixture.cs | 3 ++- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobTest.cs index 3198eb0a647..64e01dd03b4 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobTest.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobTest.cs @@ -91,11 +91,9 @@ public void TestCreateBatchJob(string jobTransformationCase) // If the job transformation case is RewriteObject, we can set the KmsKey and KmsKeyAsCryptoKeyName. if (jobTransformationCase == "RewriteObject") { - _keyRingId = GetEnvironmentVariable("STORAGE_KMS_KEYRING_ID", "This is the Key Ring ID"); - _cryptoKeyId = GetEnvironmentVariable("STORAGE_KMS_CRYPTOKEY_ID", "This is the Crypto Key ID."); - _kmsKey = $"projects/{_fixture.ProjectId}/locations/{_fixture.LocationId}/keyRings/{_keyRingId}/cryptoKeys/{_cryptoKeyId}"; - _cryptoKeyName = CryptoKeyName.FromProjectLocationKeyRingCryptoKey(_fixture.ProjectId, _fixture.LocationId, _keyRingId, _cryptoKeyId); - RewriteObject rewriteObject = new RewriteObject { KmsKey = _kmsKey, KmsKeyAsCryptoKeyName = _cryptoKeyName }; + string kmsKeyName = $"projects/{_fixture.ProjectId}/locations/{_fixture.LocationId}/keyRings/{_fixture.KmsKeyRing}/cryptoKeys/{_fixture.KmsKeyName}"; + var cryptoKeyName = CryptoKeyName.FromProjectLocationKeyRingCryptoKey(_fixture.ProjectId, _fixture.LocationId, _fixture.KmsKeyRing, _fixture.KmsKeyName); + RewriteObject rewriteObject = new RewriteObject { KmsKey = kmsKeyName, KmsKeyAsCryptoKeyName = cryptoKeyName }; jobTransformationObject = rewriteObject; } @@ -124,15 +122,4 @@ public void TestCreateBatchJob(string jobTransformationCase) Assert.NotNull(createdBatchJob.CompleteTime); _fixture.DeleteBatchJob(createdBatchJob.Name); } - - private static string GetEnvironmentVariable(string envVarName, string message = "") - { - string varValue = Environment.GetEnvironmentVariable(envVarName); - if (string.IsNullOrEmpty(varValue)) - { - throw new InvalidOperationException( - $"Please set the {envVarName} environment variable. {message}"); - } - return varValue; - } } diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/StorageFixture.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/StorageFixture.cs index 472be51c394..47ed3e0d60c 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/StorageFixture.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/StorageFixture.cs @@ -13,7 +13,6 @@ // limitations under the License. using Google.Api.Gax.ResourceNames; -using Google.Apis.Auth.OAuth2; using Google.Apis.Storage.v1.Data; using Google.Cloud.Storage.V1; using Google.Cloud.StorageBatchOperations.V1; @@ -31,6 +30,8 @@ public class StorageFixture : IDisposable, ICollectionFixture public StorageClient Client { get; } public StorageBatchOperationsClient OperationsClient { get; } public LocationName LocationName { get; } + public string KmsKeyRing { get; } = Environment.GetEnvironmentVariable("STORAGE_KMS_KEYRING"); + public string KmsKeyName { get; } = Environment.GetEnvironmentVariable("STORAGE_KMS_KEYNAME"); public StorageFixture() { From 626eb8fc765a87f25f12814a5d8679030611723e Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Fri, 10 Apr 2026 03:25:33 -0700 Subject: [PATCH 12/19] refactor(Storage Batch Operations): Remove unused KMS fields in CreateBatchJobTest Remove unused private fields `_kmsKey`, `_keyRingId`, `_cryptoKeyId`, and `_cryptoKeyName` from CreateBatchJobTest to improve code clarity and maintainability. --- .../CreateBatchJobTest.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobTest.cs index 64e01dd03b4..24bd1387b16 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobTest.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobTest.cs @@ -25,10 +25,6 @@ public class CreateBatchJobTest private readonly BucketList.Types.Bucket _bucket = new(); private readonly BucketList _bucketList = new(); private readonly PrefixList _prefixListObject = new(); - private string _kmsKey; - private string _keyRingId; - private string _cryptoKeyId; - private CryptoKeyName _cryptoKeyName; public CreateBatchJobTest(StorageFixture fixture) { From 217f1dd64b025733fc11cd185eb11aeb871874b2 Mon Sep 17 00:00:00 2001 From: Mahendra Date: Fri, 10 Apr 2026 18:58:21 +0530 Subject: [PATCH 13/19] refactor(Storage Batch Operations): Update CancelBatchJobTest to improve job cancellation logic --- .../CancelBatchJobTest.cs | 59 +++++++++++++------ 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs index d95ef8d84f9..e7d70db5a3c 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs @@ -18,7 +18,7 @@ using System; using System.IO; using System.Text; -using System.Threading.Tasks; +using System.Threading; using Xunit; [Collection(nameof(StorageFixture))] @@ -31,7 +31,7 @@ public class CancelBatchJobTest public CancelBatchJobTest(StorageFixture fixture) { - int i = 10; + int i = 20; _fixture = fixture; var bucketName = _fixture.GenerateBucketName(); _fixture.CreateBucket(bucketName, multiVersion: false, softDelete: false, registerForDeletion: true); @@ -45,6 +45,7 @@ public CancelBatchJobTest(StorageFixture fixture) _fixture.Client.UploadObject(bucketName, objectName, "application/text", streamObjectContent); i--; } + _bucket = new BucketList.Types.Bucket { Bucket_ = bucketName, @@ -67,22 +68,38 @@ public void TestCancelBatchJob() string orderBy = "create_time"; var jobId = _fixture.GenerateGuid(); + var createdJobName = CreateBatchJob(_fixture.LocationName, _bucketList, jobId); + Assert.NotNull(createdJobName); - try + // Poll until the job is in a state that can be cancelled (Running) + // This prevents attempting to cancel a job that hasn't started or has already finished. + bool isCancellable = false; + for (int attempt = 0; attempt < 10; attempt++) { - var createdJob = CreateBatchJob(_fixture.LocationName, _bucketList, jobId); - cancelBatchJob.CancelBatchJob(createdJob); - var batchJobs = listBatchJobs.ListBatchJobs(_fixture.LocationName, filter, pageSize, orderBy); - Assert.Contains(batchJobs, job => job.Name == createdJob); - Job cancelledJob = getBatchJob.GetBatchJob(createdJob); - Assert.Equal(createdJob, cancelledJob.Name.ToString()); - Assert.Equal("Canceled", cancelledJob.State.ToString()); - _fixture.DeleteBatchJob(createdJob); + Job currentJob = getBatchJob.GetBatchJob(createdJobName); + if (currentJob.State == Job.Types.State.Running) + { + isCancellable = true; + break; + } + if (currentJob.State == Job.Types.State.Succeeded) break; } - catch (InvalidOperationException ex) + + Assert.True(isCancellable, "Job did not reach a Running state in time to be cancelled."); + cancelBatchJob.CancelBatchJob(createdJobName); + + Job finalJob = null; + for (int attempt = 0; attempt < 100; attempt++) { - Assert.Equal("Job Name is Null", ex.Message); + finalJob = getBatchJob.GetBatchJob(createdJobName); + if (finalJob.State == Job.Types.State.Canceled) break; } + Assert.Equal(createdJobName, finalJob.Name.ToString()); + Assert.Equal(Job.Types.State.Canceled, finalJob.State); + + var batchJobs = listBatchJobs.ListBatchJobs(_fixture.LocationName, filter, pageSize, orderBy); + Assert.Contains(batchJobs, j => j.Name == createdJobName); + _fixture.DeleteBatchJob(createdJobName); } /// @@ -108,11 +125,15 @@ public static string CreateBatchJob(LocationName locationName, }; Operation response = storageBatchClient.CreateJob(request); - string operationName = response.Name; - Operation retrievedResponse = storageBatchClient.PollOnceCreateJob(operationName); - // Poll once asynchronously. - Task> retrievedAsyncResponse = retrievedResponse.PollOnceAsync(); - string jobName = retrievedAsyncResponse?.Result?.Metadata?.Job?.Name ?? throw new InvalidOperationException("Job Name is Null"); - return jobName; + + // We poll once to ensure the LRO has initialized the Job resource + Operation retrievedResponse = storageBatchClient.PollOnceCreateJob(response.Name); + + // Use Result to block until the initial metadata is available + var resultJob = retrievedResponse.Metadata?.Job; + + if (resultJob == null || string.IsNullOrEmpty(resultJob.Name)) + Thread.Sleep(4000); + return resultJob.Name; } } From 2a3d9182999b4cd2ff3972218f4de9c9f0ffba2b Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Mon, 13 Apr 2026 06:20:38 +0000 Subject: [PATCH 14/19] refactor(Storage Batch Operations): Replace Thread.Sleep with loop in CancelBatchJobTest --- .../CancelBatchJobTest.cs | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs index e7d70db5a3c..0db089ebe39 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs @@ -123,17 +123,19 @@ public static string CreateBatchJob(LocationName locationName, }, RequestId = jobId, }; - + Operation response = storageBatchClient.CreateJob(request); + string jobName = String.Empty; + for (int attempt = 0; attempt < 10; attempt++) + { + Operation retrievedResponse = storageBatchClient.PollOnceCreateJob(response.Name); + jobName = retrievedResponse.Metadata?.Job?.Name; - // We poll once to ensure the LRO has initialized the Job resource - Operation retrievedResponse = storageBatchClient.PollOnceCreateJob(response.Name); - - // Use Result to block until the initial metadata is available - var resultJob = retrievedResponse.Metadata?.Job; - - if (resultJob == null || string.IsNullOrEmpty(resultJob.Name)) - Thread.Sleep(4000); - return resultJob.Name; + if (!string.IsNullOrEmpty(jobName)) + { + break; + } + } + return jobName; } } From 2fe069a38132ffedfdd9c2e77e44a62c9c8e2532 Mon Sep 17 00:00:00 2001 From: Mahendra Date: Mon, 13 Apr 2026 11:57:53 +0530 Subject: [PATCH 15/19] style(Storage Batch Operations): Fix formatting in CancelBatchJobTest.cs --- .../StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs index 0db089ebe39..dd82d7f92a9 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs @@ -123,7 +123,7 @@ public static string CreateBatchJob(LocationName locationName, }, RequestId = jobId, }; - + Operation response = storageBatchClient.CreateJob(request); string jobName = String.Empty; for (int attempt = 0; attempt < 10; attempt++) From c38ef3eccef7d7c28d4caf4cb4efb968bc16f712 Mon Sep 17 00:00:00 2001 From: Mahendra Date: Mon, 13 Apr 2026 16:32:34 +0530 Subject: [PATCH 16/19] tests(Storage Batch Operations): Stabilize cancelBatchJobTest by increasing retry attempts Extended the retry attempts in cancelBatchJobTest from 100 to 1000. This ensures the test accounts for delayed state transitions in the Koko environment while maintaining efficiency via an early exit once the 'Canceled' state is reached. --- .../StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs index dd82d7f92a9..b781eeee53b 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CancelBatchJobTest.cs @@ -89,7 +89,7 @@ public void TestCancelBatchJob() cancelBatchJob.CancelBatchJob(createdJobName); Job finalJob = null; - for (int attempt = 0; attempt < 100; attempt++) + for (int attempt = 0; attempt < 1000; attempt++) { finalJob = getBatchJob.GetBatchJob(createdJobName); if (finalJob.State == Job.Types.State.Canceled) break; From f1ddbbfa937a12b5a465c4e666815ec3758a33af Mon Sep 17 00:00:00 2001 From: Mahendra Date: Mon, 13 Apr 2026 18:30:26 +0530 Subject: [PATCH 17/19] chore(Storage Batch Operations): Update dependencies Updated Google.Cloud.Storage.V1 to 4.14.0 and Google.Cloud.StorageBatchOperations.V1 to 1.0.0-beta05. --- .../StorageBatchOperations.Samples.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples/StorageBatchOperations.Samples.csproj b/storagebatchoperations/api/StorageBatchOperations.Samples/StorageBatchOperations.Samples.csproj index c1a4af4a4a9..e46f742aecc 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples/StorageBatchOperations.Samples.csproj +++ b/storagebatchoperations/api/StorageBatchOperations.Samples/StorageBatchOperations.Samples.csproj @@ -5,8 +5,8 @@ - - + + From f7ba2e6ecb8ac28bc94daffe4d4a59e0e2bb6316 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 15 Apr 2026 11:51:23 +0000 Subject: [PATCH 18/19] refactor(Storage Batch Operations): Addressing Review Feedback --- .../CreateBatchJobWithDeleteObjectTest.cs | 81 ++++++++++++ .../CreateBatchJobWithPutMetaDataTest.cs | 81 ++++++++++++ ...=> CreateBatchJobWithPutObjectHoldTest.cs} | 54 +------- .../CreateBatchJobWithRewriteObjectTest.cs | 84 ++++++++++++ .../DeleteBatchJobTest.cs | 4 +- .../GetBatchJobTest.cs | 4 +- .../ListBatchJobsTest.cs | 4 +- .../CreateBatchJob.cs | 121 ------------------ .../CreateBatchJobWithDeleteObject.cs | 59 +++++++++ .../CreateBatchJobWithPutMetadata.cs | 70 ++++++++++ .../CreateBatchJobWithPutObjectHold.cs | 66 ++++++++++ .../CreateBatchJobWithRewriteObject.cs | 69 ++++++++++ 12 files changed, 523 insertions(+), 174 deletions(-) create mode 100644 storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithDeleteObjectTest.cs create mode 100644 storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithPutMetaDataTest.cs rename storagebatchoperations/api/StorageBatchOperations.Samples.Tests/{CreateBatchJobTest.cs => CreateBatchJobWithPutObjectHoldTest.cs} (57%) create mode 100644 storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithRewriteObjectTest.cs delete mode 100644 storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJob.cs create mode 100644 storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithDeleteObject.cs create mode 100644 storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithPutMetadata.cs create mode 100644 storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithPutObjectHold.cs create mode 100644 storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithRewriteObject.cs diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithDeleteObjectTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithDeleteObjectTest.cs new file mode 100644 index 00000000000..d85a00f62d9 --- /dev/null +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithDeleteObjectTest.cs @@ -0,0 +1,81 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Google.Cloud.StorageBatchOperations.V1; +using System; +using System.IO; +using System.Text; +using Xunit; + +[Collection(nameof(StorageFixture))] +public class CreateBatchJobWithDeleteObjectTest +{ + private readonly StorageFixture _fixture; + private readonly BucketList.Types.Bucket _bucket = new(); + private readonly BucketList _bucketList = new(); + private readonly PrefixList _prefixListObject = new(); + + public CreateBatchJobWithDeleteObjectTest(StorageFixture fixture) + { + _fixture = fixture; + var bucketName = _fixture.GenerateBucketName(); + _fixture.CreateBucket(bucketName, multiVersion: false, softDelete: false, registerForDeletion: true); + + var manifestBucketName = _fixture.GenerateBucketName(); + _fixture.CreateBucket(manifestBucketName, multiVersion: false, softDelete: false, registerForDeletion: true); + + var objectName = _fixture.GenerateGuid(); + var manifestObjectName = _fixture.GenerateGuid(); + var objectContent = _fixture.GenerateGuid(); + var manifestObjectContent = $"bucket,name,generation{Environment.NewLine}{bucketName},{objectName}"; + + byte[] byteObjectContent = Encoding.UTF8.GetBytes(objectContent); + using MemoryStream streamObjectContent = new MemoryStream(byteObjectContent); + // Uploading an object to the bucket + _fixture.Client.UploadObject(bucketName, objectName, "application/text", streamObjectContent); + + byte[] byteManifestObjectContent = Encoding.UTF8.GetBytes(manifestObjectContent); + // Uploading a manifest object to the manifest bucket + using MemoryStream streamManifestObjectContent = new MemoryStream(byteManifestObjectContent); + _fixture.Client.UploadObject(manifestBucketName, $"{manifestObjectName}.csv", "text/csv", streamManifestObjectContent); + _bucket = new BucketList.Types.Bucket + { + Bucket_ = bucketName, + // The prefix list is used to specify the objects to be transformed. To match all objects, use an empty list. + PrefixList = _prefixListObject, + // Manifest location contains csv file having list of objects to be transformed" + Manifest = new Manifest { ManifestLocation = $"gs://{manifestBucketName}/{manifestObjectName}.csv" } + }; + // Adding the bucket to the bucket list. + _bucketList.Buckets.Insert(0, _bucket); + } + + [Fact] + public void TestCreateBatchJobWithDeleteObject() + { + CreateBatchJobWithDeleteObjectSample createJob = new CreateBatchJobWithDeleteObjectSample(); + var jobId = _fixture.GenerateGuid(); + + // Create a batch job with the specified transformation case and bucket list + var createdBatchJob = createJob.CreateBatchJobWithDeleteObject(_fixture.LocationName, _bucketList, jobId); + Assert.Equal(createdBatchJob.BucketList, _bucketList); + Assert.Equal("DeleteObject", createdBatchJob.TransformationCase.ToString()); + Assert.Equal(createdBatchJob.SourceCase.ToString(), _bucketList.GetType().Name); + Assert.NotNull(createdBatchJob.Name); + Assert.NotNull(createdBatchJob.JobName); + Assert.NotNull(createdBatchJob.CreateTime); + Assert.NotNull(createdBatchJob.CompleteTime); + _fixture.DeleteBatchJob(createdBatchJob.Name); + } +} diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithPutMetaDataTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithPutMetaDataTest.cs new file mode 100644 index 00000000000..b8e86411003 --- /dev/null +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithPutMetaDataTest.cs @@ -0,0 +1,81 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Google.Cloud.StorageBatchOperations.V1; +using System; +using System.IO; +using System.Text; +using Xunit; + +[Collection(nameof(StorageFixture))] +public class CreateBatchJobWithPutMetaDataTest +{ + private readonly StorageFixture _fixture; + private readonly BucketList.Types.Bucket _bucket = new(); + private readonly BucketList _bucketList = new(); + private readonly PrefixList _prefixListObject = new(); + + public CreateBatchJobWithPutMetaDataTest(StorageFixture fixture) + { + _fixture = fixture; + var bucketName = _fixture.GenerateBucketName(); + _fixture.CreateBucket(bucketName, multiVersion: false, softDelete: false, registerForDeletion: true); + + var manifestBucketName = _fixture.GenerateBucketName(); + _fixture.CreateBucket(manifestBucketName, multiVersion: false, softDelete: false, registerForDeletion: true); + + var objectName = _fixture.GenerateGuid(); + var manifestObjectName = _fixture.GenerateGuid(); + var objectContent = _fixture.GenerateGuid(); + var manifestObjectContent = $"bucket,name,generation{Environment.NewLine}{bucketName},{objectName}"; + + byte[] byteObjectContent = Encoding.UTF8.GetBytes(objectContent); + using MemoryStream streamObjectContent = new MemoryStream(byteObjectContent); + // Uploading an object to the bucket + _fixture.Client.UploadObject(bucketName, objectName, "application/text", streamObjectContent); + + byte[] byteManifestObjectContent = Encoding.UTF8.GetBytes(manifestObjectContent); + // Uploading a manifest object to the manifest bucket + using MemoryStream streamManifestObjectContent = new MemoryStream(byteManifestObjectContent); + _fixture.Client.UploadObject(manifestBucketName, $"{manifestObjectName}.csv", "text/csv", streamManifestObjectContent); + _bucket = new BucketList.Types.Bucket + { + Bucket_ = bucketName, + // The prefix list is used to specify the objects to be transformed. To match all objects, use an empty list. + PrefixList = _prefixListObject, + // Manifest location contains csv file having list of objects to be transformed" + Manifest = new Manifest { ManifestLocation = $"gs://{manifestBucketName}/{manifestObjectName}.csv" } + }; + // Adding the bucket to the bucket list. + _bucketList.Buckets.Insert(0, _bucket); + } + + [Fact] + public void TestCreateBatchJobWithPutMetaData() + { + CreateBatchJobWithPutMetadataSample createJob = new CreateBatchJobWithPutMetadataSample(); + var jobId = _fixture.GenerateGuid(); + + // Create a batch job with the specified transformation case and bucket list + var createdBatchJob = createJob.CreateBatchJobWithPutMetadata(_fixture.LocationName, _bucketList, jobId); + Assert.Equal(createdBatchJob.BucketList, _bucketList); + Assert.Equal("PutMetadata", createdBatchJob.TransformationCase.ToString()); + Assert.Equal(createdBatchJob.SourceCase.ToString(), _bucketList.GetType().Name); + Assert.NotNull(createdBatchJob.Name); + Assert.NotNull(createdBatchJob.JobName); + Assert.NotNull(createdBatchJob.CreateTime); + Assert.NotNull(createdBatchJob.CompleteTime); + _fixture.DeleteBatchJob(createdBatchJob.Name); + } +} diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithPutObjectHoldTest.cs similarity index 57% rename from storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobTest.cs rename to storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithPutObjectHoldTest.cs index 24bd1387b16..663f64a159b 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobTest.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithPutObjectHoldTest.cs @@ -19,14 +19,14 @@ using Xunit; [Collection(nameof(StorageFixture))] -public class CreateBatchJobTest +public class CreateBatchJobWithPutObjectHoldTest { private readonly StorageFixture _fixture; private readonly BucketList.Types.Bucket _bucket = new(); private readonly BucketList _bucketList = new(); private readonly PrefixList _prefixListObject = new(); - public CreateBatchJobTest(StorageFixture fixture) + public CreateBatchJobWithPutObjectHoldTest(StorageFixture fixture) { _fixture = fixture; var bucketName = _fixture.GenerateBucketName(); @@ -61,56 +61,16 @@ public CreateBatchJobTest(StorageFixture fixture) _bucketList.Buckets.Insert(0, _bucket); } - [Theory] - [InlineData("DeleteObject")] - [InlineData("PutObjectHold")] - [InlineData("RewriteObject")] - [InlineData("PutMetadata")] - public void TestCreateBatchJob(string jobTransformationCase) + [Fact] + public void TestCreateBatchJobWithPutObjectHold() { - CreateBatchJobSample createJob = new CreateBatchJobSample(); + CreateBatchJobWithPutObjectHoldSample createJob = new CreateBatchJobWithPutObjectHoldSample(); var jobId = _fixture.GenerateGuid(); - var holdState = "EventBasedHoldSet"; - var jobTransformationObject = new object(); - string jobType; - // If the job transformation case is PutObjectHold, we can set the hold state to EventBasedHoldSet or EventBasedHoldUnSet or TemporaryHoldSet or TemporaryHoldUnSet. - if (jobTransformationCase == "PutObjectHold") - { - jobType = $"{jobTransformationCase}{holdState}"; - } - // If the job transformation case is other than PutObjectHold, we dont set the hold state. - else - { - jobType = jobTransformationCase; - } - // If the job transformation case is RewriteObject, we can set the KmsKey and KmsKeyAsCryptoKeyName. - if (jobTransformationCase == "RewriteObject") - { - string kmsKeyName = $"projects/{_fixture.ProjectId}/locations/{_fixture.LocationId}/keyRings/{_fixture.KmsKeyRing}/cryptoKeys/{_fixture.KmsKeyName}"; - var cryptoKeyName = CryptoKeyName.FromProjectLocationKeyRingCryptoKey(_fixture.ProjectId, _fixture.LocationId, _fixture.KmsKeyRing, _fixture.KmsKeyName); - RewriteObject rewriteObject = new RewriteObject { KmsKey = kmsKeyName, KmsKeyAsCryptoKeyName = cryptoKeyName }; - jobTransformationObject = rewriteObject; - - } - // If the job transformation case is PutMetadata, we can set the CacheControl, ContentDisposition, ContentEncoding, ContentLanguage, ContentType and CustomTime. - else if (jobTransformationCase == "PutMetadata") - { - PutMetadata putMetadata = new PutMetadata - { - CacheControl = "no-cache", - ContentDisposition = "inline", - ContentEncoding = "gzip", - ContentLanguage = "en-US", - ContentType = "text/plain", - CustomTime = DateTime.UtcNow.ToString("o") - }; - jobTransformationObject = putMetadata; - } // Create a batch job with the specified transformation case and bucket list - var createdBatchJob = createJob.CreateBatchJob(_fixture.LocationName, _bucketList, jobId, jobType, jobTransformationObject); + var createdBatchJob = createJob.CreateBatchJobWithPutObjectHold(_fixture.LocationName, _bucketList, jobId); Assert.Equal(createdBatchJob.BucketList, _bucketList); - Assert.Equal(createdBatchJob.TransformationCase.ToString(), jobTransformationCase); + Assert.Equal("PutObjectHold", createdBatchJob.TransformationCase.ToString()); Assert.Equal(createdBatchJob.SourceCase.ToString(), _bucketList.GetType().Name); Assert.NotNull(createdBatchJob.Name); Assert.NotNull(createdBatchJob.JobName); diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithRewriteObjectTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithRewriteObjectTest.cs new file mode 100644 index 00000000000..89082267cc4 --- /dev/null +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithRewriteObjectTest.cs @@ -0,0 +1,84 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Google.Cloud.StorageBatchOperations.V1; +using System; +using System.IO; +using System.Text; +using Xunit; + +[Collection(nameof(StorageFixture))] +public class CreateBatchJobWithRewriteObjectTest +{ + private readonly StorageFixture _fixture; + private readonly BucketList.Types.Bucket _bucket = new(); + private readonly BucketList _bucketList = new(); + private readonly PrefixList _prefixListObject = new(); + + public CreateBatchJobWithRewriteObjectTest(StorageFixture fixture) + { + _fixture = fixture; + var bucketName = _fixture.GenerateBucketName(); + _fixture.CreateBucket(bucketName, multiVersion: false, softDelete: false, registerForDeletion: true); + + var manifestBucketName = _fixture.GenerateBucketName(); + _fixture.CreateBucket(manifestBucketName, multiVersion: false, softDelete: false, registerForDeletion: true); + + var objectName = _fixture.GenerateGuid(); + var manifestObjectName = _fixture.GenerateGuid(); + var objectContent = _fixture.GenerateGuid(); + var manifestObjectContent = $"bucket,name,generation{Environment.NewLine}{bucketName},{objectName}"; + + byte[] byteObjectContent = Encoding.UTF8.GetBytes(objectContent); + using MemoryStream streamObjectContent = new MemoryStream(byteObjectContent); + // Uploading an object to the bucket + _fixture.Client.UploadObject(bucketName, objectName, "application/text", streamObjectContent); + + byte[] byteManifestObjectContent = Encoding.UTF8.GetBytes(manifestObjectContent); + // Uploading a manifest object to the manifest bucket + using MemoryStream streamManifestObjectContent = new MemoryStream(byteManifestObjectContent); + _fixture.Client.UploadObject(manifestBucketName, $"{manifestObjectName}.csv", "text/csv", streamManifestObjectContent); + _bucket = new BucketList.Types.Bucket + { + Bucket_ = bucketName, + // The prefix list is used to specify the objects to be transformed. To match all objects, use an empty list. + PrefixList = _prefixListObject, + // Manifest location contains csv file having list of objects to be transformed" + Manifest = new Manifest { ManifestLocation = $"gs://{manifestBucketName}/{manifestObjectName}.csv" } + }; + // Adding the bucket to the bucket list. + _bucketList.Buckets.Insert(0, _bucket); + } + + [Fact] + public void TestCreateBatchJobWithRewriteObject() + { + CreateBatchJobWithRewriteObjectSample createJob = new CreateBatchJobWithRewriteObjectSample(); + var jobId = _fixture.GenerateGuid(); + + string kmsKeyName = $"projects/{_fixture.ProjectId}/locations/{_fixture.LocationId}/keyRings/{_fixture.KmsKeyRing}/cryptoKeys/{_fixture.KmsKeyName}"; + var cryptoKeyName = CryptoKeyName.FromProjectLocationKeyRingCryptoKey(_fixture.ProjectId, _fixture.LocationId, _fixture.KmsKeyRing, _fixture.KmsKeyName); + + // Create a batch job with the specified transformation case and bucket list + var createdBatchJob = createJob.CreateBatchJobWithRewriteObject(_fixture.LocationName, _bucketList, jobId, kmsKeyName, cryptoKeyName); + Assert.Equal(createdBatchJob.BucketList, _bucketList); + Assert.Equal("RewriteObject", createdBatchJob.TransformationCase.ToString()); + Assert.Equal(createdBatchJob.SourceCase.ToString(), _bucketList.GetType().Name); + Assert.NotNull(createdBatchJob.Name); + Assert.NotNull(createdBatchJob.JobName); + Assert.NotNull(createdBatchJob.CreateTime); + Assert.NotNull(createdBatchJob.CompleteTime); + _fixture.DeleteBatchJob(createdBatchJob.Name); + } +} diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/DeleteBatchJobTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/DeleteBatchJobTest.cs index 8954c6caf41..fbeb3da4d92 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/DeleteBatchJobTest.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/DeleteBatchJobTest.cs @@ -44,14 +44,14 @@ public void TestDeleteBatchJob() DeleteBatchJobSample deleteBatchJob = new DeleteBatchJobSample(); ListBatchJobsSample listBatchJobs = new ListBatchJobsSample(); GetBatchJobSample getBatchJob = new GetBatchJobSample(); - CreateBatchJobSample createBatchJob = new CreateBatchJobSample(); + CreateBatchJobWithDeleteObjectSample createBatchJob = new CreateBatchJobWithDeleteObjectSample(); string filter = ""; int pageSize = 10; string orderBy = "create_time"; var jobId = _fixture.GenerateGuid(); - var createdJob = createBatchJob.CreateBatchJob(_fixture.LocationName, _bucketList, jobId); + var createdJob = createBatchJob.CreateBatchJobWithDeleteObject(_fixture.LocationName, _bucketList, jobId); // Delete the created job. deleteBatchJob.DeleteBatchJob(createdJob.Name); var batchJobs = listBatchJobs.ListBatchJobs(_fixture.LocationName, filter, pageSize, orderBy); diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/GetBatchJobTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/GetBatchJobTest.cs index 2b5a0040894..fb1d9837e88 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/GetBatchJobTest.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/GetBatchJobTest.cs @@ -42,11 +42,11 @@ public GetBatchJobTest(StorageFixture fixture) public void TestGetBatchJob() { GetBatchJobSample getJob = new GetBatchJobSample(); - CreateBatchJobSample createJob = new CreateBatchJobSample(); + CreateBatchJobWithDeleteObjectSample createJob = new CreateBatchJobWithDeleteObjectSample(); var jobId = _fixture.GenerateGuid(); // Create a batch job with the specified bucket list and job ID. - var createdJob = createJob.CreateBatchJob(_fixture.LocationName, _bucketList, jobId); + var createdJob = createJob.CreateBatchJobWithDeleteObject(_fixture.LocationName, _bucketList, jobId); // Get the created job using its name. var retrievedJob = getJob.GetBatchJob(createdJob.Name); // Assert that the retrieved job is not null. diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/ListBatchJobsTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/ListBatchJobsTest.cs index f37ad645f93..ae7a7961c06 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/ListBatchJobsTest.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/ListBatchJobsTest.cs @@ -42,7 +42,7 @@ public ListBatchJobsTest(StorageFixture fixture) public void TestListBatchJobs() { ListBatchJobsSample listBatchJobs = new ListBatchJobsSample(); - CreateBatchJobSample createBatchJob = new CreateBatchJobSample(); + CreateBatchJobWithDeleteObjectSample createBatchJob = new CreateBatchJobWithDeleteObjectSample(); // Filter to list only succeeded jobs string filter = "state:succeeded"; @@ -50,7 +50,7 @@ public void TestListBatchJobs() string orderBy = "create_time"; var jobId = _fixture.GenerateGuid(); - var createdJob = createBatchJob.CreateBatchJob(_fixture.LocationName, _bucketList, jobId); + var createdJob = createBatchJob.CreateBatchJobWithDeleteObject(_fixture.LocationName, _bucketList, jobId); // List batch jobs with the specified filter, page size, and order by criteria var batchJobs = listBatchJobs.ListBatchJobs(_fixture.LocationName, filter, pageSize, orderBy); // Assert that the created job is in the list of batch jobs. diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJob.cs b/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJob.cs deleted file mode 100644 index 27f73b52bf4..00000000000 --- a/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJob.cs +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"). -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// [START storage_batch_create_job] - -using Google.Api.Gax.ResourceNames; -using Google.Cloud.StorageBatchOperations.V1; -using Google.LongRunning; -using System; - -public class CreateBatchJobSample -{ - /// - /// Creates a storage batch operation job. - /// - /// A resource name with pattern projects/{project}/locations/{location}. - /// A bucket list contains list of buckets and their objects to be transformed. - /// It is id for the job and it should not be more than 128 characters and must include only - /// characters available in DNS names, as defined by RFC-1123. - /// It is type of storage batch operation job. - /// It is object of type either RewriteObject or PutMetadata. - public Job CreateBatchJob(LocationName locationName, - BucketList bucketList, - string jobId = "12345678910", - string jobType = "DeleteObject", - object jobTransformationObject = null) - { - StorageBatchOperationsClient operationsClient = StorageBatchOperationsClient.Create(); - Job job = null; - PutObjectHold putObjectHold; - switch (jobType) - { - case "RewriteObject": - job = new Job - { - RewriteObject = (RewriteObject) jobTransformationObject, - BucketList = bucketList - }; - break; - case "PutMetadata": - job = new Job - { - PutMetadata = (PutMetadata) jobTransformationObject, - BucketList = bucketList - }; - break; - case "DeleteObject": - var deleteObject = new DeleteObject { PermanentObjectDeletionEnabled = true }; - - job = new Job - { - DeleteObject = deleteObject, - BucketList = bucketList - }; - break; - case "PutObjectHoldEventBasedHoldSet": - putObjectHold = new PutObjectHold { EventBasedHold = PutObjectHold.Types.HoldStatus.Set }; - - job = new Job - { - PutObjectHold = putObjectHold, - BucketList = bucketList - }; - break; - case "PutObjectHoldEventBasedHoldUnSet": - putObjectHold = new PutObjectHold { EventBasedHold = PutObjectHold.Types.HoldStatus.Unset }; - - job = new Job - { - PutObjectHold = putObjectHold, - BucketList = bucketList - }; - break; - case "PutObjectHoldTemporaryHoldSet": - putObjectHold = new PutObjectHold { TemporaryHold = PutObjectHold.Types.HoldStatus.Set }; - - job = new Job - { - PutObjectHold = putObjectHold, - BucketList = bucketList - }; - break; - case "PutObjectHoldTemporaryHoldUnSet": - putObjectHold = new PutObjectHold { TemporaryHold = PutObjectHold.Types.HoldStatus.Unset }; - - job = new Job - { - PutObjectHold = putObjectHold, - BucketList = bucketList - }; - break; - } - - // Create a job request with the specified location, job ID, and job details. - CreateJobRequest request = new CreateJobRequest - { - ParentAsLocationName = locationName, - JobId = jobId, - Job = job, - RequestId = jobId, - }; - - Operation response = operationsClient.CreateJob(request); - Operation completedResponse = response.PollUntilCompleted(); - Job result = completedResponse.Result; - Console.WriteLine($"The Storage Batch Operation Job (Name: {result.Name}) is created"); - return result; - } -} -// [END storage_batch_create_job] diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithDeleteObject.cs b/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithDeleteObject.cs new file mode 100644 index 00000000000..8d587dcb038 --- /dev/null +++ b/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithDeleteObject.cs @@ -0,0 +1,59 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// [START storage_batch_create_job] + +using Google.Api.Gax.ResourceNames; +using Google.Cloud.StorageBatchOperations.V1; +using Google.LongRunning; +using System; + +public class CreateBatchJobWithDeleteObjectSample +{ + /// + /// Creates a storage batch operation job. + /// + /// A resource name with pattern projects/{project}/locations/{location}. + /// A bucket list contains list of buckets and their objects to be transformed. + /// It is id for the job and it should not be more than 128 characters and must include only + /// characters available in DNS names, as defined by RFC-1123. + public Job CreateBatchJobWithDeleteObject(LocationName locationName, + BucketList bucketList, + string jobId = "your-job-id") + { + StorageBatchOperationsClient operationsClient = StorageBatchOperationsClient.Create(); + + Job job = new Job + { + DeleteObject = new DeleteObject { PermanentObjectDeletionEnabled = true }, + BucketList = bucketList + }; + + // Create a job request with the specified location, job ID, and job details. + CreateJobRequest request = new CreateJobRequest + { + ParentAsLocationName = locationName, + JobId = jobId, + Job = job, + RequestId = jobId, + }; + + Operation response = operationsClient.CreateJob(request); + Operation completedResponse = response.PollUntilCompleted(); + Job result = completedResponse.Result; + Console.WriteLine($"The Storage Batch Operation Job (Name: {result.Name}) is created"); + return result; + } +} +// [END storage_batch_create_job] diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithPutMetadata.cs b/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithPutMetadata.cs new file mode 100644 index 00000000000..9eff59cd6a4 --- /dev/null +++ b/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithPutMetadata.cs @@ -0,0 +1,70 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// [START storage_batch_create_job] + +using Google.Api.Gax.ResourceNames; +using Google.Cloud.StorageBatchOperations.V1; +using Google.LongRunning; +using Grpc.Core; +using System; + +public class CreateBatchJobWithPutMetadataSample +{ + /// + /// Creates a storage batch operation job. + /// + /// A resource name with pattern projects/{project}/locations/{location}. + /// A bucket list contains list of buckets and their objects to be transformed. + /// It is id for the job and it should not be more than 128 characters and must include only + /// characters available in DNS names, as defined by RFC-1123. + public Job CreateBatchJobWithPutMetadata(LocationName locationName, + BucketList bucketList, + string jobId = "your-job-id") + { + StorageBatchOperationsClient operationsClient = StorageBatchOperationsClient.Create(); + + PutMetadata putMetadata = new PutMetadata + { + CacheControl = "no-cache", + ContentDisposition = "inline", + ContentEncoding = "gzip", + ContentLanguage = "en-US", + ContentType = "text/plain", + CustomTime = DateTime.UtcNow.ToString("o") + }; + + Job job = new Job + { + PutMetadata = putMetadata, + BucketList = bucketList + }; + + // Create a job request with the specified location, job ID, and job details. + CreateJobRequest request = new CreateJobRequest + { + ParentAsLocationName = locationName, + JobId = jobId, + Job = job, + RequestId = jobId, + }; + + Operation response = operationsClient.CreateJob(request); + Operation completedResponse = response.PollUntilCompleted(); + Job result = completedResponse.Result; + Console.WriteLine($"The Storage Batch Operation Job (Name: {result.Name}) is created"); + return result; + } +} +// [END storage_batch_create_job] diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithPutObjectHold.cs b/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithPutObjectHold.cs new file mode 100644 index 00000000000..f6985a8f5d7 --- /dev/null +++ b/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithPutObjectHold.cs @@ -0,0 +1,66 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// [START storage_batch_create_job] + +using Google.Api.Gax.ResourceNames; +using Google.Cloud.StorageBatchOperations.V1; +using Google.LongRunning; +using System; + +public class CreateBatchJobWithPutObjectHoldSample +{ + /// + /// Creates a storage batch operation job. + /// + /// A resource name with pattern projects/{project}/locations/{location}. + /// A bucket list contains list of buckets and their objects to be transformed. + /// It is id for the job and it should not be more than 128 characters and must include only + /// characters available in DNS names, as defined by RFC-1123. + + public Job CreateBatchJobWithPutObjectHold(LocationName locationName, + BucketList bucketList, + string jobId = "your-job-id") + { + StorageBatchOperationsClient operationsClient = StorageBatchOperationsClient.Create(); + + + PutObjectHold putObjectHold = new PutObjectHold + { + EventBasedHold = PutObjectHold.Types.HoldStatus.Set + }; + + Job job = new Job + { + PutObjectHold = putObjectHold, + BucketList = bucketList + }; + + // Create a job request with the specified location, job ID, and job details. + CreateJobRequest request = new CreateJobRequest + { + ParentAsLocationName = locationName, + JobId = jobId, + Job = job, + RequestId = jobId, + }; + + Operation response = operationsClient.CreateJob(request); + Operation completedResponse = response.PollUntilCompleted(); + Job result = completedResponse.Result; + Console.WriteLine($"The Storage Batch Operation Job (Name: {result.Name}) is created"); + return result; + } +} +// [END storage_batch_create_job] diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithRewriteObject.cs b/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithRewriteObject.cs new file mode 100644 index 00000000000..e68df8aba86 --- /dev/null +++ b/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithRewriteObject.cs @@ -0,0 +1,69 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// [START storage_batch_create_job] + +using Google.Api.Gax.ResourceNames; +using Google.Cloud.StorageBatchOperations.V1; +using Google.LongRunning; +using Grpc.Core; +using System; + +public class CreateBatchJobWithRewriteObjectSample +{ + /// + /// Creates a storage batch operation job. + /// + /// A resource name with pattern projects/{project}/locations/{location}. + /// A bucket list contains list of buckets and their objects to be transformed. + /// It is id for the job and it should not be more than 128 characters and must include only + /// characters available in DNS names, as defined by RFC-1123. + /// Resource name of the Cloud KMS key that will be used to encrypt the object.The Cloud + /// KMS key must be located in same location as the object. + /// + /// -typed view over the resource name property./param> + + public Job CreateBatchJobWithRewriteObject(LocationName locationName, + BucketList bucketList, + string jobId = "your-job-id", + string kmsKeyName = "your-kms-key", + CryptoKeyName cryptoKeyName = null) + { + StorageBatchOperationsClient operationsClient = StorageBatchOperationsClient.Create(); + + RewriteObject rewriteObject = new RewriteObject { KmsKey = kmsKeyName, KmsKeyAsCryptoKeyName = cryptoKeyName }; + + Job job = new Job + { + RewriteObject = rewriteObject, + BucketList = bucketList + }; + + // Create a job request with the specified location, job ID, and job details. + CreateJobRequest request = new CreateJobRequest + { + ParentAsLocationName = locationName, + JobId = jobId, + Job = job, + RequestId = jobId, + }; + + Operation response = operationsClient.CreateJob(request); + Operation completedResponse = response.PollUntilCompleted(); + Job result = completedResponse.Result; + Console.WriteLine($"The Storage Batch Operation Job (Name: {result.Name}) is created"); + return result; + } +} +// [END storage_batch_create_job] From e583cdf9b3c26a1a1ec2202d9285d1c0b8763c73 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 15 Apr 2026 05:38:14 -0700 Subject: [PATCH 19/19] refactor(Storage Batch Operations): Improve console output messages and update XML docs --- .../CreateBatchJobWithDeleteObjectTest.cs | 3 +-- .../CreateBatchJobWithPutMetaDataTest.cs | 3 +-- .../CreateBatchJobWithPutObjectHoldTest.cs | 1 - .../CreateBatchJobWithRewriteObjectTest.cs | 3 +-- .../CreateBatchJobWithDeleteObject.cs | 4 ++-- .../CreateBatchJobWithPutMetadata.cs | 5 ++--- .../CreateBatchJobWithPutObjectHold.cs | 6 ++---- .../CreateBatchJobWithRewriteObject.cs | 9 +++------ 8 files changed, 12 insertions(+), 22 deletions(-) diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithDeleteObjectTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithDeleteObjectTest.cs index d85a00f62d9..8f4eb162473 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithDeleteObjectTest.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithDeleteObjectTest.cs @@ -66,8 +66,7 @@ public void TestCreateBatchJobWithDeleteObject() { CreateBatchJobWithDeleteObjectSample createJob = new CreateBatchJobWithDeleteObjectSample(); var jobId = _fixture.GenerateGuid(); - - // Create a batch job with the specified transformation case and bucket list + var createdBatchJob = createJob.CreateBatchJobWithDeleteObject(_fixture.LocationName, _bucketList, jobId); Assert.Equal(createdBatchJob.BucketList, _bucketList); Assert.Equal("DeleteObject", createdBatchJob.TransformationCase.ToString()); diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithPutMetaDataTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithPutMetaDataTest.cs index b8e86411003..cd2a5edb21b 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithPutMetaDataTest.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithPutMetaDataTest.cs @@ -66,8 +66,7 @@ public void TestCreateBatchJobWithPutMetaData() { CreateBatchJobWithPutMetadataSample createJob = new CreateBatchJobWithPutMetadataSample(); var jobId = _fixture.GenerateGuid(); - - // Create a batch job with the specified transformation case and bucket list + var createdBatchJob = createJob.CreateBatchJobWithPutMetadata(_fixture.LocationName, _bucketList, jobId); Assert.Equal(createdBatchJob.BucketList, _bucketList); Assert.Equal("PutMetadata", createdBatchJob.TransformationCase.ToString()); diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithPutObjectHoldTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithPutObjectHoldTest.cs index 663f64a159b..01925673c2e 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithPutObjectHoldTest.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithPutObjectHoldTest.cs @@ -67,7 +67,6 @@ public void TestCreateBatchJobWithPutObjectHold() CreateBatchJobWithPutObjectHoldSample createJob = new CreateBatchJobWithPutObjectHoldSample(); var jobId = _fixture.GenerateGuid(); - // Create a batch job with the specified transformation case and bucket list var createdBatchJob = createJob.CreateBatchJobWithPutObjectHold(_fixture.LocationName, _bucketList, jobId); Assert.Equal(createdBatchJob.BucketList, _bucketList); Assert.Equal("PutObjectHold", createdBatchJob.TransformationCase.ToString()); diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithRewriteObjectTest.cs b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithRewriteObjectTest.cs index 89082267cc4..304dac88d8b 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithRewriteObjectTest.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples.Tests/CreateBatchJobWithRewriteObjectTest.cs @@ -66,11 +66,10 @@ public void TestCreateBatchJobWithRewriteObject() { CreateBatchJobWithRewriteObjectSample createJob = new CreateBatchJobWithRewriteObjectSample(); var jobId = _fixture.GenerateGuid(); - + string kmsKeyName = $"projects/{_fixture.ProjectId}/locations/{_fixture.LocationId}/keyRings/{_fixture.KmsKeyRing}/cryptoKeys/{_fixture.KmsKeyName}"; var cryptoKeyName = CryptoKeyName.FromProjectLocationKeyRingCryptoKey(_fixture.ProjectId, _fixture.LocationId, _fixture.KmsKeyRing, _fixture.KmsKeyName); - // Create a batch job with the specified transformation case and bucket list var createdBatchJob = createJob.CreateBatchJobWithRewriteObject(_fixture.LocationName, _bucketList, jobId, kmsKeyName, cryptoKeyName); Assert.Equal(createdBatchJob.BucketList, _bucketList); Assert.Equal("RewriteObject", createdBatchJob.TransformationCase.ToString()); diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithDeleteObject.cs b/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithDeleteObject.cs index 8d587dcb038..f180ba0af4f 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithDeleteObject.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithDeleteObject.cs @@ -22,7 +22,7 @@ public class CreateBatchJobWithDeleteObjectSample { /// - /// Creates a storage batch operation job. + /// Creates a storage batch operation job with the option of delete objects. /// /// A resource name with pattern projects/{project}/locations/{location}. /// A bucket list contains list of buckets and their objects to be transformed. @@ -52,7 +52,7 @@ public Job CreateBatchJobWithDeleteObject(LocationName locationName, Operation response = operationsClient.CreateJob(request); Operation completedResponse = response.PollUntilCompleted(); Job result = completedResponse.Result; - Console.WriteLine($"The Storage Batch Operation Job (Name: {result.Name}) is created"); + Console.WriteLine($"The Storage Batch Operation Job (Name: {result.Name}) is created with the option of Delete Objects"); return result; } } diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithPutMetadata.cs b/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithPutMetadata.cs index 9eff59cd6a4..84cdd6a0fbb 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithPutMetadata.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithPutMetadata.cs @@ -17,13 +17,12 @@ using Google.Api.Gax.ResourceNames; using Google.Cloud.StorageBatchOperations.V1; using Google.LongRunning; -using Grpc.Core; using System; public class CreateBatchJobWithPutMetadataSample { /// - /// Creates a storage batch operation job. + /// Creates a storage batch operation job with the options of object metadata update. /// /// A resource name with pattern projects/{project}/locations/{location}. /// A bucket list contains list of buckets and their objects to be transformed. @@ -63,7 +62,7 @@ public Job CreateBatchJobWithPutMetadata(LocationName locationName, Operation response = operationsClient.CreateJob(request); Operation completedResponse = response.PollUntilCompleted(); Job result = completedResponse.Result; - Console.WriteLine($"The Storage Batch Operation Job (Name: {result.Name}) is created"); + Console.WriteLine($"The Storage Batch Operation Job (Name: {result.Name}) is created with the Object Metadata Update"); return result; } } diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithPutObjectHold.cs b/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithPutObjectHold.cs index f6985a8f5d7..a9f92e86d12 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithPutObjectHold.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithPutObjectHold.cs @@ -22,20 +22,18 @@ public class CreateBatchJobWithPutObjectHoldSample { /// - /// Creates a storage batch operation job. + /// Creates a storage batch operation job with the options to update object hold. /// /// A resource name with pattern projects/{project}/locations/{location}. /// A bucket list contains list of buckets and their objects to be transformed. /// It is id for the job and it should not be more than 128 characters and must include only /// characters available in DNS names, as defined by RFC-1123. - public Job CreateBatchJobWithPutObjectHold(LocationName locationName, BucketList bucketList, string jobId = "your-job-id") { StorageBatchOperationsClient operationsClient = StorageBatchOperationsClient.Create(); - PutObjectHold putObjectHold = new PutObjectHold { EventBasedHold = PutObjectHold.Types.HoldStatus.Set @@ -59,7 +57,7 @@ public Job CreateBatchJobWithPutObjectHold(LocationName locationName, Operation response = operationsClient.CreateJob(request); Operation completedResponse = response.PollUntilCompleted(); Job result = completedResponse.Result; - Console.WriteLine($"The Storage Batch Operation Job (Name: {result.Name}) is created"); + Console.WriteLine($"The Storage Batch Operation Job (Name: {result.Name}) is created with the Object Hold Update"); return result; } } diff --git a/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithRewriteObject.cs b/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithRewriteObject.cs index e68df8aba86..854fbbcf90f 100644 --- a/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithRewriteObject.cs +++ b/storagebatchoperations/api/StorageBatchOperations.Samples/CreateBatchJobWithRewriteObject.cs @@ -17,23 +17,20 @@ using Google.Api.Gax.ResourceNames; using Google.Cloud.StorageBatchOperations.V1; using Google.LongRunning; -using Grpc.Core; using System; public class CreateBatchJobWithRewriteObjectSample { /// - /// Creates a storage batch operation job. + /// Creates a storage batch operation job with the options for object rewrite. /// /// A resource name with pattern projects/{project}/locations/{location}. /// A bucket list contains list of buckets and their objects to be transformed. /// It is id for the job and it should not be more than 128 characters and must include only /// characters available in DNS names, as defined by RFC-1123. /// Resource name of the Cloud KMS key that will be used to encrypt the object.The Cloud - /// KMS key must be located in same location as the object. - /// + /// KMS key must be located in same location as the object. /// -typed view over the resource name property./param> - public Job CreateBatchJobWithRewriteObject(LocationName locationName, BucketList bucketList, string jobId = "your-job-id", @@ -62,7 +59,7 @@ public Job CreateBatchJobWithRewriteObject(LocationName locationName, Operation response = operationsClient.CreateJob(request); Operation completedResponse = response.PollUntilCompleted(); Job result = completedResponse.Result; - Console.WriteLine($"The Storage Batch Operation Job (Name: {result.Name}) is created"); + Console.WriteLine($"The Storage Batch Operation Job (Name: {result.Name}) is created with the Rewrite Object"); return result; } }