From ba3c128633f7fb9f4686af3893820a60d3050ffe Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 9 Oct 2024 01:50:09 -0700 Subject: [PATCH 01/58] feat(Storage Transfer) : Addition of Sample and Test Case for storagetransfer_transfer_to_nearline. --- .../TransferToNearlineTest.cs | 67 ++++++++++++++++++ .../TransferToNearlineSample.cs | 68 +++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs create mode 100644 storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs new file mode 100644 index 00000000000..bf688b6eb7f --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -0,0 +1,67 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +using Google.Cloud.Storage.V1; +using Google.Cloud.StorageTransfer.V1; +using System; +using Xunit; + +namespace StorageTransfer.Samples.Tests; + +[Collection(nameof(StorageFixture))] +public class TransferToNearlineTest : IDisposable +{ + private readonly StorageFixture _fixture; + private string _transferJobName; + public TransferToNearlineTest(StorageFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public void TestTransferToNearline() + { + TransferToNearlineSample transferToNearlineSample = new TransferToNearlineSample(); + var storage = StorageClient.Create(); + var bucket = storage.GetBucket(_fixture.BucketNameSink); + string storageClass = StorageClasses.Nearline; + bucket.StorageClass = storageClass; + bucket = storage.UpdateBucket(bucket); + var transferJob = transferToNearlineSample.TransferToNearline(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink); + Assert.Contains("transferJobs/", transferJob.Name); + _transferJobName = transferJob.Name; + } + + public void Dispose() + { + try + { + _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest() + { + ProjectId = _fixture.ProjectId, + JobName = _transferJobName, + TransferJob = new TransferJob() + { + Name = _transferJobName, + Status = TransferJob.Types.Status.Deleted + } + }); + } + catch (Exception) + { + // Do nothing, we delete on a best effort basis. + } + } +} diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs new file mode 100644 index 00000000000..c62ec5eb74d --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -0,0 +1,68 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// [START storagetransfer_transfer_to_nearline] + +using Google.Cloud.StorageTransfer.V1; +using Google.Protobuf.WellKnownTypes; +using System; + + +namespace StorageTransfer.Samples +{ + + public class TransferToNearlineSample + { + /*Creates a one-off transfer job that transfers objects from a standard GCS bucket that are more + than 30 days old to a Nearline GCS bucket.*/ + public TransferJob TransferToNearline( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The GCS bucket to transfer objects from + string sourceBucket = "my-source-bucket", + // The GCS Nearline bucket to transfer old objects to + string sinkBucket = "my-sink-bucket") + { + // A description of this job + string jobDescription = $"Transfers old objects from standard bucket ({sourceBucket}) that haven't been modified in the last 30 days to a Nearline bucket ({sinkBucket})"; + + TransferJob transferJob = new TransferJob + { + ProjectId = projectId, + Description = jobDescription, + TransferSpec = new TransferSpec + { + GcsDataSink = new GcsData { BucketName = sinkBucket }, + GcsDataSource = new GcsData { BucketName = sourceBucket }, + ObjectConditions = new ObjectConditions { MinTimeElapsedSinceLastModification = Duration.FromTimeSpan(TimeSpan.FromSeconds(2592000)) }, + TransferOptions = new TransferOptions { DeleteObjectsFromSourceAfterTransfer = true }, + }, + Status = TransferJob.Types.Status.Enabled, + Schedule = new Schedule { ScheduleStartDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)) } + }; + // Create a Transfer Service client + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + // Create a Transfer job + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + client.RunTransferJob(new RunTransferJobRequest + { + JobName = response.Name, + ProjectId = projectId + }); + + Console.WriteLine($"Created transfer job from standard bucket {sourceBucket} to Nearline bucket {sinkBucket} with name {response.Name}"); + return response; + } + } +} +// [END storagetransfer_transfer_to_nearline] From 36706a44a3919436d4c8515d19afc9436f167998 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 9 Oct 2024 02:17:01 -0700 Subject: [PATCH 02/58] feat(Storage Transfer) : Addition of Schedule End Date in Sample for storagetransfer_transfer_to_nearline. --- .../api/StorageTransfer.Samples/TransferToNearlineSample.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index c62ec5eb74d..7b50d3e015e 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -48,7 +48,7 @@ public TransferJob TransferToNearline( TransferOptions = new TransferOptions { DeleteObjectsFromSourceAfterTransfer = true }, }, Status = TransferJob.Types.Status.Enabled, - Schedule = new Schedule { ScheduleStartDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)) } + Schedule = new Schedule { ScheduleStartDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)), ScheduleEndDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)) } }; // Create a Transfer Service client StorageTransferServiceClient client = StorageTransferServiceClient.Create(); @@ -60,7 +60,7 @@ public TransferJob TransferToNearline( ProjectId = projectId }); - Console.WriteLine($"Created transfer job from standard bucket {sourceBucket} to Nearline bucket {sinkBucket} with name {response.Name}"); + Console.WriteLine($"Created one-off transfer job from standard bucket {sourceBucket} to Nearline bucket {sinkBucket} with name {response.Name}"); return response; } } From afcba298ce660c521a296791ed6f1c28098f10a6 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 9 Oct 2024 02:17:01 -0700 Subject: [PATCH 03/58] samples(storage transfer) : Addition of schedule end date in sample for storagetransfer_transfer_to_nearline. --- .../api/StorageTransfer.Samples/TransferToNearlineSample.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index c62ec5eb74d..7b50d3e015e 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -48,7 +48,7 @@ public TransferJob TransferToNearline( TransferOptions = new TransferOptions { DeleteObjectsFromSourceAfterTransfer = true }, }, Status = TransferJob.Types.Status.Enabled, - Schedule = new Schedule { ScheduleStartDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)) } + Schedule = new Schedule { ScheduleStartDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)), ScheduleEndDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)) } }; // Create a Transfer Service client StorageTransferServiceClient client = StorageTransferServiceClient.Create(); @@ -60,7 +60,7 @@ public TransferJob TransferToNearline( ProjectId = projectId }); - Console.WriteLine($"Created transfer job from standard bucket {sourceBucket} to Nearline bucket {sinkBucket} with name {response.Name}"); + Console.WriteLine($"Created one-off transfer job from standard bucket {sourceBucket} to Nearline bucket {sinkBucket} with name {response.Name}"); return response; } } From 3efcf764d5d7521dd8f60c335be6346bc889637d Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 10 Oct 2024 02:55:33 -0700 Subject: [PATCH 04/58] chore(deps): add dependency xunit.abstractions version 2.0.3 to see output message in sample output and related small changes. --- .../QuickstartTest.cs | 11 +++++++---- .../QuickstartSample.cs | 18 +++++++++++------- .../StorageTransfer.Samples.csproj | 1 + 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs index 30cac132e63..5b96b7519c6 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs @@ -1,5 +1,5 @@ -/** - * Copyright 2021 Google Inc. +/** + * Copyright 2024 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ using System; using Google.Cloud.StorageTransfer.V1; using Xunit; +using Xunit.Abstractions; namespace StorageTransfer.Samples.Tests { @@ -25,16 +26,18 @@ public class QuickstartTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; + private readonly ITestOutputHelper _outputHelper; - public QuickstartTest(StorageFixture fixture) + public QuickstartTest(StorageFixture fixture , ITestOutputHelper outputHelper) { _fixture = fixture; + _outputHelper = outputHelper; } [Fact] public void TestQuickstart() { - QuickstartSample quickstartSample = new QuickstartSample(); + QuickstartSample quickstartSample = new QuickstartSample(_outputHelper); var transferJob = quickstartSample.Quickstart(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs index cc84bfe0620..68e60844d0d 100644 --- a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs @@ -1,5 +1,5 @@ -/** - * Copyright 2021 Google Inc. +/** + * Copyright 2024 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,15 +14,19 @@ * limitations under the License. */ - // [START storagetransfer_quickstart] -using System; using Google.Cloud.StorageTransfer.V1; +using Xunit.Abstractions; namespace StorageTransfer.Samples { public class QuickstartSample { + private readonly ITestOutputHelper _output; + public QuickstartSample(ITestOutputHelper output) + { + _output = output; + } public TransferJob Quickstart( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -36,8 +40,8 @@ public TransferJob Quickstart( ProjectId = projectId, TransferSpec = new TransferSpec { - GcsDataSink = new GcsData { BucketName = sourceBucket }, - GcsDataSource = new GcsData { BucketName = sinkBucket } + GcsDataSink = new GcsData { BucketName = sinkBucket }, + GcsDataSource = new GcsData { BucketName = sourceBucket } }, Status = TransferJob.Types.Status.Enabled }; @@ -50,7 +54,7 @@ public TransferJob Quickstart( ProjectId = projectId }); - Console.WriteLine($"Created and ran transfer job from {sourceBucket} to {sinkBucket} with name {response.Name}"); + _output.WriteLine($"Created and ran transfer job from {sourceBucket} to {sinkBucket} with name {response.Name}"); return response; } diff --git a/storagetransfer/api/StorageTransfer.Samples/StorageTransfer.Samples.csproj b/storagetransfer/api/StorageTransfer.Samples/StorageTransfer.Samples.csproj index cb35bdd48fa..2b68373281f 100644 --- a/storagetransfer/api/StorageTransfer.Samples/StorageTransfer.Samples.csproj +++ b/storagetransfer/api/StorageTransfer.Samples/StorageTransfer.Samples.csproj @@ -6,5 +6,6 @@ + From 35e18a36bb880ab70a9205a7e732df3059c98cc3 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 10 Oct 2024 04:37:44 -0700 Subject: [PATCH 05/58] code changes in transfer to nearline related to capture message in standard output of test log summary. --- .../TransferToNearlineTest.cs | 7 +++++-- .../StorageTransfer.Samples/TransferToNearlineSample.cs | 8 +++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs index bf688b6eb7f..c3d1856f192 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -17,6 +17,7 @@ using Google.Cloud.StorageTransfer.V1; using System; using Xunit; +using Xunit.Abstractions; namespace StorageTransfer.Samples.Tests; @@ -25,15 +26,17 @@ public class TransferToNearlineTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; - public TransferToNearlineTest(StorageFixture fixture) + private readonly ITestOutputHelper _outputHelper; + public TransferToNearlineTest(StorageFixture fixture, ITestOutputHelper outputHelper) { _fixture = fixture; + _outputHelper = outputHelper; } [Fact] public void TestTransferToNearline() { - TransferToNearlineSample transferToNearlineSample = new TransferToNearlineSample(); + TransferToNearlineSample transferToNearlineSample = new TransferToNearlineSample(_outputHelper); var storage = StorageClient.Create(); var bucket = storage.GetBucket(_fixture.BucketNameSink); string storageClass = StorageClasses.Nearline; diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index 7b50d3e015e..2d7238eea76 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -16,6 +16,7 @@ using Google.Cloud.StorageTransfer.V1; using Google.Protobuf.WellKnownTypes; using System; +using Xunit.Abstractions; namespace StorageTransfer.Samples @@ -25,6 +26,11 @@ public class TransferToNearlineSample { /*Creates a one-off transfer job that transfers objects from a standard GCS bucket that are more than 30 days old to a Nearline GCS bucket.*/ + private readonly ITestOutputHelper _output; + public TransferToNearlineSample(ITestOutputHelper output) + { + _output = output; + } public TransferJob TransferToNearline( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -60,7 +66,7 @@ public TransferJob TransferToNearline( ProjectId = projectId }); - Console.WriteLine($"Created one-off transfer job from standard bucket {sourceBucket} to Nearline bucket {sinkBucket} with name {response.Name}"); + _output.WriteLine($"Created one-off transfer job from standard bucket {sourceBucket} to Nearline bucket {sinkBucket} with the name {response.Name}"); return response; } } From 0acb7fdb5d1a113ec95fbb0d20fc50126e6a953a Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 10 Oct 2024 07:14:02 -0700 Subject: [PATCH 06/58] feat(storage transfer) : addition of sample and test case to check latest transfer operation. --- .../CheckLatestTransferOperationTest.cs | 40 +++++++++++ .../StorageFixture.cs | 7 +- .../CheckLatestTransferOperationSample.cs | 70 +++++++++++++++++++ 3 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs create mode 100644 storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs new file mode 100644 index 00000000000..4790fc473b4 --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs @@ -0,0 +1,40 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +using Xunit; +using Xunit.Abstractions; + +namespace StorageTransfer.Samples.Tests; + +[Collection(nameof(StorageFixture))] +public class CheckLatestTransferOperationTest +{ + + private readonly StorageFixture _fixture; + private string _jobName; + private readonly ITestOutputHelper _outputHelper; + public CheckLatestTransferOperationTest(StorageFixture fixture, ITestOutputHelper outputHelper) + { + _outputHelper = outputHelper; + _fixture = fixture; + } + + [Fact] + public void CheckLatestTransferOperation() + { + CheckLatestTransferOperationSample checkLatestTransferOperationSample = new CheckLatestTransferOperationSample(_outputHelper); + var transferJob = checkLatestTransferOperationSample.CheckLatestTransferOperation(_fixture.ProjectId,_fixture.JobName); + Assert.Contains("transferJobs/", transferJob.Name); + _jobName = transferJob.Name; + } +} diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index 2938516a605..08e7c9ce72f 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -1,4 +1,4 @@ -/** +/** * Copyright 2021 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,11 +28,16 @@ public class StorageFixture : IDisposable, ICollectionFixture public string ProjectId { get; } public string BucketNameSource { get; } = Guid.NewGuid().ToString(); public string BucketNameSink { get; } = Guid.NewGuid().ToString(); + public string JobName { get; } public StorageClient Storage { get; } = StorageClient.Create(); public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create(); public StorageFixture() { + // Instantiate random number generator + Random random = new Random(); + JobName = "transferJobs/" + random.NextInt64(1000000000000000, 9223372036854775807) + " "; + ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); if (string.IsNullOrWhiteSpace(ProjectId)) { diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs new file mode 100644 index 00000000000..c8f447e3b91 --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -0,0 +1,70 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// [START storagetransfer_get_latest_transfer_operation] + +using Google.Cloud.StorageTransfer.V1; +using System; +using Xunit.Abstractions; + +namespace StorageTransfer.Samples +{ + public class CheckLatestTransferOperationSample + { + private readonly ITestOutputHelper _output; + public CheckLatestTransferOperationSample(ITestOutputHelper output) + { + _output = output; + + } + //Checks the latest transfer operation for a given transfer job. + public TransferJob CheckLatestTransferOperation( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The name of the job to check + string jobName = "transferJobs/1234567890") + { + if(string.IsNullOrEmpty(jobName)) + { + throw new Exception("JobName can not be null or empty"); + } + // Create a Transfer Service client + StorageTransferServiceClient storageTransfer = StorageTransferServiceClient.Create(); + + GetTransferJobRequest getTransferJobRequest = new GetTransferJobRequest { ProjectId = projectId, JobName = jobName }; + try + { + // Get Transfer job + TransferJob transferJob = storageTransfer.GetTransferJob(getTransferJobRequest); + // Get Latest operation name from tranfer job + string latestOperationName = transferJob.LatestOperationName; + + + if (!string.IsNullOrEmpty(latestOperationName)) + { + _output.WriteLine("The latest operation for transfer job " +jobName+ " is: " +latestOperationName+ ""); + } + else + { + _output.WriteLine("Transfer job "+ jobName +" hasn't run yet, try again once after job started running."); + } + return transferJob; + } + catch (Exception) + { + throw new Exception("Failed to get transfer job "+ jobName + ""); + } + } + } +} +// [END storagetransfer_get_latest_transfer_operation] From b5d9b10f9cbd0ad35d8e78a91b08e5a8ff1c9fc1 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Tue, 29 Oct 2024 06:14:42 -0700 Subject: [PATCH 07/58] storage transfer from POSIX file system to GCS bucket using manifest file --- .../StorageFixture.cs | 20 ++++- .../TransferUsingManifestTest.cs | 72 +++++++++++++++ .../TransferUsingManifestSample.cs | 89 +++++++++++++++++++ 3 files changed, 179 insertions(+), 2 deletions(-) create mode 100644 storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs create mode 100644 storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index 08e7c9ce72f..9ba481fa175 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -29,7 +29,10 @@ public class StorageFixture : IDisposable, ICollectionFixture public string BucketNameSource { get; } = Guid.NewGuid().ToString(); public string BucketNameSink { get; } = Guid.NewGuid().ToString(); public string JobName { get; } + public string SourceAgentPoolName { get; } + public string RootDirectory { get; } = "/tmp/uploads"; public StorageClient Storage { get; } = StorageClient.Create(); + public string ManifestObjectName { get; } = "manifest.csv"; public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create(); public StorageFixture() @@ -37,6 +40,7 @@ public StorageFixture() // Instantiate random number generator Random random = new Random(); JobName = "transferJobs/" + random.NextInt64(1000000000000000, 9223372036854775807) + " "; + SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/test_dotnet"; ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); if (string.IsNullOrWhiteSpace(ProjectId)) @@ -102,7 +106,13 @@ public void Dispose() } catch (Exception) { - // Do nothing, we delete on a best effort basis. + // If bucket is not empty, we delete on a best effort basis. + foreach (var storageObject in Storage.ListObjects(BucketNameSink, "")) + { + Storage.DeleteObject(BucketNameSink, storageObject.Name); + + } + Storage.DeleteBucket(BucketNameSink); } try { @@ -110,7 +120,13 @@ public void Dispose() } catch (Exception) { - // Do nothing, we delete on a best effort basis. + // If bucket is not empty, we delete on a best effort basis. + foreach (var storageObject in Storage.ListObjects(BucketNameSource, "")) + { + Storage.DeleteObject(BucketNameSource, storageObject.Name); + + } + Storage.DeleteBucket(BucketNameSource); } } } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs new file mode 100644 index 00000000000..5cd2f4f150e --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -0,0 +1,72 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Google.Cloud.Storage.V1; +using Google.Cloud.StorageTransfer.V1; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Xunit.Abstractions; +using Xunit; +using System.IO; + +namespace StorageTransfer.Samples.Tests; +[Collection(nameof(StorageFixture))] +public class TransferUsingManifestTest : IDisposable +{ + private readonly StorageFixture _fixture; + private string _transferJobName; + private readonly ITestOutputHelper _outputHelper; + public TransferUsingManifestTest(StorageFixture fixture, ITestOutputHelper outputHelper) + { + _fixture = fixture; + _outputHelper = outputHelper; + } + + [Fact] + public void TransferUsingManifest() + { + TransferUsingManifestSample transferUsingManifestSample = new TransferUsingManifestSample(_outputHelper); + var storage = StorageClient.Create(); + byte[] byteArray = Encoding.UTF8.GetBytes("flower.jpeg"); + MemoryStream stream = new MemoryStream(byteArray); + storage.UploadObject(_fixture.BucketNameSource, _fixture.ManifestObjectName, "application/octet-stream", stream); + var transferJob = transferUsingManifestSample.TransferUsingManifest(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.RootDirectory, _fixture.BucketNameSource, _fixture.BucketNameSink, _fixture.ManifestObjectName); + Assert.Contains("transferJobs/", transferJob.Name); + _transferJobName = transferJob.Name; + } + + public void Dispose() + { + try + { + _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest() + { + ProjectId = _fixture.ProjectId, + JobName = _transferJobName, + TransferJob = new TransferJob() + { + Name = _transferJobName, + Status = TransferJob.Types.Status.Deleted + } + }); + } + catch (Exception) + { + // Do nothing, we delete on a best effort basis. + } + } +} diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs new file mode 100644 index 00000000000..2f7b2102953 --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -0,0 +1,89 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// [START storagetransfer_manifest_request] +using Google.Cloud.StorageTransfer.V1; +using Google.Protobuf.WellKnownTypes; +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using Xunit.Abstractions; + + +namespace StorageTransfer.Samples +{ + public class TransferUsingManifestSample + { + /*Create a transfer from a POSIX file system to a GCS bucket using + a manifest file*/ + private readonly ITestOutputHelper _output; + public TransferUsingManifestSample(ITestOutputHelper output) + { + _output = output; + } + public TransferJob TransferUsingManifest( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The agent pool associated with the POSIX data source. If not provided, defaults to the default agent + string sourceAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", + // The root directory path on the source filesystem + string rootDirectory = "/tmp/uploads", + // The GCS bucket which has your manifest file + string manifestBucket = "my-source-bucket", + // The GCS bucket to transfer data to + string sinkBucket = "my-sink-bucket", + // The name of the manifest file in manifestBucket that specifies which objects to transfer + string manifestObjectName = "path/to/manifest.csv") + { + string manifestLocation = "gs://" + manifestBucket + "/" + manifestObjectName; + + // # A useful description for your transfer job + string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket}) using manifest file"; + + TransferJob transferJob = new TransferJob + { + ProjectId = projectId, + Description = jobDescription, + TransferSpec = new TransferSpec + { + GcsDataSink = new GcsData { BucketName = sinkBucket }, + GcsDataSource = new GcsData { BucketName = manifestBucket }, + SourceAgentPoolName = sourceAgentPoolName, + PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory }, + TransferManifest = new TransferManifest { Location = manifestLocation } + }, + Status = TransferJob.Types.Status.Enabled, + }; + + + // Create a Transfer Service client + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + + // Create a Transfer job + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + + client.RunTransferJob(new RunTransferJobRequest + { + JobName = response.Name, + ProjectId = projectId + }); + + _output.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} using manifest file {manifestLocation} with the name {response.Name}"); + return response; + + + } + } +} +// [END storagetransfer_manifest_request] From 3c9d243bc0e425939be7b325e1d24e369c454829 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 30 Oct 2024 04:23:27 -0700 Subject: [PATCH 08/58] minor changes --- .../api/StorageTransfer.Samples.Tests/StorageFixture.cs | 9 +++------ .../TransferUsingManifestTest.cs | 3 --- .../TransferUsingManifestSample.cs | 5 ----- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index 9ba481fa175..feca9a4068a 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -1,5 +1,5 @@ /** - * Copyright 2021 Google Inc. + * Copyright 2024 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -39,10 +39,9 @@ public StorageFixture() { // Instantiate random number generator Random random = new Random(); - JobName = "transferJobs/" + random.NextInt64(1000000000000000, 9223372036854775807) + " "; - SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/test_dotnet"; - + JobName = "transferJobs/" + random.NextInt64(1000000000000000, 9223372036854775807) + " "; ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); + SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/test_dotnet"; if (string.IsNullOrWhiteSpace(ProjectId)) { throw new Exception("You need to set the Environment variable 'GOOGLE_PROJECT_ID' with your Google Cloud Project's project id."); @@ -110,7 +109,6 @@ public void Dispose() foreach (var storageObject in Storage.ListObjects(BucketNameSink, "")) { Storage.DeleteObject(BucketNameSink, storageObject.Name); - } Storage.DeleteBucket(BucketNameSink); } @@ -124,7 +122,6 @@ public void Dispose() foreach (var storageObject in Storage.ListObjects(BucketNameSource, "")) { Storage.DeleteObject(BucketNameSource, storageObject.Name); - } Storage.DeleteBucket(BucketNameSource); } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index 5cd2f4f150e..c19f33a4fd4 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -15,10 +15,7 @@ using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using System; -using System.Collections.Generic; -using System.Linq; using System.Text; -using System.Threading.Tasks; using Xunit.Abstractions; using Xunit; using System.IO; diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index 2f7b2102953..5625a17177e 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -13,11 +13,6 @@ // limitations under the License. // [START storagetransfer_manifest_request] using Google.Cloud.StorageTransfer.V1; -using Google.Protobuf.WellKnownTypes; -using System; -using System.Collections.Generic; -using System.IO; -using System.Text; using Xunit.Abstractions; From 375f1e011f7447be278345dbdf78d7e842dee696 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 31 Oct 2024 09:28:27 +0000 Subject: [PATCH 09/58] addition of sample and test case for transfer from posix --- .../TransferFromPosixTest.cs | 65 ++++++++++++++++ .../TransferFromPosixSample.cs | 78 +++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs create mode 100644 storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs new file mode 100644 index 00000000000..6f512b11fc2 --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs @@ -0,0 +1,65 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Google.Cloud.Storage.V1; +using Google.Cloud.StorageTransfer.V1; +using System; +using Xunit.Abstractions; +using Xunit; + + +namespace StorageTransfer.Samples.Tests; +[Collection(nameof(StorageFixture))] +public class TransferFromPosixTest : IDisposable +{ + private readonly StorageFixture _fixture; + private string _transferJobName; + private readonly ITestOutputHelper _outputHelper; + public TransferFromPosixTest(StorageFixture fixture, ITestOutputHelper outputHelper) + { + _fixture = fixture; + _outputHelper = outputHelper; + } + + [Fact] + public void TransferFromPosix() + { + TransferFromPosixSample transferFromPosixSample = new TransferFromPosixSample(_outputHelper); + var transferJob = transferFromPosixSample.TransferFromPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.RootDirectory,_fixture.BucketNameSink); + var storage = StorageClient.Create(); + Assert.Contains("transferJobs/", transferJob.Name); + _transferJobName = transferJob.Name; + } + + public void Dispose() + { + try + { + _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest() + { + ProjectId = _fixture.ProjectId, + JobName = _transferJobName, + TransferJob = new TransferJob() + { + Name = _transferJobName, + Status = TransferJob.Types.Status.Deleted + } + }); + } + catch (Exception) + { + // Do nothing, we delete on a best effort basis. + } + } +} diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs new file mode 100644 index 00000000000..6bf91095722 --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs @@ -0,0 +1,78 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// [START storagetransfer_transfer_from_posix] +using Google.Cloud.StorageTransfer.V1; +using Xunit.Abstractions; + + +namespace StorageTransfer.Samples +{ + public class TransferFromPosixSample + { + /*Create a transfer from a POSIX file system to a GCS sink bucket*/ + private readonly ITestOutputHelper _output; + public TransferFromPosixSample(ITestOutputHelper output) + { + _output = output; + } + public TransferJob TransferFromPosix( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The agent pool associated with the POSIX data source. If not provided, defaults to the default agent + string sourceAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", + // The root directory path on the source filesystem + string rootDirectory = "/tmp/uploads", + // The GCS bucket to transfer data to + string sinkBucket = "my-sink-bucket") + { + + // # A useful description for your transfer job + string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket})"; + + TransferJob transferJob = new TransferJob + { + ProjectId = projectId, + Description = jobDescription, + TransferSpec = new TransferSpec + { + GcsDataSink = new GcsData { BucketName = sinkBucket }, + SourceAgentPoolName = sourceAgentPoolName, + PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory } + }, + Status = TransferJob.Types.Status.Enabled, + }; + + + // Create a Transfer Service client + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + + // Create a Transfer job + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + + client.RunTransferJob(new RunTransferJobRequest + { + JobName = response.Name, + ProjectId = projectId + }); + + _output.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} with the name {response.Name}"); + return response; + + + } + } +} +// [END storagetransfer_transfer_from_posix] + + From 139190a621408f413c9fcf01096d7320a23afaff Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Mon, 4 Nov 2024 11:10:29 +0000 Subject: [PATCH 10/58] added sample and test case for download to posix storage transfer service. --- .../DownloadToPosixTest.cs | 70 ++++++++++++++++ .../StorageFixture.cs | 6 +- .../DownloadToPosixSample.cs | 79 +++++++++++++++++++ 3 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs create mode 100644 storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs new file mode 100644 index 00000000000..f5aedff1291 --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -0,0 +1,70 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Google.Cloud.Storage.V1; +using Google.Cloud.StorageTransfer.V1; +using System; +using Xunit.Abstractions; +using Xunit; +using System.Text; +using System.IO; + + +namespace StorageTransfer.Samples.Tests; +[Collection(nameof(StorageFixture))] +public class DownloadToPosixTest : IDisposable +{ + private readonly StorageFixture _fixture; + private string _transferJobName; + private readonly ITestOutputHelper _outputHelper; + public DownloadToPosixTest(StorageFixture fixture, ITestOutputHelper outputHelper) + { + _fixture = fixture; + _outputHelper = outputHelper; + } + + [Fact] + public void DownloadToPosix() + { + DownloadToPosixSample downloadToPosixSample = new DownloadToPosixSample(_outputHelper); + var storage = StorageClient.Create(); + byte[] byteArray = Encoding.UTF8.GetBytes("flower.jpeg"); + MemoryStream stream = new MemoryStream(byteArray); + storage.UploadObject(_fixture.BucketNameSource,"DownloadToPosixTestFile", "application/octet-stream", stream); + var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId,_fixture.SinkAgentPoolName,_fixture.BucketNameSource,_fixture.GcsSourcePath,_fixture.RootDirectory); + Assert.Contains("transferJobs/", transferJob.Name); + _transferJobName = transferJob.Name; + } + + public void Dispose() + { + try + { + _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest() + { + ProjectId = _fixture.ProjectId, + JobName = _transferJobName, + TransferJob = new TransferJob() + { + Name = _transferJobName, + Status = TransferJob.Types.Status.Deleted + } + }); + } + catch (Exception) + { + // Do nothing, we delete on a best effort basis. + } + } +} diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index feca9a4068a..4e19c17d07f 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -30,6 +30,8 @@ public class StorageFixture : IDisposable, ICollectionFixture public string BucketNameSink { get; } = Guid.NewGuid().ToString(); public string JobName { get; } public string SourceAgentPoolName { get; } + public string SinkAgentPoolName { get; } + public string GcsSourcePath { get;} public string RootDirectory { get; } = "/tmp/uploads"; public StorageClient Storage { get; } = StorageClient.Create(); public string ManifestObjectName { get; } = "manifest.csv"; @@ -41,7 +43,9 @@ public StorageFixture() Random random = new Random(); JobName = "transferJobs/" + random.NextInt64(1000000000000000, 9223372036854775807) + " "; ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); - SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/test_dotnet"; + SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/source_test_dotnet"; + SinkAgentPoolName = "projects/" + ProjectId + "/agentPools/sink_test_dotnet"; + GcsSourcePath = "foo/bar/"; if (string.IsNullOrWhiteSpace(ProjectId)) { throw new Exception("You need to set the Environment variable 'GOOGLE_PROJECT_ID' with your Google Cloud Project's project id."); diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs new file mode 100644 index 00000000000..3a7e114d51a --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -0,0 +1,79 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// [START storagetransfer_download_to_posix] +using Google.Cloud.StorageTransfer.V1; +using Xunit.Abstractions; + + +namespace StorageTransfer.Samples +{ + public class DownloadToPosixSample + { + /*Create a transfer from a GCS bucket to a POSIX file system.*/ + private readonly ITestOutputHelper _output; + public DownloadToPosixSample(ITestOutputHelper output) + { + _output = output; + } + public TransferJob DownloadToPosix( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent + string sinkAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", + // Your GCS source bucket name + string gcsSourceBucket = "my-gcs-source-bucket", + // An optional path on the Google Cloud Storage bucket to download from + string gcsSourcePath = "foo/bar/", + // The root directory path on the source filesystem + string rootDirectory = "/tmp/uploads") + { + // # A useful description for your transfer job + string jobDescription = $"Download objects from a GCS source bucket ({gcsSourceBucket}/{gcsSourcePath}) to the root directory of POSIX file system"; + + TransferJob transferJob = new TransferJob + { + ProjectId = projectId, + Description = jobDescription, + TransferSpec = new TransferSpec + { + GcsDataSource = new GcsData { BucketName = gcsSourceBucket, Path = gcsSourcePath}, + SinkAgentPoolName = sinkAgentPoolName, + PosixDataSink = new PosixFilesystem { RootDirectory = rootDirectory } + }, + Status = TransferJob.Types.Status.Enabled, + }; + + + // Create a Transfer Service client + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + + // Create a Transfer job + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + + client.RunTransferJob(new RunTransferJobRequest + { + JobName = response.Name, + ProjectId = projectId + }); + + _output.WriteLine($"Created and ran transfer job from ({gcsSourceBucket}/{gcsSourcePath}) to {rootDirectory} with the name {response.Name}"); + return response; + + + } + } +} +//[END storagetransfer_download_to_posix] + + From bb66a633f26680d8916342851b25e17b3bd7b9b7 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Tue, 5 Nov 2024 11:58:18 +0000 Subject: [PATCH 11/58] code changes for creation of temp folder in posix file system --- .../StorageTransfer.Samples.Tests/DownloadToPosixTest.cs | 7 +++++-- .../api/StorageTransfer.Samples.Tests/StorageFixture.cs | 8 +++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index f5aedff1291..9edf6dbf59b 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -38,11 +38,13 @@ public DownloadToPosixTest(StorageFixture fixture, ITestOutputHelper outputHelpe public void DownloadToPosix() { DownloadToPosixSample downloadToPosixSample = new DownloadToPosixSample(_outputHelper); + Directory.CreateDirectory(_fixture.TempDirectory); var storage = StorageClient.Create(); byte[] byteArray = Encoding.UTF8.GetBytes("flower.jpeg"); MemoryStream stream = new MemoryStream(byteArray); - storage.UploadObject(_fixture.BucketNameSource,"DownloadToPosixTestFile", "application/octet-stream", stream); - var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId,_fixture.SinkAgentPoolName,_fixture.BucketNameSource,_fixture.GcsSourcePath,_fixture.RootDirectory); + string fileName = $"{_fixture.GcsSourcePath}{DateTime.Now.ToString("yyyyMMddHHmmss")}.txt"; + storage.UploadObject(_fixture.BucketNameSource,fileName, "application/octet-stream", stream); + var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId,_fixture.SinkAgentPoolName,_fixture.BucketNameSource,_fixture.GcsSourcePath,_fixture.TempDirectory); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } @@ -61,6 +63,7 @@ public void Dispose() Status = TransferJob.Types.Status.Deleted } }); + Directory.Delete(_fixture.TempDirectory, true); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index 4e19c17d07f..51019127d8c 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -16,6 +16,7 @@ using System; using System.Collections.Generic; +using System.IO; using Google.Apis.Storage.v1.Data; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; @@ -32,7 +33,8 @@ public class StorageFixture : IDisposable, ICollectionFixture public string SourceAgentPoolName { get; } public string SinkAgentPoolName { get; } public string GcsSourcePath { get;} - public string RootDirectory { get; } = "/tmp/uploads"; + public string RootDirectory { get; } = System.IO.Path.GetTempPath(); + public string TempDirectory { get; } = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); public StorageClient Storage { get; } = StorageClient.Create(); public string ManifestObjectName { get; } = "manifest.csv"; public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create(); @@ -43,8 +45,8 @@ public StorageFixture() Random random = new Random(); JobName = "transferJobs/" + random.NextInt64(1000000000000000, 9223372036854775807) + " "; ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); - SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/source_test_dotnet"; - SinkAgentPoolName = "projects/" + ProjectId + "/agentPools/sink_test_dotnet"; + SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/transfer_service_default"; + SinkAgentPoolName = "projects/" + ProjectId + "/agentPools/transfer_service_default"; GcsSourcePath = "foo/bar/"; if (string.IsNullOrWhiteSpace(ProjectId)) { From 526b4452d1aa49cca75c7b9569a104116e1f698e Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 6 Nov 2024 07:02:48 +0000 Subject: [PATCH 12/58] couple of test checks added in download to posix --- .../StorageTransfer.Samples.Tests/DownloadToPosixTest.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index 9edf6dbf59b..cb1abf8769e 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -19,6 +19,8 @@ using Xunit; using System.Text; using System.IO; +using System.Linq; +using System.Threading.Tasks; namespace StorageTransfer.Samples.Tests; @@ -43,9 +45,13 @@ public void DownloadToPosix() byte[] byteArray = Encoding.UTF8.GetBytes("flower.jpeg"); MemoryStream stream = new MemoryStream(byteArray); string fileName = $"{_fixture.GcsSourcePath}{DateTime.Now.ToString("yyyyMMddHHmmss")}.txt"; + string filePath = $"{_fixture.TempDirectory}/{fileName.Split('/').Last()}"; storage.UploadObject(_fixture.BucketNameSource,fileName, "application/octet-stream", stream); var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId,_fixture.SinkAgentPoolName,_fixture.BucketNameSource,_fixture.GcsSourcePath,_fixture.TempDirectory); Assert.Contains("transferJobs/", transferJob.Name); + Assert.True( Directory.Exists(_fixture.TempDirectory)); + System.Threading.Thread.Sleep(TimeSpan.FromSeconds(45)); + Assert.True( File.Exists(filePath)); _transferJobName = transferJob.Name; } From edc89e412bbd35cc9cebf9cdb7f1141e956be953 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 6 Nov 2024 13:58:32 +0000 Subject: [PATCH 13/58] added sample and test case for posix to posix storage transfer --- .../StorageFixture.cs | 2 + .../TransferBetweenPosixTest.cs | 88 +++++++++++++++++++ .../TransferBetweenPosixSample.cs | 82 +++++++++++++++++ 3 files changed, 172 insertions(+) create mode 100644 storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs create mode 100644 storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index 51019127d8c..22b43f7ea8e 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -34,7 +34,9 @@ public class StorageFixture : IDisposable, ICollectionFixture public string SinkAgentPoolName { get; } public string GcsSourcePath { get;} public string RootDirectory { get; } = System.IO.Path.GetTempPath(); + public string DestinationDirectory { get; } = System.IO.Path.GetTempPath(); public string TempDirectory { get; } = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + public string TempDestinationDirectory { get; } = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); public StorageClient Storage { get; } = StorageClient.Create(); public string ManifestObjectName { get; } = "manifest.csv"; public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create(); diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs new file mode 100644 index 00000000000..e8fd90bc63b --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -0,0 +1,88 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using Google.Cloud.Storage.V1; +using Google.Cloud.StorageTransfer.V1; +using System; +using Xunit.Abstractions; +using Xunit; +using System.IO; +using System.Linq; + + +namespace StorageTransfer.Samples.Tests; +[Collection(nameof(StorageFixture))] +public class TransferBetweenPosixTest : IDisposable +{ + private readonly StorageFixture _fixture; + private string _transferJobName; + private readonly ITestOutputHelper _outputHelper; + public TransferBetweenPosixTest(StorageFixture fixture, ITestOutputHelper outputHelper) + { + _fixture = fixture; + _outputHelper = outputHelper; + } + + [Fact] + public void TransferBetweenPosix() + { + TransferBetweenPosixSample transferBetweenPosixSample = new TransferBetweenPosixSample(_outputHelper); + Directory.CreateDirectory(_fixture.TempDirectory); + Directory.CreateDirectory(_fixture.TempDestinationDirectory); + string sourceDir =_fixture.RootDirectory; + string[] txtList = Directory.GetFiles(sourceDir, "*.txt"); + // Copy one txt file. + foreach (string f in txtList) + { + // Remove path from the file name. + string fName = f.Split('/').Last(); + // Copy only one file from source directory to temp source directory. Will overwrite if the destination file already exists. + File.Copy(Path.Combine(sourceDir, fName), Path.Combine(_fixture.TempDirectory, fName), true); + break; + } + var storage = StorageClient.Create(); + var transferJob = transferBetweenPosixSample.TransferBetweenPosix(_fixture.ProjectId,_fixture.SourceAgentPoolName,_fixture.SinkAgentPoolName,_fixture.TempDirectory,_fixture.TempDestinationDirectory,_fixture.BucketNameSource); + Assert.Contains("transferJobs/", transferJob.Name); + Assert.True(Directory.Exists(_fixture.TempDirectory)); + Assert.True(Directory.Exists(_fixture.TempDestinationDirectory)); + Assert.True(File.Exists(txtList[0])); + System.Threading.Thread.Sleep(TimeSpan.FromSeconds(45)); + string destinationFilePath = txtList[0].Replace(_fixture.RootDirectory.Substring(0,_fixture.RootDirectory.Length - 1),_fixture.TempDestinationDirectory); + Assert.True(File.Exists(destinationFilePath)); + _transferJobName = transferJob.Name; + } + + public void Dispose() + { + try + { + _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest() + { + ProjectId = _fixture.ProjectId, + JobName = _transferJobName, + TransferJob = new TransferJob() + { + Name = _transferJobName, + Status = TransferJob.Types.Status.Deleted + } + }); + Directory.Delete(_fixture.TempDirectory, true); + Directory.Delete(_fixture.TempDestinationDirectory, true); + } + catch (Exception) + { + // Do nothing, we delete on a best effort basis. + } + } +} diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs new file mode 100644 index 00000000000..3cbc42dd7a0 --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -0,0 +1,82 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"). +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// [START storagetransfer_transfer_posix_to_posix] +using Google.Cloud.StorageTransfer.V1; +using Xunit.Abstractions; + + +namespace StorageTransfer.Samples +{ + public class TransferBetweenPosixSample + { + /*Creates a transfer between POSIX file systems.*/ + private readonly ITestOutputHelper _output; + public TransferBetweenPosixSample(ITestOutputHelper output) + { + _output = output; + } + public TransferJob TransferBetweenPosix( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The agent pool associated with the POSIX data source. If not provided, defaults to the default agent + string sourceAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", + // The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent + string sinkAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", + // The root directory path on the source filesystem + string rootDirectory = "/tmp/uploads", + // The root directory path on the sink filesystem + string destinationDirectory = "/directory/to/transfer/sink", + // The name of GCS bucket for intermediate storage + string intermediate_bucket = "my-intermediate-bucket") + { + // # A useful description for your transfer job + string jobDescription = $"Transfer objects from {rootDirectory} to the {destinationDirectory} between POSIX file system"; + + TransferJob transferJob = new TransferJob + { + ProjectId = projectId, + Description = jobDescription, + TransferSpec = new TransferSpec + { + SourceAgentPoolName = sourceAgentPoolName, + SinkAgentPoolName = sinkAgentPoolName, + PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory }, + PosixDataSink = new PosixFilesystem { RootDirectory = destinationDirectory }, + GcsIntermediateDataLocation = new GcsData { BucketName = intermediate_bucket } + }, + Status = TransferJob.Types.Status.Enabled, + }; + + + // Create a Transfer Service client + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + + // Create a Transfer job + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + + client.RunTransferJob(new RunTransferJobRequest + { + JobName = response.Name, + ProjectId = projectId + }); + + _output.WriteLine($"Created and ran transfer job from {rootDirectory} to {destinationDirectory} with the name {response.Name}"); + return response; + + } + } +} +//[END storagetransfer_transfer_posix_to_posix] + + From 7d100d4b8e5d433a8660caebdc7b5ab3e379b079 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 7 Nov 2024 11:26:48 +0000 Subject: [PATCH 14/58] one test check added in posix to posix storage transfer --- .../StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index e8fd90bc63b..43a2f86d888 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -54,13 +54,15 @@ public void TransferBetweenPosix() var storage = StorageClient.Create(); var transferJob = transferBetweenPosixSample.TransferBetweenPosix(_fixture.ProjectId,_fixture.SourceAgentPoolName,_fixture.SinkAgentPoolName,_fixture.TempDirectory,_fixture.TempDestinationDirectory,_fixture.BucketNameSource); Assert.Contains("transferJobs/", transferJob.Name); + _transferJobName = transferJob.Name; Assert.True(Directory.Exists(_fixture.TempDirectory)); Assert.True(Directory.Exists(_fixture.TempDestinationDirectory)); Assert.True(File.Exists(txtList[0])); + string sourceFilePath = txtList[0].Replace(_fixture.RootDirectory.Substring(0,_fixture.RootDirectory.Length - 1),_fixture.TempDirectory); + Assert.True(File.Exists(sourceFilePath)); System.Threading.Thread.Sleep(TimeSpan.FromSeconds(45)); string destinationFilePath = txtList[0].Replace(_fixture.RootDirectory.Substring(0,_fixture.RootDirectory.Length - 1),_fixture.TempDestinationDirectory); Assert.True(File.Exists(destinationFilePath)); - _transferJobName = transferJob.Name; } public void Dispose() From 4b8848067694d7634fe486a654d43c51adf64999 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Mon, 11 Nov 2024 10:35:52 +0000 Subject: [PATCH 15/58] add sample and test case for event driven transfer and few storage fixture changes --- .../CreateEventDrivenGcsTransferTest.cs | 67 +++++++++++++++ .../StorageFixture.cs | 85 +++++++++++++++++-- .../StorageTransfer.Samples.Tests.csproj | 1 + .../CreateEventDrivenGcsTransferSample.cs | 65 ++++++++++++++ 4 files changed, 212 insertions(+), 6 deletions(-) create mode 100644 storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs create mode 100644 storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs new file mode 100644 index 00000000000..a301d040476 --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -0,0 +1,67 @@ +/** + * Copyright 2024 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using Google.Cloud.StorageTransfer.V1; +using Xunit; +using Xunit.Abstractions; + +namespace StorageTransfer.Samples.Tests +{ + [Collection(nameof(StorageFixture))] + public class CreateEventDrivenGcsTransferTest : IDisposable + { + private readonly StorageFixture _fixture; + private string _transferJobName; + private readonly ITestOutputHelper _outputHelper; + + public CreateEventDrivenGcsTransferTest(StorageFixture fixture , ITestOutputHelper outputHelper) + { + _fixture = fixture; + _outputHelper = outputHelper; + } + + [Fact] + public void CreateEventDrivenGcsTransfer() + { + CreateEventDrivenGcsTransferSample createEventDrivenGcsTransferSample = new CreateEventDrivenGcsTransferSample(_outputHelper); + var transferJob = createEventDrivenGcsTransferSample.CreateEventDrivenGcsTransfer(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink, _fixture.PubSubId); + Assert.Contains("transferJobs/", transferJob.Name); + _transferJobName = transferJob.Name; + } + + public void Dispose() + { + try + { + _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest() + { + ProjectId = _fixture.ProjectId, + JobName = _transferJobName, + TransferJob = new TransferJob() + { + Name = _transferJobName, + Status = TransferJob.Types.Status.Deleted + } + }); + } + catch (Exception) + { + // Do nothing, we delete on a best effort basis. + } + } + } +} diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index 22b43f7ea8e..d1cb7fe2972 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -21,34 +21,41 @@ using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using Xunit; +using Google.Cloud.PubSub.V1; +using Grpc.Core; namespace StorageTransfer.Samples.Tests { [CollectionDefinition(nameof(StorageFixture))] public class StorageFixture : IDisposable, ICollectionFixture - { + { public string ProjectId { get; } public string BucketNameSource { get; } = Guid.NewGuid().ToString(); public string BucketNameSink { get; } = Guid.NewGuid().ToString(); public string JobName { get; } public string SourceAgentPoolName { get; } public string SinkAgentPoolName { get; } - public string GcsSourcePath { get;} + public string GcsSourcePath { get; } public string RootDirectory { get; } = System.IO.Path.GetTempPath(); public string DestinationDirectory { get; } = System.IO.Path.GetTempPath(); public string TempDirectory { get; } = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); public string TempDestinationDirectory { get; } = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); public StorageClient Storage { get; } = StorageClient.Create(); public string ManifestObjectName { get; } = "manifest.csv"; + public string TopicId { get; } = "DotNetTopic" + Guid.NewGuid().ToString(); + public string SubscriptionId { get; } = "DotNetSubscription" + Guid.NewGuid().ToString(); + public string PubSubId { get; } public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create(); + + public SubscriberServiceApiClient SubscriberClient { get; } = SubscriberServiceApiClient.Create(); + + public PublisherServiceApiClient PublisherClient { get; } = PublisherServiceApiClient.Create(); public StorageFixture() { - // Instantiate random number generator - Random random = new Random(); - JobName = "transferJobs/" + random.NextInt64(1000000000000000, 9223372036854775807) + " "; ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/transfer_service_default"; SinkAgentPoolName = "projects/" + ProjectId + "/agentPools/transfer_service_default"; + PubSubId = "projects/" + ProjectId + "/subscriptions/" + SubscriptionId + ""; GcsSourcePath = "foo/bar/"; if (string.IsNullOrWhiteSpace(ProjectId)) { @@ -57,6 +64,49 @@ public StorageFixture() CreateBucketAndGrantStsPermissions(BucketNameSink); CreateBucketAndGrantStsPermissions(BucketNameSource); + // Initialize request argument(s) + TransferJob transferJob = new TransferJob{ + ProjectId = ProjectId, + TransferSpec = new TransferSpec + { + GcsDataSink = new GcsData { BucketName = BucketNameSource }, + GcsDataSource = new GcsData { BucketName = BucketNameSink } + }, + Status = TransferJob.Types.Status.Enabled}; + CreateTransferJobRequest request = new CreateTransferJobRequest + { + TransferJob = transferJob + }; + // Make the request + TransferJob response = Sts.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + JobName = response.Name; + string email = Sts.GetGoogleServiceAccount(new GetGoogleServiceAccountRequest() + { + ProjectId = ProjectId + }).AccountEmail; + string memberServiceAccount = "serviceAccount:" + email; + // Create subscription name + SubscriptionName subscriptionName = new SubscriptionName(ProjectId, SubscriptionId); + // Create topic name + TopicName topicName = new TopicName(ProjectId, TopicId); + // Create topic + PublisherClient.CreateTopic(topicName); + // Create subscription. + SubscriberClient.CreateSubscription(subscriptionName, topicName, pushConfig: null, ackDeadlineSeconds: 500); + var policyIamPolicyTopic = new Google.Cloud.Iam.V1.Policy(); + policyIamPolicyTopic.AddRoleMember("roles/pubsub.publisher", memberServiceAccount); + PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest + { + ResourceAsResourceName = topicName, + Policy = policyIamPolicyTopic + }); + var policyIamPolicySubscriber = new Google.Cloud.Iam.V1.Policy(); + policyIamPolicySubscriber.AddRoleMember("roles/pubsub.subscriber", memberServiceAccount); + PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest + { + ResourceAsResourceName = subscriptionName, + Policy = policyIamPolicySubscriber + }); } private void CreateBucketAndGrantStsPermissions(string bucketName) @@ -74,7 +124,7 @@ private void CreateBucketAndGrantStsPermissions(string bucketName) string objectViewer = "roles/storage.objectViewer"; string bucketReader = "roles/storage.legacyBucketReader"; string bucketWriter = "roles/storage.legacyBucketWriter"; - + var policy = Storage.GetBucketIamPolicy(bucketName, new GetBucketIamPolicyOptions { RequestedPolicyVersion = 3 @@ -97,12 +147,14 @@ private void CreateBucketAndGrantStsPermissions(string bucketName) Role = bucketWriter, Members = new List { member } }; + policy.Bindings.Add(objectViewerBinding); policy.Bindings.Add(bucketReaderBinding); policy.Bindings.Add(bucketWriterBinding); Storage.SetBucketIamPolicy(bucketName, policy); + } public void Dispose() @@ -133,6 +185,27 @@ public void Dispose() } Storage.DeleteBucket(BucketNameSource); } + + try + { + TopicName topicName = TopicName.FromProjectTopic(ProjectId,TopicId); + PublisherClient.DeleteTopic(topicName); + } + catch (RpcException ex) + { + throw new Exception ($"Exception occur while deleting Topic {TopicId} Exception: {ex}"); + } + + try + { + SubscriptionName subscriptionName = SubscriptionName.FromProjectSubscription(ProjectId, SubscriptionId); + SubscriberClient.DeleteSubscription(subscriptionName); + } + catch (RpcException ex) + { + throw new Exception($"Exception occur while deleting subscription {SubscriptionId} Exception: {ex}"); + } + } } } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageTransfer.Samples.Tests.csproj b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageTransfer.Samples.Tests.csproj index 4991e2fa1a4..38402a526aa 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageTransfer.Samples.Tests.csproj +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageTransfer.Samples.Tests.csproj @@ -16,6 +16,7 @@ + \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs new file mode 100644 index 00000000000..d865d532468 --- /dev/null +++ b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs @@ -0,0 +1,65 @@ +/** + * Copyright 2024 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// [START storagetransfer_create_event_driven_gcs_transfer] +using Google.Cloud.StorageTransfer.V1; +using Xunit.Abstractions; + +namespace StorageTransfer.Samples +{ + public class CreateEventDrivenGcsTransferSample + { + private readonly ITestOutputHelper _output; + public CreateEventDrivenGcsTransferSample(ITestOutputHelper output) + { + _output = output; + } + public TransferJob CreateEventDrivenGcsTransfer( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The GCS bucket to transfer data from + string sourceBucket = "my-source-bucket", + // The GCS bucket to transfer data to + string sinkBucket = "my-sink-bucket", + // The subscription ID to a Pubsub queue to track + string pubSubId = "projects/PROJECT_NAME/subscriptions/SUBSCRIPTION_ID") + { + // # A useful description for your transfer job + string jobDescription = $"Event driven gcs data transfer from {sourceBucket} to {sinkBucket} subscribed to {pubSubId} "; + + TransferJob transferJob = new TransferJob + { + ProjectId = projectId, + Description = jobDescription, + TransferSpec = new TransferSpec + { + GcsDataSink = new GcsData { BucketName = sinkBucket }, + GcsDataSource = new GcsData { BucketName = sourceBucket }, + + }, + Status = TransferJob.Types.Status.Enabled, + EventStream = new EventStream { Name = pubSubId } + }; + + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + _output.WriteLine($"Created an event driven transfer job from {sourceBucket} to {sinkBucket} subscribed to {pubSubId} with name {response.Name}"); + + return response; + } + } +} +// [END storagetransfer_create_event_driven_gcs_transfer] + From effd0c2ed5a67be5ed46d64fc71de255970b7d71 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Tue, 12 Nov 2024 03:59:58 -0800 Subject: [PATCH 16/58] samples(storage transfer): code changes for printing messages to console , rewording , removing some unnecessary symbols, few test checks removed related to posix file transfer. --- .../CheckLatestTransferOperationTest.cs | 9 ++-- .../CreateEventDrivenGcsTransferTest.cs | 10 ++--- .../DownloadToPosixTest.cs | 21 ++++------ .../QuickstartTest.cs | 10 ++--- .../StorageFixture.cs | 41 ++++++++++--------- .../TransferBetweenPosixTest.cs | 19 +++------ .../TransferFromPosixTest.cs | 10 ++--- .../TransferToNearlineTest.cs | 8 +--- .../TransferUsingManifestTest.cs | 9 ++-- .../CheckLatestTransferOperationSample.cs | 31 ++++++-------- .../CreateEventDrivenGcsTransferSample.cs | 14 +++---- .../DownloadToPosixSample.cs | 12 ++---- .../QuickstartSample.cs | 9 +--- .../TransferBetweenPosixSample.cs | 12 ++---- .../TransferFromPosixSample.cs | 14 ++----- .../TransferToNearlineSample.cs | 9 +--- .../TransferUsingManifestSample.cs | 12 ++---- 17 files changed, 85 insertions(+), 165 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs index 4790fc473b4..39d3dfce59e 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. using Xunit; -using Xunit.Abstractions; namespace StorageTransfer.Samples.Tests; @@ -22,18 +21,16 @@ public class CheckLatestTransferOperationTest private readonly StorageFixture _fixture; private string _jobName; - private readonly ITestOutputHelper _outputHelper; - public CheckLatestTransferOperationTest(StorageFixture fixture, ITestOutputHelper outputHelper) + public CheckLatestTransferOperationTest(StorageFixture fixture) { - _outputHelper = outputHelper; _fixture = fixture; } [Fact] public void CheckLatestTransferOperation() { - CheckLatestTransferOperationSample checkLatestTransferOperationSample = new CheckLatestTransferOperationSample(_outputHelper); - var transferJob = checkLatestTransferOperationSample.CheckLatestTransferOperation(_fixture.ProjectId,_fixture.JobName); + CheckLatestTransferOperationSample checkLatestTransferOperationSample = new CheckLatestTransferOperationSample(); + var transferJob = checkLatestTransferOperationSample.CheckLatestTransferOperation(_fixture.ProjectId, _fixture.JobName); Assert.Contains("transferJobs/", transferJob.Name); _jobName = transferJob.Name; } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs index a301d040476..af042168a98 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -14,10 +14,9 @@ * limitations under the License. */ -using System; using Google.Cloud.StorageTransfer.V1; +using System; using Xunit; -using Xunit.Abstractions; namespace StorageTransfer.Samples.Tests { @@ -26,18 +25,15 @@ public class CreateEventDrivenGcsTransferTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; - private readonly ITestOutputHelper _outputHelper; - - public CreateEventDrivenGcsTransferTest(StorageFixture fixture , ITestOutputHelper outputHelper) + public CreateEventDrivenGcsTransferTest(StorageFixture fixture) { _fixture = fixture; - _outputHelper = outputHelper; } [Fact] public void CreateEventDrivenGcsTransfer() { - CreateEventDrivenGcsTransferSample createEventDrivenGcsTransferSample = new CreateEventDrivenGcsTransferSample(_outputHelper); + CreateEventDrivenGcsTransferSample createEventDrivenGcsTransferSample = new CreateEventDrivenGcsTransferSample(); var transferJob = createEventDrivenGcsTransferSample.CreateEventDrivenGcsTransfer(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink, _fixture.PubSubId); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index cb1abf8769e..95a7ef904cf 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -15,13 +15,11 @@ using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using System; -using Xunit.Abstractions; -using Xunit; -using System.Text; using System.IO; using System.Linq; +using System.Text; using System.Threading.Tasks; - +using Xunit; namespace StorageTransfer.Samples.Tests; [Collection(nameof(StorageFixture))] @@ -29,29 +27,24 @@ public class DownloadToPosixTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; - private readonly ITestOutputHelper _outputHelper; - public DownloadToPosixTest(StorageFixture fixture, ITestOutputHelper outputHelper) + public DownloadToPosixTest(StorageFixture fixture) { _fixture = fixture; - _outputHelper = outputHelper; } [Fact] public void DownloadToPosix() { - DownloadToPosixSample downloadToPosixSample = new DownloadToPosixSample(_outputHelper); + DownloadToPosixSample downloadToPosixSample = new DownloadToPosixSample(); Directory.CreateDirectory(_fixture.TempDirectory); var storage = StorageClient.Create(); byte[] byteArray = Encoding.UTF8.GetBytes("flower.jpeg"); MemoryStream stream = new MemoryStream(byteArray); string fileName = $"{_fixture.GcsSourcePath}{DateTime.Now.ToString("yyyyMMddHHmmss")}.txt"; - string filePath = $"{_fixture.TempDirectory}/{fileName.Split('/').Last()}"; - storage.UploadObject(_fixture.BucketNameSource,fileName, "application/octet-stream", stream); - var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId,_fixture.SinkAgentPoolName,_fixture.BucketNameSource,_fixture.GcsSourcePath,_fixture.TempDirectory); + storage.UploadObject(_fixture.BucketNameSource, fileName, "application/octet-stream", stream); + var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId, _fixture.SinkAgentPoolName, _fixture.BucketNameSource, _fixture.GcsSourcePath, _fixture.TempDirectory); Assert.Contains("transferJobs/", transferJob.Name); - Assert.True( Directory.Exists(_fixture.TempDirectory)); - System.Threading.Thread.Sleep(TimeSpan.FromSeconds(45)); - Assert.True( File.Exists(filePath)); + Assert.True(Directory.Exists(_fixture.TempDirectory)); _transferJobName = transferJob.Name; } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs index 5b96b7519c6..e826771b11b 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs @@ -14,10 +14,9 @@ * limitations under the License. */ -using System; using Google.Cloud.StorageTransfer.V1; +using System; using Xunit; -using Xunit.Abstractions; namespace StorageTransfer.Samples.Tests { @@ -26,18 +25,15 @@ public class QuickstartTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; - private readonly ITestOutputHelper _outputHelper; - - public QuickstartTest(StorageFixture fixture , ITestOutputHelper outputHelper) + public QuickstartTest(StorageFixture fixture) { _fixture = fixture; - _outputHelper = outputHelper; } [Fact] public void TestQuickstart() { - QuickstartSample quickstartSample = new QuickstartSample(_outputHelper); + QuickstartSample quickstartSample = new QuickstartSample(); var transferJob = quickstartSample.Quickstart(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index d1cb7fe2972..e9853e71828 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -14,20 +14,21 @@ * limitations under the License. */ -using System; -using System.Collections.Generic; -using System.IO; using Google.Apis.Storage.v1.Data; +using Google.Cloud.PubSub.V1; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; -using Xunit; -using Google.Cloud.PubSub.V1; using Grpc.Core; +using System; +using System.Collections.Generic; +using System.IO; +using Xunit; + namespace StorageTransfer.Samples.Tests { [CollectionDefinition(nameof(StorageFixture))] public class StorageFixture : IDisposable, ICollectionFixture - { + { public string ProjectId { get; } public string BucketNameSource { get; } = Guid.NewGuid().ToString(); public string BucketNameSink { get; } = Guid.NewGuid().ToString(); @@ -45,9 +46,9 @@ public class StorageFixture : IDisposable, ICollectionFixture public string SubscriptionId { get; } = "DotNetSubscription" + Guid.NewGuid().ToString(); public string PubSubId { get; } public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create(); - + public SubscriberServiceApiClient SubscriberClient { get; } = SubscriberServiceApiClient.Create(); - + public PublisherServiceApiClient PublisherClient { get; } = PublisherServiceApiClient.Create(); public StorageFixture() @@ -65,14 +66,16 @@ public StorageFixture() CreateBucketAndGrantStsPermissions(BucketNameSink); CreateBucketAndGrantStsPermissions(BucketNameSource); // Initialize request argument(s) - TransferJob transferJob = new TransferJob{ - ProjectId = ProjectId, - TransferSpec = new TransferSpec + TransferJob transferJob = new TransferJob + { + ProjectId = ProjectId, + TransferSpec = new TransferSpec { GcsDataSink = new GcsData { BucketName = BucketNameSource }, GcsDataSource = new GcsData { BucketName = BucketNameSink } }, - Status = TransferJob.Types.Status.Enabled}; + Status = TransferJob.Types.Status.Enabled + }; CreateTransferJobRequest request = new CreateTransferJobRequest { TransferJob = transferJob @@ -124,7 +127,7 @@ private void CreateBucketAndGrantStsPermissions(string bucketName) string objectViewer = "roles/storage.objectViewer"; string bucketReader = "roles/storage.legacyBucketReader"; string bucketWriter = "roles/storage.legacyBucketWriter"; - + var policy = Storage.GetBucketIamPolicy(bucketName, new GetBucketIamPolicyOptions { RequestedPolicyVersion = 3 @@ -147,14 +150,14 @@ private void CreateBucketAndGrantStsPermissions(string bucketName) Role = bucketWriter, Members = new List { member } }; - + policy.Bindings.Add(objectViewerBinding); policy.Bindings.Add(bucketReaderBinding); policy.Bindings.Add(bucketWriterBinding); Storage.SetBucketIamPolicy(bucketName, policy); - + } public void Dispose() @@ -185,15 +188,15 @@ public void Dispose() } Storage.DeleteBucket(BucketNameSource); } - + try { - TopicName topicName = TopicName.FromProjectTopic(ProjectId,TopicId); + TopicName topicName = TopicName.FromProjectTopic(ProjectId, TopicId); PublisherClient.DeleteTopic(topicName); } catch (RpcException ex) { - throw new Exception ($"Exception occur while deleting Topic {TopicId} Exception: {ex}"); + throw new Exception($"Exception occur while deleting Topic {TopicId} Exception: {ex}"); } try @@ -205,7 +208,7 @@ public void Dispose() { throw new Exception($"Exception occur while deleting subscription {SubscriptionId} Exception: {ex}"); } - + } } } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index 43a2f86d888..52fba525b7f 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -15,11 +15,9 @@ using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using System; -using Xunit.Abstractions; -using Xunit; using System.IO; using System.Linq; - +using Xunit; namespace StorageTransfer.Samples.Tests; [Collection(nameof(StorageFixture))] @@ -27,20 +25,18 @@ public class TransferBetweenPosixTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; - private readonly ITestOutputHelper _outputHelper; - public TransferBetweenPosixTest(StorageFixture fixture, ITestOutputHelper outputHelper) + public TransferBetweenPosixTest(StorageFixture fixture) { _fixture = fixture; - _outputHelper = outputHelper; } [Fact] public void TransferBetweenPosix() { - TransferBetweenPosixSample transferBetweenPosixSample = new TransferBetweenPosixSample(_outputHelper); + TransferBetweenPosixSample transferBetweenPosixSample = new TransferBetweenPosixSample(); Directory.CreateDirectory(_fixture.TempDirectory); Directory.CreateDirectory(_fixture.TempDestinationDirectory); - string sourceDir =_fixture.RootDirectory; + string sourceDir = _fixture.RootDirectory; string[] txtList = Directory.GetFiles(sourceDir, "*.txt"); // Copy one txt file. foreach (string f in txtList) @@ -52,17 +48,14 @@ public void TransferBetweenPosix() break; } var storage = StorageClient.Create(); - var transferJob = transferBetweenPosixSample.TransferBetweenPosix(_fixture.ProjectId,_fixture.SourceAgentPoolName,_fixture.SinkAgentPoolName,_fixture.TempDirectory,_fixture.TempDestinationDirectory,_fixture.BucketNameSource); + var transferJob = transferBetweenPosixSample.TransferBetweenPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.SinkAgentPoolName, _fixture.TempDirectory, _fixture.TempDestinationDirectory, _fixture.BucketNameSource); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; Assert.True(Directory.Exists(_fixture.TempDirectory)); Assert.True(Directory.Exists(_fixture.TempDestinationDirectory)); Assert.True(File.Exists(txtList[0])); - string sourceFilePath = txtList[0].Replace(_fixture.RootDirectory.Substring(0,_fixture.RootDirectory.Length - 1),_fixture.TempDirectory); + string sourceFilePath = txtList[0].Replace(_fixture.RootDirectory.Substring(0, _fixture.RootDirectory.Length - 1), _fixture.TempDirectory); Assert.True(File.Exists(sourceFilePath)); - System.Threading.Thread.Sleep(TimeSpan.FromSeconds(45)); - string destinationFilePath = txtList[0].Replace(_fixture.RootDirectory.Substring(0,_fixture.RootDirectory.Length - 1),_fixture.TempDestinationDirectory); - Assert.True(File.Exists(destinationFilePath)); } public void Dispose() diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs index 6f512b11fc2..d6ab8163804 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs @@ -15,28 +15,24 @@ using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using System; -using Xunit.Abstractions; using Xunit; - namespace StorageTransfer.Samples.Tests; [Collection(nameof(StorageFixture))] public class TransferFromPosixTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; - private readonly ITestOutputHelper _outputHelper; - public TransferFromPosixTest(StorageFixture fixture, ITestOutputHelper outputHelper) + public TransferFromPosixTest(StorageFixture fixture) { _fixture = fixture; - _outputHelper = outputHelper; } [Fact] public void TransferFromPosix() { - TransferFromPosixSample transferFromPosixSample = new TransferFromPosixSample(_outputHelper); - var transferJob = transferFromPosixSample.TransferFromPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.RootDirectory,_fixture.BucketNameSink); + TransferFromPosixSample transferFromPosixSample = new TransferFromPosixSample(); + var transferJob = transferFromPosixSample.TransferFromPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.RootDirectory, _fixture.BucketNameSink); var storage = StorageClient.Create(); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs index c3d1856f192..b03cdb68c12 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -17,26 +17,22 @@ using Google.Cloud.StorageTransfer.V1; using System; using Xunit; -using Xunit.Abstractions; namespace StorageTransfer.Samples.Tests; - [Collection(nameof(StorageFixture))] public class TransferToNearlineTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; - private readonly ITestOutputHelper _outputHelper; - public TransferToNearlineTest(StorageFixture fixture, ITestOutputHelper outputHelper) + public TransferToNearlineTest(StorageFixture fixture) { _fixture = fixture; - _outputHelper = outputHelper; } [Fact] public void TestTransferToNearline() { - TransferToNearlineSample transferToNearlineSample = new TransferToNearlineSample(_outputHelper); + TransferToNearlineSample transferToNearlineSample = new TransferToNearlineSample(); var storage = StorageClient.Create(); var bucket = storage.GetBucket(_fixture.BucketNameSink); string storageClass = StorageClasses.Nearline; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index c19f33a4fd4..66fde006ad2 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -15,10 +15,9 @@ using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using System; +using System.IO; using System.Text; -using Xunit.Abstractions; using Xunit; -using System.IO; namespace StorageTransfer.Samples.Tests; [Collection(nameof(StorageFixture))] @@ -26,17 +25,15 @@ public class TransferUsingManifestTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; - private readonly ITestOutputHelper _outputHelper; - public TransferUsingManifestTest(StorageFixture fixture, ITestOutputHelper outputHelper) + public TransferUsingManifestTest(StorageFixture fixture) { _fixture = fixture; - _outputHelper = outputHelper; } [Fact] public void TransferUsingManifest() { - TransferUsingManifestSample transferUsingManifestSample = new TransferUsingManifestSample(_outputHelper); + TransferUsingManifestSample transferUsingManifestSample = new TransferUsingManifestSample(); var storage = StorageClient.Create(); byte[] byteArray = Encoding.UTF8.GetBytes("flower.jpeg"); MemoryStream stream = new MemoryStream(byteArray); diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs index c8f447e3b91..9a7e7a369c7 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -15,26 +15,19 @@ using Google.Cloud.StorageTransfer.V1; using System; -using Xunit.Abstractions; namespace StorageTransfer.Samples { public class CheckLatestTransferOperationSample { - private readonly ITestOutputHelper _output; - public CheckLatestTransferOperationSample(ITestOutputHelper output) + //Checks the latest transfer operation for a given transfer job. + public TransferJob CheckLatestTransferOperation( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The name of the job to check + string jobName = "transferJobs/1234567890") { - _output = output; - - } - //Checks the latest transfer operation for a given transfer job. - public TransferJob CheckLatestTransferOperation( - // Your Google Cloud Project ID - string projectId = "my-project-id", - // The name of the job to check - string jobName = "transferJobs/1234567890") - { - if(string.IsNullOrEmpty(jobName)) + if (string.IsNullOrEmpty(jobName)) { throw new Exception("JobName can not be null or empty"); } @@ -46,23 +39,23 @@ public TransferJob CheckLatestTransferOperation( { // Get Transfer job TransferJob transferJob = storageTransfer.GetTransferJob(getTransferJobRequest); - // Get Latest operation name from tranfer job + // Get Latest operation name from transfer job string latestOperationName = transferJob.LatestOperationName; - + if (!string.IsNullOrEmpty(latestOperationName)) { - _output.WriteLine("The latest operation for transfer job " +jobName+ " is: " +latestOperationName+ ""); + Console.WriteLine("The latest operation for transfer job " + jobName + " is: " + latestOperationName + ""); } else { - _output.WriteLine("Transfer job "+ jobName +" hasn't run yet, try again once after job started running."); + Console.WriteLine("Transfer job " + jobName + " hasn't run yet, try again after the job has started running."); } return transferJob; } catch (Exception) { - throw new Exception("Failed to get transfer job "+ jobName + ""); + throw new Exception("Failed to get transfer job " + jobName + ""); } } } diff --git a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs index d865d532468..30e80f3281b 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs @@ -15,17 +15,13 @@ */ // [START storagetransfer_create_event_driven_gcs_transfer] using Google.Cloud.StorageTransfer.V1; -using Xunit.Abstractions; +using System; namespace StorageTransfer.Samples { public class CreateEventDrivenGcsTransferSample { - private readonly ITestOutputHelper _output; - public CreateEventDrivenGcsTransferSample(ITestOutputHelper output) - { - _output = output; - } + public TransferJob CreateEventDrivenGcsTransfer( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -36,7 +32,7 @@ public TransferJob CreateEventDrivenGcsTransfer( // The subscription ID to a Pubsub queue to track string pubSubId = "projects/PROJECT_NAME/subscriptions/SUBSCRIPTION_ID") { - // # A useful description for your transfer job + // A useful description for your transfer job string jobDescription = $"Event driven gcs data transfer from {sourceBucket} to {sinkBucket} subscribed to {pubSubId} "; TransferJob transferJob = new TransferJob @@ -47,7 +43,7 @@ public TransferJob CreateEventDrivenGcsTransfer( { GcsDataSink = new GcsData { BucketName = sinkBucket }, GcsDataSource = new GcsData { BucketName = sourceBucket }, - + }, Status = TransferJob.Types.Status.Enabled, EventStream = new EventStream { Name = pubSubId } @@ -55,7 +51,7 @@ public TransferJob CreateEventDrivenGcsTransfer( StorageTransferServiceClient client = StorageTransferServiceClient.Create(); TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); - _output.WriteLine($"Created an event driven transfer job from {sourceBucket} to {sinkBucket} subscribed to {pubSubId} with name {response.Name}"); + Console.WriteLine($"Created an event driven transfer job from {sourceBucket} to {sinkBucket} subscribed to {pubSubId} with name {response.Name}"); return response; } diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index 3a7e114d51a..4161ebf8304 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -13,19 +13,13 @@ // limitations under the License. // [START storagetransfer_download_to_posix] using Google.Cloud.StorageTransfer.V1; -using Xunit.Abstractions; - +using System; namespace StorageTransfer.Samples { public class DownloadToPosixSample { /*Create a transfer from a GCS bucket to a POSIX file system.*/ - private readonly ITestOutputHelper _output; - public DownloadToPosixSample(ITestOutputHelper output) - { - _output = output; - } public TransferJob DownloadToPosix( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -47,7 +41,7 @@ public TransferJob DownloadToPosix( Description = jobDescription, TransferSpec = new TransferSpec { - GcsDataSource = new GcsData { BucketName = gcsSourceBucket, Path = gcsSourcePath}, + GcsDataSource = new GcsData { BucketName = gcsSourceBucket, Path = gcsSourcePath }, SinkAgentPoolName = sinkAgentPoolName, PosixDataSink = new PosixFilesystem { RootDirectory = rootDirectory } }, @@ -67,7 +61,7 @@ public TransferJob DownloadToPosix( ProjectId = projectId }); - _output.WriteLine($"Created and ran transfer job from ({gcsSourceBucket}/{gcsSourcePath}) to {rootDirectory} with the name {response.Name}"); + Console.WriteLine($"Created and ran transfer job from ({gcsSourceBucket}/{gcsSourcePath}) to {rootDirectory} with the name {response.Name}"); return response; diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs index 68e60844d0d..b69fe4b2c28 100644 --- a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs @@ -16,17 +16,12 @@ // [START storagetransfer_quickstart] using Google.Cloud.StorageTransfer.V1; -using Xunit.Abstractions; +using System; namespace StorageTransfer.Samples { public class QuickstartSample { - private readonly ITestOutputHelper _output; - public QuickstartSample(ITestOutputHelper output) - { - _output = output; - } public TransferJob Quickstart( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -54,7 +49,7 @@ public TransferJob Quickstart( ProjectId = projectId }); - _output.WriteLine($"Created and ran transfer job from {sourceBucket} to {sinkBucket} with name {response.Name}"); + Console.WriteLine($"Created and ran transfer job from {sourceBucket} to {sinkBucket} with name {response.Name}"); return response; } diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs index 3cbc42dd7a0..7fa63a9b666 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -13,19 +13,13 @@ // limitations under the License. // [START storagetransfer_transfer_posix_to_posix] using Google.Cloud.StorageTransfer.V1; -using Xunit.Abstractions; - +using System; namespace StorageTransfer.Samples { public class TransferBetweenPosixSample { /*Creates a transfer between POSIX file systems.*/ - private readonly ITestOutputHelper _output; - public TransferBetweenPosixSample(ITestOutputHelper output) - { - _output = output; - } public TransferJob TransferBetweenPosix( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -40,7 +34,7 @@ public TransferJob TransferBetweenPosix( // The name of GCS bucket for intermediate storage string intermediate_bucket = "my-intermediate-bucket") { - // # A useful description for your transfer job + // A useful description for your transfer job string jobDescription = $"Transfer objects from {rootDirectory} to the {destinationDirectory} between POSIX file system"; TransferJob transferJob = new TransferJob @@ -71,7 +65,7 @@ public TransferJob TransferBetweenPosix( ProjectId = projectId }); - _output.WriteLine($"Created and ran transfer job from {rootDirectory} to {destinationDirectory} with the name {response.Name}"); + Console.WriteLine($"Created and ran transfer job from {rootDirectory} to {destinationDirectory} with the name {response.Name}"); return response; } diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs index 6bf91095722..7cc1e600028 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs @@ -13,19 +13,13 @@ // limitations under the License. // [START storagetransfer_transfer_from_posix] using Google.Cloud.StorageTransfer.V1; -using Xunit.Abstractions; - +using System; namespace StorageTransfer.Samples { public class TransferFromPosixSample { /*Create a transfer from a POSIX file system to a GCS sink bucket*/ - private readonly ITestOutputHelper _output; - public TransferFromPosixSample(ITestOutputHelper output) - { - _output = output; - } public TransferJob TransferFromPosix( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -36,8 +30,8 @@ public TransferJob TransferFromPosix( // The GCS bucket to transfer data to string sinkBucket = "my-sink-bucket") { - - // # A useful description for your transfer job + + // A useful description for your transfer job string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket})"; TransferJob transferJob = new TransferJob @@ -66,7 +60,7 @@ public TransferJob TransferFromPosix( ProjectId = projectId }); - _output.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} with the name {response.Name}"); + Console.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} with the name {response.Name}"); return response; diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index 2d7238eea76..83c4dcc0858 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -16,8 +16,6 @@ using Google.Cloud.StorageTransfer.V1; using Google.Protobuf.WellKnownTypes; using System; -using Xunit.Abstractions; - namespace StorageTransfer.Samples { @@ -26,11 +24,6 @@ public class TransferToNearlineSample { /*Creates a one-off transfer job that transfers objects from a standard GCS bucket that are more than 30 days old to a Nearline GCS bucket.*/ - private readonly ITestOutputHelper _output; - public TransferToNearlineSample(ITestOutputHelper output) - { - _output = output; - } public TransferJob TransferToNearline( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -66,7 +59,7 @@ public TransferJob TransferToNearline( ProjectId = projectId }); - _output.WriteLine($"Created one-off transfer job from standard bucket {sourceBucket} to Nearline bucket {sinkBucket} with the name {response.Name}"); + Console.WriteLine($"Created one-off transfer job from standard bucket {sourceBucket} to Nearline bucket {sinkBucket} with the name {response.Name}"); return response; } } diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index 5625a17177e..4f4d26168fd 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -13,8 +13,7 @@ // limitations under the License. // [START storagetransfer_manifest_request] using Google.Cloud.StorageTransfer.V1; -using Xunit.Abstractions; - +using System; namespace StorageTransfer.Samples { @@ -22,11 +21,6 @@ public class TransferUsingManifestSample { /*Create a transfer from a POSIX file system to a GCS bucket using a manifest file*/ - private readonly ITestOutputHelper _output; - public TransferUsingManifestSample(ITestOutputHelper output) - { - _output = output; - } public TransferJob TransferUsingManifest( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -43,7 +37,7 @@ public TransferJob TransferUsingManifest( { string manifestLocation = "gs://" + manifestBucket + "/" + manifestObjectName; - // # A useful description for your transfer job + // A useful description for your transfer job string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket}) using manifest file"; TransferJob transferJob = new TransferJob @@ -74,7 +68,7 @@ public TransferJob TransferUsingManifest( ProjectId = projectId }); - _output.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} using manifest file {manifestLocation} with the name {response.Name}"); + Console.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} using manifest file {manifestLocation} with the name {response.Name}"); return response; From f5d8252eb0b6984f5681cc296a3817c8a91824f7 Mon Sep 17 00:00:00 2001 From: Mahendra Date: Thu, 19 Dec 2024 14:49:12 +0530 Subject: [PATCH 17/58] Update storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs Co-authored-by: Amanda Tarafa Mas <14878252+amanda-tarafa@users.noreply.github.com> --- .../api/StorageTransfer.Samples/TransferUsingManifestSample.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index 4f4d26168fd..35fa1c5641b 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + // [START storagetransfer_manifest_request] using Google.Cloud.StorageTransfer.V1; using System; From be9c4bb6bb44ef03481d4040469051b26c58b0dd Mon Sep 17 00:00:00 2001 From: Mahendra Date: Thu, 19 Dec 2024 14:53:55 +0530 Subject: [PATCH 18/58] Update storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs Co-authored-by: Amanda Tarafa Mas <14878252+amanda-tarafa@users.noreply.github.com> --- .../StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index 66fde006ad2..391274b245b 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -20,6 +20,7 @@ using Xunit; namespace StorageTransfer.Samples.Tests; + [Collection(nameof(StorageFixture))] public class TransferUsingManifestTest : IDisposable { From be748187bf23c8af8aae45bd7e8865a2c442f23c Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 19 Dec 2024 01:58:32 -0800 Subject: [PATCH 19/58] code refactoring changes --- .../CreateEventDrivenGcsTransferTest.cs | 2 +- .../DownloadToPosixTest.cs | 4 +-- .../QuickstartTest.cs | 5 ++- .../StorageFixture.cs | 33 ++++++------------- .../TransferBetweenPosixTest.cs | 4 +-- .../TransferFromPosixTest.cs | 2 +- .../TransferToNearlineTest.cs | 3 +- .../TransferUsingManifestTest.cs | 4 +-- .../CheckLatestTransferOperationSample.cs | 2 +- .../CreateEventDrivenGcsTransferSample.cs | 3 +- .../DownloadToPosixSample.cs | 3 +- .../QuickstartSample.cs | 7 ++-- .../StorageTransfer.Samples.csproj | 1 - .../TransferBetweenPosixSample.cs | 3 +- .../TransferFromPosixSample.cs | 3 +- .../TransferToNearlineSample.cs | 2 +- .../TransferUsingManifestSample.cs | 5 +-- 17 files changed, 38 insertions(+), 48 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs index af042168a98..7e3f2cb1724 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -14,8 +14,8 @@ * limitations under the License. */ -using Google.Cloud.StorageTransfer.V1; using System; +using Google.Cloud.StorageTransfer.V1; using Xunit; namespace StorageTransfer.Samples.Tests diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index 95a7ef904cf..0cbab538ccf 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -using Google.Cloud.Storage.V1; -using Google.Cloud.StorageTransfer.V1; using System; using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; +using Google.Cloud.Storage.V1; +using Google.Cloud.StorageTransfer.V1; using Xunit; namespace StorageTransfer.Samples.Tests; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs index e826771b11b..b72b1900c5f 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs @@ -1,5 +1,5 @@ /** - * Copyright 2024 Google Inc. + * Copyright 2021 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,9 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -using Google.Cloud.StorageTransfer.V1; using System; +using Google.Cloud.StorageTransfer.V1; using Xunit; namespace StorageTransfer.Samples.Tests diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index e9853e71828..e2e06260b91 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -150,53 +150,40 @@ private void CreateBucketAndGrantStsPermissions(string bucketName) Role = bucketWriter, Members = new List { member } }; - - policy.Bindings.Add(objectViewerBinding); policy.Bindings.Add(bucketReaderBinding); policy.Bindings.Add(bucketWriterBinding); - Storage.SetBucketIamPolicy(bucketName, policy); - } public void Dispose() { try { - Storage.DeleteBucket(BucketNameSink); + Storage.DeleteBucket(BucketNameSink, new DeleteBucketOptions { DeleteObjects = true }); } catch (Exception) { - // If bucket is not empty, we delete on a best effort basis. - foreach (var storageObject in Storage.ListObjects(BucketNameSink, "")) - { - Storage.DeleteObject(BucketNameSink, storageObject.Name); - } - Storage.DeleteBucket(BucketNameSink); + // Do nothing, we delete on a best effort basis. + } try { - Storage.DeleteBucket(BucketNameSource); + Storage.DeleteBucket(BucketNameSource, new DeleteBucketOptions { DeleteObjects = true }); } catch (Exception) { - // If bucket is not empty, we delete on a best effort basis. - foreach (var storageObject in Storage.ListObjects(BucketNameSource, "")) - { - Storage.DeleteObject(BucketNameSource, storageObject.Name); - } - Storage.DeleteBucket(BucketNameSource); - } + // Do nothing, we delete on a best effort basis. + } try { TopicName topicName = TopicName.FromProjectTopic(ProjectId, TopicId); PublisherClient.DeleteTopic(topicName); } - catch (RpcException ex) + catch (RpcException) { - throw new Exception($"Exception occur while deleting Topic {TopicId} Exception: {ex}"); + // Do nothing, we delete on a best effort basis. } try @@ -204,9 +191,9 @@ public void Dispose() SubscriptionName subscriptionName = SubscriptionName.FromProjectSubscription(ProjectId, SubscriptionId); SubscriberClient.DeleteSubscription(subscriptionName); } - catch (RpcException ex) + catch (RpcException) { - throw new Exception($"Exception occur while deleting subscription {SubscriptionId} Exception: {ex}"); + // Do nothing, we delete on a best effort basis. } } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index 52fba525b7f..02d941cabc1 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -using Google.Cloud.Storage.V1; -using Google.Cloud.StorageTransfer.V1; using System; using System.IO; using System.Linq; +using Google.Cloud.Storage.V1; +using Google.Cloud.StorageTransfer.V1; using Xunit; namespace StorageTransfer.Samples.Tests; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs index d6ab8163804..f9f0d3e68c9 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +using System; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; -using System; using Xunit; namespace StorageTransfer.Samples.Tests; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs index b03cdb68c12..6fe8e41abcb 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -12,10 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. - +using System; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; -using System; using Xunit; namespace StorageTransfer.Samples.Tests; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index 66fde006ad2..96910225ac7 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -using Google.Cloud.Storage.V1; -using Google.Cloud.StorageTransfer.V1; using System; using System.IO; using System.Text; +using Google.Cloud.Storage.V1; +using Google.Cloud.StorageTransfer.V1; using Xunit; namespace StorageTransfer.Samples.Tests; diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs index 9a7e7a369c7..e6c7614d227 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -13,8 +13,8 @@ // limitations under the License. // [START storagetransfer_get_latest_transfer_operation] -using Google.Cloud.StorageTransfer.V1; using System; +using Google.Cloud.StorageTransfer.V1; namespace StorageTransfer.Samples { diff --git a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs index 30e80f3281b..717d3ed6207 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs @@ -14,8 +14,9 @@ * limitations under the License. */ // [START storagetransfer_create_event_driven_gcs_transfer] -using Google.Cloud.StorageTransfer.V1; + using System; +using Google.Cloud.StorageTransfer.V1; namespace StorageTransfer.Samples { diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index 4161ebf8304..719c3553ce6 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. // [START storagetransfer_download_to_posix] -using Google.Cloud.StorageTransfer.V1; + using System; +using Google.Cloud.StorageTransfer.V1; namespace StorageTransfer.Samples { diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs index b69fe4b2c28..3adcc527053 100644 --- a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs @@ -1,5 +1,5 @@ /** - * Copyright 2024 Google Inc. + * Copyright 2021 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,10 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - // [START storagetransfer_quickstart] -using Google.Cloud.StorageTransfer.V1; + using System; +using Google.Cloud.StorageTransfer.V1; + namespace StorageTransfer.Samples { diff --git a/storagetransfer/api/StorageTransfer.Samples/StorageTransfer.Samples.csproj b/storagetransfer/api/StorageTransfer.Samples/StorageTransfer.Samples.csproj index 2b68373281f..cb35bdd48fa 100644 --- a/storagetransfer/api/StorageTransfer.Samples/StorageTransfer.Samples.csproj +++ b/storagetransfer/api/StorageTransfer.Samples/StorageTransfer.Samples.csproj @@ -6,6 +6,5 @@ - diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs index 7fa63a9b666..6dcf7d9ea3c 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. // [START storagetransfer_transfer_posix_to_posix] -using Google.Cloud.StorageTransfer.V1; + using System; +using Google.Cloud.StorageTransfer.V1; namespace StorageTransfer.Samples { diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs index 7cc1e600028..69af8cad20a 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. // [START storagetransfer_transfer_from_posix] -using Google.Cloud.StorageTransfer.V1; + using System; +using Google.Cloud.StorageTransfer.V1; namespace StorageTransfer.Samples { diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index 83c4dcc0858..3f9ad9b655c 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -13,9 +13,9 @@ // limitations under the License. // [START storagetransfer_transfer_to_nearline] +using System; using Google.Cloud.StorageTransfer.V1; using Google.Protobuf.WellKnownTypes; -using System; namespace StorageTransfer.Samples { diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index 4f4d26168fd..5ef812cb7ef 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. // [START storagetransfer_manifest_request] -using Google.Cloud.StorageTransfer.V1; + using System; +using Google.Cloud.StorageTransfer.V1; namespace StorageTransfer.Samples { @@ -35,7 +36,7 @@ public TransferJob TransferUsingManifest( // The name of the manifest file in manifestBucket that specifies which objects to transfer string manifestObjectName = "path/to/manifest.csv") { - string manifestLocation = "gs://" + manifestBucket + "/" + manifestObjectName; + string manifestLocation = $"gs://{manifestBucket}/{manifestObjectName}"; // A useful description for your transfer job string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket}) using manifest file"; From 4b5d2e555bae5e091681cf9e8196a91dc7f9bafa Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 19 Dec 2024 04:27:48 -0800 Subject: [PATCH 20/58] code changes --- .../DownloadToPosixTest.cs | 7 +--- .../StorageFixture.cs | 35 +++++++++++++++---- .../TransferBetweenPosixTest.cs | 22 ++++++------ .../TransferUsingManifestTest.cs | 8 +---- .../CheckLatestTransferOperationSample.cs | 4 +-- .../CreateEventDrivenGcsTransferSample.cs | 4 +-- .../DownloadToPosixSample.cs | 8 ++--- .../QuickstartSample.cs | 5 +-- .../TransferBetweenPosixSample.cs | 6 ++-- .../TransferFromPosixSample.cs | 6 ++-- .../TransferToNearlineSample.cs | 9 ++--- .../TransferUsingManifestSample.cs | 8 +---- 12 files changed, 55 insertions(+), 67 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index 0cbab538ccf..aa60ca5deef 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -37,12 +37,7 @@ public void DownloadToPosix() { DownloadToPosixSample downloadToPosixSample = new DownloadToPosixSample(); Directory.CreateDirectory(_fixture.TempDirectory); - var storage = StorageClient.Create(); - byte[] byteArray = Encoding.UTF8.GetBytes("flower.jpeg"); - MemoryStream stream = new MemoryStream(byteArray); - string fileName = $"{_fixture.GcsSourcePath}{DateTime.Now.ToString("yyyyMMddHHmmss")}.txt"; - storage.UploadObject(_fixture.BucketNameSource, fileName, "application/octet-stream", stream); - var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId, _fixture.SinkAgentPoolName, _fixture.BucketNameSource, _fixture.GcsSourcePath, _fixture.TempDirectory); + var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId, _fixture.SinkAgentPoolName, _fixture.BucketNamePosixSource, _fixture.GcsSourcePath, _fixture.TempDirectory); Assert.Contains("transferJobs/", transferJob.Name); Assert.True(Directory.Exists(_fixture.TempDirectory)); _transferJobName = transferJob.Name; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index e2e06260b91..e0e084fb9cf 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -14,14 +14,14 @@ * limitations under the License. */ +using System; +using System.Collections.Generic; +using System.IO; using Google.Apis.Storage.v1.Data; using Google.Cloud.PubSub.V1; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using Grpc.Core; -using System; -using System.Collections.Generic; -using System.IO; using Xunit; namespace StorageTransfer.Samples.Tests @@ -32,6 +32,8 @@ public class StorageFixture : IDisposable, ICollectionFixture public string ProjectId { get; } public string BucketNameSource { get; } = Guid.NewGuid().ToString(); public string BucketNameSink { get; } = Guid.NewGuid().ToString(); + public string BucketNameManifestSource { get; } = Guid.NewGuid().ToString(); + public string BucketNamePosixSource { get; } = Guid.NewGuid().ToString(); public string JobName { get; } public string SourceAgentPoolName { get; } public string SinkAgentPoolName { get; } @@ -54,9 +56,9 @@ public class StorageFixture : IDisposable, ICollectionFixture public StorageFixture() { ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); - SourceAgentPoolName = "projects/" + ProjectId + "/agentPools/transfer_service_default"; - SinkAgentPoolName = "projects/" + ProjectId + "/agentPools/transfer_service_default"; - PubSubId = "projects/" + ProjectId + "/subscriptions/" + SubscriptionId + ""; + SourceAgentPoolName = $"projects/{ProjectId}/agentPools/transfer_service_default"; + SinkAgentPoolName = $"projects/{ProjectId}/agentPools/transfer_service_default"; + PubSubId = $"projects/{ProjectId}/subscriptions/{SubscriptionId}"; GcsSourcePath = "foo/bar/"; if (string.IsNullOrWhiteSpace(ProjectId)) { @@ -65,6 +67,10 @@ public StorageFixture() CreateBucketAndGrantStsPermissions(BucketNameSink); CreateBucketAndGrantStsPermissions(BucketNameSource); + CreateBucketAndGrantStsPermissions(BucketNameManifestSource); + CreateBucketAndGrantStsPermissions(BucketNamePosixSource); + UploadObjectToManifestBucket(BucketNameManifestSource); + UploadObjectToPosixBucket(BucketNamePosixSource); // Initialize request argument(s) TransferJob transferJob = new TransferJob { @@ -156,6 +162,23 @@ private void CreateBucketAndGrantStsPermissions(string bucketName) Storage.SetBucketIamPolicy(bucketName, policy); } + private void UploadObjectToManifestBucket(string bucketName) + { + var storage = StorageClient.Create(); + byte[] byteArray = System.Text.Encoding.UTF8.GetBytes("flower.jpeg"); + MemoryStream stream = new MemoryStream(byteArray); + storage.UploadObject(bucketName,ManifestObjectName, "application/octet-stream", stream); + } + + private void UploadObjectToPosixBucket(string bucketName) + { + var storage = StorageClient.Create(); + byte[] byteArray = System.Text.Encoding.UTF8.GetBytes("flower.jpeg"); + MemoryStream stream = new MemoryStream(byteArray); + string fileName = $"{GcsSourcePath}{DateTime.Now.ToString("yyyyMMddHHmmss")}.txt"; + storage.UploadObject(bucketName, fileName, "application/octet-stream", stream); + } + public void Dispose() { try diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index 02d941cabc1..7c22c2bbc32 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -15,6 +15,7 @@ using System; using System.IO; using System.Linq; +using System.Xml.Linq; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using Xunit; @@ -37,25 +38,22 @@ public void TransferBetweenPosix() Directory.CreateDirectory(_fixture.TempDirectory); Directory.CreateDirectory(_fixture.TempDestinationDirectory); string sourceDir = _fixture.RootDirectory; - string[] txtList = Directory.GetFiles(sourceDir, "*.txt"); - // Copy one txt file. - foreach (string f in txtList) + string fileName = Path.Combine(_fixture.TempDirectory, "test.txt"); + // Check if file already exists. If yes, delete it. + if (File.Exists(fileName)) { - // Remove path from the file name. - string fName = f.Split('/').Last(); - // Copy only one file from source directory to temp source directory. Will overwrite if the destination file already exists. - File.Copy(Path.Combine(sourceDir, fName), Path.Combine(_fixture.TempDirectory, fName), true); - break; + File.Delete(fileName); + } + using (StreamWriter sw = new StreamWriter(fileName)) + { + sw.WriteLine("test message"); } - var storage = StorageClient.Create(); var transferJob = transferBetweenPosixSample.TransferBetweenPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.SinkAgentPoolName, _fixture.TempDirectory, _fixture.TempDestinationDirectory, _fixture.BucketNameSource); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; Assert.True(Directory.Exists(_fixture.TempDirectory)); Assert.True(Directory.Exists(_fixture.TempDestinationDirectory)); - Assert.True(File.Exists(txtList[0])); - string sourceFilePath = txtList[0].Replace(_fixture.RootDirectory.Substring(0, _fixture.RootDirectory.Length - 1), _fixture.TempDirectory); - Assert.True(File.Exists(sourceFilePath)); + Assert.True(File.Exists(fileName)); } public void Dispose() diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index 3181c95eed7..7149781b88b 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -13,8 +13,6 @@ // limitations under the License. using System; -using System.IO; -using System.Text; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using Xunit; @@ -35,11 +33,7 @@ public TransferUsingManifestTest(StorageFixture fixture) public void TransferUsingManifest() { TransferUsingManifestSample transferUsingManifestSample = new TransferUsingManifestSample(); - var storage = StorageClient.Create(); - byte[] byteArray = Encoding.UTF8.GetBytes("flower.jpeg"); - MemoryStream stream = new MemoryStream(byteArray); - storage.UploadObject(_fixture.BucketNameSource, _fixture.ManifestObjectName, "application/octet-stream", stream); - var transferJob = transferUsingManifestSample.TransferUsingManifest(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.RootDirectory, _fixture.BucketNameSource, _fixture.BucketNameSink, _fixture.ManifestObjectName); + var transferJob = transferUsingManifestSample.TransferUsingManifest(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.RootDirectory, _fixture.BucketNameManifestSource, _fixture.BucketNameSink, _fixture.ManifestObjectName); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs index e6c7614d227..58b3258563e 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -16,8 +16,6 @@ using System; using Google.Cloud.StorageTransfer.V1; -namespace StorageTransfer.Samples -{ public class CheckLatestTransferOperationSample { //Checks the latest transfer operation for a given transfer job. @@ -59,5 +57,5 @@ public TransferJob CheckLatestTransferOperation( } } } -} + // [END storagetransfer_get_latest_transfer_operation] diff --git a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs index 717d3ed6207..a7363b9be93 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs @@ -18,8 +18,6 @@ using System; using Google.Cloud.StorageTransfer.V1; -namespace StorageTransfer.Samples -{ public class CreateEventDrivenGcsTransferSample { @@ -57,6 +55,6 @@ public TransferJob CreateEventDrivenGcsTransfer( return response; } } -} + // [END storagetransfer_create_event_driven_gcs_transfer] diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index 719c3553ce6..e1c1feedfb2 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -16,11 +16,9 @@ using System; using Google.Cloud.StorageTransfer.V1; -namespace StorageTransfer.Samples -{ public class DownloadToPosixSample { - /*Create a transfer from a GCS bucket to a POSIX file system.*/ + // Create a transfer from a GCS bucket to a POSIX file system. public TransferJob DownloadToPosix( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -33,7 +31,7 @@ public TransferJob DownloadToPosix( // The root directory path on the source filesystem string rootDirectory = "/tmp/uploads") { - // # A useful description for your transfer job + // A useful description for your transfer job string jobDescription = $"Download objects from a GCS source bucket ({gcsSourceBucket}/{gcsSourcePath}) to the root directory of POSIX file system"; TransferJob transferJob = new TransferJob @@ -68,7 +66,7 @@ public TransferJob DownloadToPosix( } } -} + //[END storagetransfer_download_to_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs index 3adcc527053..bbc1003ef6c 100644 --- a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs @@ -18,9 +18,6 @@ using System; using Google.Cloud.StorageTransfer.V1; - -namespace StorageTransfer.Samples -{ public class QuickstartSample { public TransferJob Quickstart( @@ -55,5 +52,5 @@ public TransferJob Quickstart( return response; } } -} + // [END storagetransfer_quickstart] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs index 6dcf7d9ea3c..b48a20d1045 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -16,11 +16,9 @@ using System; using Google.Cloud.StorageTransfer.V1; -namespace StorageTransfer.Samples -{ public class TransferBetweenPosixSample { - /*Creates a transfer between POSIX file systems.*/ + // Creates a transfer between POSIX file systems. public TransferJob TransferBetweenPosix( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -71,7 +69,7 @@ public TransferJob TransferBetweenPosix( } } -} + //[END storagetransfer_transfer_posix_to_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs index 69af8cad20a..7fc30f98650 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs @@ -16,11 +16,9 @@ using System; using Google.Cloud.StorageTransfer.V1; -namespace StorageTransfer.Samples -{ public class TransferFromPosixSample { - /*Create a transfer from a POSIX file system to a GCS sink bucket*/ + // Create a transfer from a POSIX file system to a GCS sink bucket public TransferJob TransferFromPosix( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -67,7 +65,7 @@ public TransferJob TransferFromPosix( } } -} + // [END storagetransfer_transfer_from_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index 3f9ad9b655c..d946b04f4cd 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -17,13 +17,10 @@ using Google.Cloud.StorageTransfer.V1; using Google.Protobuf.WellKnownTypes; -namespace StorageTransfer.Samples -{ - public class TransferToNearlineSample { - /*Creates a one-off transfer job that transfers objects from a standard GCS bucket that are more - than 30 days old to a Nearline GCS bucket.*/ + //Creates a one-off transfer job that transfers objects from a standard GCS bucket that are more + // than 30 days old to a Nearline GCS bucket. public TransferJob TransferToNearline( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -63,5 +60,5 @@ public TransferJob TransferToNearline( return response; } } -} + // [END storagetransfer_transfer_to_nearline] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index 41db94e96f0..32b9ab884e0 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -17,12 +17,9 @@ using System; using Google.Cloud.StorageTransfer.V1; -namespace StorageTransfer.Samples -{ public class TransferUsingManifestSample { - /*Create a transfer from a POSIX file system to a GCS bucket using - a manifest file*/ + //Create a transfer from a POSIX file system to a GCS bucket using a manifest file*/ public TransferJob TransferUsingManifest( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -72,9 +69,6 @@ public TransferJob TransferUsingManifest( Console.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} using manifest file {manifestLocation} with the name {response.Name}"); return response; - - } } -} // [END storagetransfer_manifest_request] From f57ea408cb20c77ebd4c6507126044748005ea0c Mon Sep 17 00:00:00 2001 From: Mahendra Date: Thu, 30 Jan 2025 15:42:38 +0530 Subject: [PATCH 21/58] Copyright year reverted for old files --- .../api/StorageTransfer.Samples.Tests/StorageFixture.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index e0e084fb9cf..087d111872d 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -1,5 +1,5 @@ /** - * Copyright 2024 Google Inc. + * Copyright 2021 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From a2192fc33480cc7fa1163a67331566e6884b6deb Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 30 Jan 2025 02:57:08 -0800 Subject: [PATCH 22/58] Blank line removed --- .../api/StorageTransfer.Samples/TransferUsingManifestSample.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index 32b9ab884e0..e279e8bcf6c 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -54,7 +54,6 @@ public TransferJob TransferUsingManifest( Status = TransferJob.Types.Status.Enabled, }; - // Create a Transfer Service client StorageTransferServiceClient client = StorageTransferServiceClient.Create(); From 2d068d29d03b074f52dc852b488180b980e58192 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 30 Jan 2025 05:30:23 -0800 Subject: [PATCH 23/58] changes related to blank spaces --- .../CheckLatestTransferOperationSample.cs | 7 +++---- .../CreateEventDrivenGcsTransferSample.cs | 5 +---- .../api/StorageTransfer.Samples/DownloadToPosixSample.cs | 7 ++----- .../api/StorageTransfer.Samples/QuickstartSample.cs | 3 +-- .../StorageTransfer.Samples/TransferBetweenPosixSample.cs | 6 ++---- .../StorageTransfer.Samples/TransferFromPosixSample.cs | 8 ++------ .../StorageTransfer.Samples/TransferToNearlineSample.cs | 8 +++++--- .../TransferUsingManifestSample.cs | 2 +- 8 files changed, 17 insertions(+), 29 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs index 58b3258563e..f17b6f604b1 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + // [START storagetransfer_get_latest_transfer_operation] using System; @@ -18,7 +19,7 @@ public class CheckLatestTransferOperationSample { - //Checks the latest transfer operation for a given transfer job. + // Checks the latest transfer operation for a given transfer job public TransferJob CheckLatestTransferOperation( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -29,9 +30,9 @@ public TransferJob CheckLatestTransferOperation( { throw new Exception("JobName can not be null or empty"); } + // Create a Transfer Service client StorageTransferServiceClient storageTransfer = StorageTransferServiceClient.Create(); - GetTransferJobRequest getTransferJobRequest = new GetTransferJobRequest { ProjectId = projectId, JobName = jobName }; try { @@ -40,7 +41,6 @@ public TransferJob CheckLatestTransferOperation( // Get Latest operation name from transfer job string latestOperationName = transferJob.LatestOperationName; - if (!string.IsNullOrEmpty(latestOperationName)) { Console.WriteLine("The latest operation for transfer job " + jobName + " is: " + latestOperationName + ""); @@ -57,5 +57,4 @@ public TransferJob CheckLatestTransferOperation( } } } - // [END storagetransfer_get_latest_transfer_operation] diff --git a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs index a7363b9be93..de783bdf219 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + // [START storagetransfer_create_event_driven_gcs_transfer] using System; @@ -20,7 +21,6 @@ public class CreateEventDrivenGcsTransferSample { - public TransferJob CreateEventDrivenGcsTransfer( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -42,7 +42,6 @@ public TransferJob CreateEventDrivenGcsTransfer( { GcsDataSink = new GcsData { BucketName = sinkBucket }, GcsDataSource = new GcsData { BucketName = sourceBucket }, - }, Status = TransferJob.Types.Status.Enabled, EventStream = new EventStream { Name = pubSubId } @@ -51,10 +50,8 @@ public TransferJob CreateEventDrivenGcsTransfer( StorageTransferServiceClient client = StorageTransferServiceClient.Create(); TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); Console.WriteLine($"Created an event driven transfer job from {sourceBucket} to {sinkBucket} subscribed to {pubSubId} with name {response.Name}"); - return response; } } - // [END storagetransfer_create_event_driven_gcs_transfer] diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index e1c1feedfb2..5c70ee2b6a9 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + // [START storagetransfer_download_to_posix] using System; @@ -18,7 +19,7 @@ public class DownloadToPosixSample { - // Create a transfer from a GCS bucket to a POSIX file system. + // Create a transfer from a GCS bucket to a POSIX file system public TransferJob DownloadToPosix( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -47,7 +48,6 @@ public TransferJob DownloadToPosix( Status = TransferJob.Types.Status.Enabled, }; - // Create a Transfer Service client StorageTransferServiceClient client = StorageTransferServiceClient.Create(); @@ -62,11 +62,8 @@ public TransferJob DownloadToPosix( Console.WriteLine($"Created and ran transfer job from ({gcsSourceBucket}/{gcsSourcePath}) to {rootDirectory} with the name {response.Name}"); return response; - - } } - //[END storagetransfer_download_to_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs index bbc1003ef6c..76295086cbf 100644 --- a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + // [START storagetransfer_quickstart] using System; @@ -48,9 +49,7 @@ public TransferJob Quickstart( }); Console.WriteLine($"Created and ran transfer job from {sourceBucket} to {sinkBucket} with name {response.Name}"); - return response; } } - // [END storagetransfer_quickstart] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs index b48a20d1045..807e8d998f1 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + // [START storagetransfer_transfer_posix_to_posix] using System; @@ -18,7 +19,7 @@ public class TransferBetweenPosixSample { - // Creates a transfer between POSIX file systems. + // Creates a transfer between POSIX file systems public TransferJob TransferBetweenPosix( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -51,7 +52,6 @@ public TransferJob TransferBetweenPosix( Status = TransferJob.Types.Status.Enabled, }; - // Create a Transfer Service client StorageTransferServiceClient client = StorageTransferServiceClient.Create(); @@ -66,10 +66,8 @@ public TransferJob TransferBetweenPosix( Console.WriteLine($"Created and ran transfer job from {rootDirectory} to {destinationDirectory} with the name {response.Name}"); return response; - } } - //[END storagetransfer_transfer_posix_to_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs index 7fc30f98650..b46df62cf89 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + // [START storagetransfer_transfer_from_posix] using System; @@ -18,7 +19,7 @@ public class TransferFromPosixSample { - // Create a transfer from a POSIX file system to a GCS sink bucket + // Create a transfer from a POSIX file system to a GCS sink bucket public TransferJob TransferFromPosix( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -29,7 +30,6 @@ public TransferJob TransferFromPosix( // The GCS bucket to transfer data to string sinkBucket = "my-sink-bucket") { - // A useful description for your transfer job string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket})"; @@ -46,7 +46,6 @@ public TransferJob TransferFromPosix( Status = TransferJob.Types.Status.Enabled, }; - // Create a Transfer Service client StorageTransferServiceClient client = StorageTransferServiceClient.Create(); @@ -61,11 +60,8 @@ public TransferJob TransferFromPosix( Console.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} with the name {response.Name}"); return response; - - } } - // [END storagetransfer_transfer_from_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index d946b04f4cd..5d97f7204a8 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + // [START storagetransfer_transfer_to_nearline] using System; @@ -19,8 +20,8 @@ public class TransferToNearlineSample { - //Creates a one-off transfer job that transfers objects from a standard GCS bucket that are more - // than 30 days old to a Nearline GCS bucket. + // Creates a one-off transfer job that transfers objects from a standard GCS bucket that are more + // than 30 days old to a Nearline GCS bucket public TransferJob TransferToNearline( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -46,8 +47,10 @@ public TransferJob TransferToNearline( Status = TransferJob.Types.Status.Enabled, Schedule = new Schedule { ScheduleStartDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)), ScheduleEndDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)) } }; + // Create a Transfer Service client StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + // Create a Transfer job TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); client.RunTransferJob(new RunTransferJobRequest @@ -60,5 +63,4 @@ public TransferJob TransferToNearline( return response; } } - // [END storagetransfer_transfer_to_nearline] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index e279e8bcf6c..73fab93c6fd 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -19,7 +19,7 @@ public class TransferUsingManifestSample { - //Create a transfer from a POSIX file system to a GCS bucket using a manifest file*/ + // Create a transfer from a POSIX file system to a GCS bucket using a manifest file public TransferJob TransferUsingManifest( // Your Google Cloud Project ID string projectId = "my-project-id", From 2e7aa6229e2d115d2690ac6b73b749af0bac6684 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 30 Jan 2025 23:43:16 -0800 Subject: [PATCH 24/58] Blank spaces reverted in QuickStartTest --- .../api/StorageTransfer.Samples.Tests/QuickstartTest.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs index b72b1900c5f..33fa4dd687d 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + using System; using Google.Cloud.StorageTransfer.V1; using Xunit; @@ -24,6 +25,7 @@ public class QuickstartTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; + public QuickstartTest(StorageFixture fixture) { _fixture = fixture; From 02a0e140b97ff35962d08a8bebbf84715ca62259 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Mon, 3 Feb 2025 01:51:30 -0800 Subject: [PATCH 25/58] resources for event driven transfer are moved from storage fixture to test --- .../CreateEventDrivenGcsTransferTest.cs | 50 ++++++++++++++++- .../StorageFixture.cs | 54 +------------------ 2 files changed, 50 insertions(+), 54 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs index 7e3f2cb1724..e40c7971474 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -15,6 +15,8 @@ */ using System; +using Google.Cloud.PubSub.V1; +using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using Xunit; @@ -24,17 +26,58 @@ namespace StorageTransfer.Samples.Tests public class CreateEventDrivenGcsTransferTest : IDisposable { private readonly StorageFixture _fixture; + private readonly StorageTransferServiceClient _sts; + private readonly string _pubSubId; private string _transferJobName; + private string TopicId { get; } = "DotNetTopic" + Guid.NewGuid().ToString(); + private string SubscriptionId { get; } = "DotNetSubscription" + Guid.NewGuid().ToString(); + private SubscriberServiceApiClient SubscriberClient { get; } = SubscriberServiceApiClient.Create(); + private PublisherServiceApiClient PublisherClient { get; } = PublisherServiceApiClient.Create(); public CreateEventDrivenGcsTransferTest(StorageFixture fixture) { _fixture = fixture; + _pubSubId = $"projects/{_fixture.ProjectId}/subscriptions/{SubscriptionId}"; + string email = _fixture.Sts.GetGoogleServiceAccount(new GetGoogleServiceAccountRequest() + { + ProjectId = _fixture.ProjectId + }).AccountEmail; + + string memberServiceAccount = "serviceAccount:" + email; + // Create subscription name + SubscriptionName subscriptionName = new SubscriptionName(_fixture.ProjectId, SubscriptionId); + // Create topic name + TopicName topicName = new TopicName(_fixture.ProjectId, TopicId); + // Create topic + PublisherClient.CreateTopic(topicName); + // Create subscription. + SubscriberClient.CreateSubscription(subscriptionName, topicName, pushConfig: null, ackDeadlineSeconds: 500); + + var policyIamPolicyTopic = new Google.Cloud.Iam.V1.Policy(); + + policyIamPolicyTopic.AddRoleMember("roles/pubsub.publisher", memberServiceAccount); + + PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest + { + ResourceAsResourceName = topicName, + Policy = policyIamPolicyTopic + }); + + var policyIamPolicySubscriber = new Google.Cloud.Iam.V1.Policy(); + + policyIamPolicySubscriber.AddRoleMember("roles/pubsub.subscriber", memberServiceAccount); + + PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest + { + ResourceAsResourceName = subscriptionName, + Policy = policyIamPolicySubscriber + }); } [Fact] public void CreateEventDrivenGcsTransfer() { CreateEventDrivenGcsTransferSample createEventDrivenGcsTransferSample = new CreateEventDrivenGcsTransferSample(); - var transferJob = createEventDrivenGcsTransferSample.CreateEventDrivenGcsTransfer(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink, _fixture.PubSubId); + var transferJob = createEventDrivenGcsTransferSample.CreateEventDrivenGcsTransfer(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink, _pubSubId); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } @@ -53,6 +96,11 @@ public void Dispose() Status = TransferJob.Types.Status.Deleted } }); + + TopicName topicName = TopicName.FromProjectTopic(_fixture.ProjectId, TopicId); + PublisherClient.DeleteTopic(topicName); + SubscriptionName subscriptionName = SubscriptionName.FromProjectSubscription(_fixture.ProjectId, SubscriptionId); + SubscriberClient.DeleteSubscription(subscriptionName); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index 087d111872d..fc5b73438b3 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -44,21 +44,13 @@ public class StorageFixture : IDisposable, ICollectionFixture public string TempDestinationDirectory { get; } = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); public StorageClient Storage { get; } = StorageClient.Create(); public string ManifestObjectName { get; } = "manifest.csv"; - public string TopicId { get; } = "DotNetTopic" + Guid.NewGuid().ToString(); - public string SubscriptionId { get; } = "DotNetSubscription" + Guid.NewGuid().ToString(); - public string PubSubId { get; } public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create(); - public SubscriberServiceApiClient SubscriberClient { get; } = SubscriberServiceApiClient.Create(); - - public PublisherServiceApiClient PublisherClient { get; } = PublisherServiceApiClient.Create(); - public StorageFixture() { ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); SourceAgentPoolName = $"projects/{ProjectId}/agentPools/transfer_service_default"; SinkAgentPoolName = $"projects/{ProjectId}/agentPools/transfer_service_default"; - PubSubId = $"projects/{ProjectId}/subscriptions/{SubscriptionId}"; GcsSourcePath = "foo/bar/"; if (string.IsNullOrWhiteSpace(ProjectId)) { @@ -94,28 +86,6 @@ public StorageFixture() ProjectId = ProjectId }).AccountEmail; string memberServiceAccount = "serviceAccount:" + email; - // Create subscription name - SubscriptionName subscriptionName = new SubscriptionName(ProjectId, SubscriptionId); - // Create topic name - TopicName topicName = new TopicName(ProjectId, TopicId); - // Create topic - PublisherClient.CreateTopic(topicName); - // Create subscription. - SubscriberClient.CreateSubscription(subscriptionName, topicName, pushConfig: null, ackDeadlineSeconds: 500); - var policyIamPolicyTopic = new Google.Cloud.Iam.V1.Policy(); - policyIamPolicyTopic.AddRoleMember("roles/pubsub.publisher", memberServiceAccount); - PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest - { - ResourceAsResourceName = topicName, - Policy = policyIamPolicyTopic - }); - var policyIamPolicySubscriber = new Google.Cloud.Iam.V1.Policy(); - policyIamPolicySubscriber.AddRoleMember("roles/pubsub.subscriber", memberServiceAccount); - PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest - { - ResourceAsResourceName = subscriptionName, - Policy = policyIamPolicySubscriber - }); } private void CreateBucketAndGrantStsPermissions(string bucketName) @@ -167,7 +137,7 @@ private void UploadObjectToManifestBucket(string bucketName) var storage = StorageClient.Create(); byte[] byteArray = System.Text.Encoding.UTF8.GetBytes("flower.jpeg"); MemoryStream stream = new MemoryStream(byteArray); - storage.UploadObject(bucketName,ManifestObjectName, "application/octet-stream", stream); + storage.UploadObject(bucketName, ManifestObjectName, "application/octet-stream", stream); } private void UploadObjectToPosixBucket(string bucketName) @@ -188,37 +158,15 @@ public void Dispose() catch (Exception) { // Do nothing, we delete on a best effort basis. - } try { Storage.DeleteBucket(BucketNameSource, new DeleteBucketOptions { DeleteObjects = true }); } catch (Exception) - { - // Do nothing, we delete on a best effort basis. - - } - try - { - TopicName topicName = TopicName.FromProjectTopic(ProjectId, TopicId); - PublisherClient.DeleteTopic(topicName); - } - catch (RpcException) { // Do nothing, we delete on a best effort basis. } - - try - { - SubscriptionName subscriptionName = SubscriptionName.FromProjectSubscription(ProjectId, SubscriptionId); - SubscriberClient.DeleteSubscription(subscriptionName); - } - catch (RpcException) - { - // Do nothing, we delete on a best effort basis. - } - } } } From 5e709004081caa6ba9ced75c3fe28e0cefdb9cf5 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Tue, 4 Feb 2025 03:25:46 -0800 Subject: [PATCH 26/58] movement of temporary folder path from storage fixture to respective tests --- .../CreateEventDrivenGcsTransferTest.cs | 1 - .../DownloadToPosixTest.cs | 10 +++++---- .../StorageFixture.cs | 9 ++------ .../TransferBetweenPosixTest.cs | 21 +++++++++++-------- .../TransferFromPosixTest.cs | 5 +++-- .../TransferUsingManifestTest.cs | 4 +++- 6 files changed, 26 insertions(+), 24 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs index e40c7971474..60833043c26 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -26,7 +26,6 @@ namespace StorageTransfer.Samples.Tests public class CreateEventDrivenGcsTransferTest : IDisposable { private readonly StorageFixture _fixture; - private readonly StorageTransferServiceClient _sts; private readonly string _pubSubId; private string _transferJobName; private string TopicId { get; } = "DotNetTopic" + Guid.NewGuid().ToString(); diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index aa60ca5deef..e23f6f4f1c7 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -27,19 +27,21 @@ public class DownloadToPosixTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; + private readonly string _tempDirectory; public DownloadToPosixTest(StorageFixture fixture) { _fixture = fixture; + _tempDirectory = fixture.GenerateTempFolderPath(); } [Fact] public void DownloadToPosix() { DownloadToPosixSample downloadToPosixSample = new DownloadToPosixSample(); - Directory.CreateDirectory(_fixture.TempDirectory); - var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId, _fixture.SinkAgentPoolName, _fixture.BucketNamePosixSource, _fixture.GcsSourcePath, _fixture.TempDirectory); + Directory.CreateDirectory(_tempDirectory); + var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId, _fixture.SinkAgentPoolName, _fixture.BucketNamePosixSource, _fixture.GcsSourcePath, _tempDirectory); Assert.Contains("transferJobs/", transferJob.Name); - Assert.True(Directory.Exists(_fixture.TempDirectory)); + Assert.True(Directory.Exists(_tempDirectory)); _transferJobName = transferJob.Name; } @@ -57,7 +59,7 @@ public void Dispose() Status = TransferJob.Types.Status.Deleted } }); - Directory.Delete(_fixture.TempDirectory, true); + Directory.Delete(_tempDirectory, true); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index fc5b73438b3..386cd12120d 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -18,10 +18,8 @@ using System.Collections.Generic; using System.IO; using Google.Apis.Storage.v1.Data; -using Google.Cloud.PubSub.V1; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; -using Grpc.Core; using Xunit; namespace StorageTransfer.Samples.Tests @@ -38,10 +36,6 @@ public class StorageFixture : IDisposable, ICollectionFixture public string SourceAgentPoolName { get; } public string SinkAgentPoolName { get; } public string GcsSourcePath { get; } - public string RootDirectory { get; } = System.IO.Path.GetTempPath(); - public string DestinationDirectory { get; } = System.IO.Path.GetTempPath(); - public string TempDirectory { get; } = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); - public string TempDestinationDirectory { get; } = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); public StorageClient Storage { get; } = StorageClient.Create(); public string ManifestObjectName { get; } = "manifest.csv"; public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create(); @@ -148,7 +142,8 @@ private void UploadObjectToPosixBucket(string bucketName) string fileName = $"{GcsSourcePath}{DateTime.Now.ToString("yyyyMMddHHmmss")}.txt"; storage.UploadObject(bucketName, fileName, "application/octet-stream", stream); } - + internal string GetCurrentUserTempFolderPath() => System.IO.Path.GetTempPath(); + internal string GenerateTempFolderPath() => Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); public void Dispose() { try diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index 7c22c2bbc32..4185a094b4f 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -26,19 +26,22 @@ public class TransferBetweenPosixTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; + private readonly string _tempDirectory; + private readonly string _tempDestinationDirectory; public TransferBetweenPosixTest(StorageFixture fixture) { _fixture = fixture; + _tempDirectory = fixture.GenerateTempFolderPath(); + _tempDestinationDirectory = fixture.GenerateTempFolderPath(); } [Fact] public void TransferBetweenPosix() { TransferBetweenPosixSample transferBetweenPosixSample = new TransferBetweenPosixSample(); - Directory.CreateDirectory(_fixture.TempDirectory); - Directory.CreateDirectory(_fixture.TempDestinationDirectory); - string sourceDir = _fixture.RootDirectory; - string fileName = Path.Combine(_fixture.TempDirectory, "test.txt"); + Directory.CreateDirectory(_tempDirectory); + Directory.CreateDirectory(_tempDestinationDirectory); + string fileName = Path.Combine(_tempDirectory, "test.txt"); // Check if file already exists. If yes, delete it. if (File.Exists(fileName)) { @@ -48,11 +51,11 @@ public void TransferBetweenPosix() { sw.WriteLine("test message"); } - var transferJob = transferBetweenPosixSample.TransferBetweenPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.SinkAgentPoolName, _fixture.TempDirectory, _fixture.TempDestinationDirectory, _fixture.BucketNameSource); + var transferJob = transferBetweenPosixSample.TransferBetweenPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.SinkAgentPoolName, _tempDirectory, _tempDestinationDirectory, _fixture.BucketNameSource); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; - Assert.True(Directory.Exists(_fixture.TempDirectory)); - Assert.True(Directory.Exists(_fixture.TempDestinationDirectory)); + Assert.True(Directory.Exists(_tempDirectory)); + Assert.True(Directory.Exists(_tempDestinationDirectory)); Assert.True(File.Exists(fileName)); } @@ -70,8 +73,8 @@ public void Dispose() Status = TransferJob.Types.Status.Deleted } }); - Directory.Delete(_fixture.TempDirectory, true); - Directory.Delete(_fixture.TempDestinationDirectory, true); + Directory.Delete(_tempDirectory, true); + Directory.Delete(_tempDestinationDirectory, true); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs index f9f0d3e68c9..0cfeab95a4a 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs @@ -23,17 +23,18 @@ public class TransferFromPosixTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; + private readonly string _rootDirectory; public TransferFromPosixTest(StorageFixture fixture) { _fixture = fixture; + _rootDirectory = fixture.GetCurrentUserTempFolderPath(); } [Fact] public void TransferFromPosix() { TransferFromPosixSample transferFromPosixSample = new TransferFromPosixSample(); - var transferJob = transferFromPosixSample.TransferFromPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.RootDirectory, _fixture.BucketNameSink); - var storage = StorageClient.Create(); + var transferJob = transferFromPosixSample.TransferFromPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _rootDirectory, _fixture.BucketNameSink); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index 7149781b88b..ddd6c5e1662 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -24,16 +24,18 @@ public class TransferUsingManifestTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; + private readonly string _rootDirectory; public TransferUsingManifestTest(StorageFixture fixture) { _fixture = fixture; + _rootDirectory = fixture.GetCurrentUserTempFolderPath(); } [Fact] public void TransferUsingManifest() { TransferUsingManifestSample transferUsingManifestSample = new TransferUsingManifestSample(); - var transferJob = transferUsingManifestSample.TransferUsingManifest(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.RootDirectory, _fixture.BucketNameManifestSource, _fixture.BucketNameSink, _fixture.ManifestObjectName); + var transferJob = transferUsingManifestSample.TransferUsingManifest(_fixture.ProjectId, _fixture.SourceAgentPoolName, _rootDirectory, _fixture.BucketNameManifestSource, _fixture.BucketNameSink, _fixture.ManifestObjectName); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } From d9421774384122c52bf101fef8b9108b3b5b27a7 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 5 Feb 2025 03:01:39 -0800 Subject: [PATCH 27/58] gcssourcepath , manifestobjectname is moved from fixture to test --- .../DownloadToPosixTest.cs | 4 +++- .../StorageFixture.cs | 14 ++++---------- .../TransferUsingManifestTest.cs | 4 +++- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index e23f6f4f1c7..325905aa0bb 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -28,10 +28,12 @@ public class DownloadToPosixTest : IDisposable private readonly StorageFixture _fixture; private string _transferJobName; private readonly string _tempDirectory; + private readonly string _gcsSourcePath; public DownloadToPosixTest(StorageFixture fixture) { _fixture = fixture; _tempDirectory = fixture.GenerateTempFolderPath(); + _gcsSourcePath = "foo/bar/"; } [Fact] @@ -39,7 +41,7 @@ public void DownloadToPosix() { DownloadToPosixSample downloadToPosixSample = new DownloadToPosixSample(); Directory.CreateDirectory(_tempDirectory); - var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId, _fixture.SinkAgentPoolName, _fixture.BucketNamePosixSource, _fixture.GcsSourcePath, _tempDirectory); + var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId, _fixture.SinkAgentPoolName, _fixture.BucketNamePosixSource, _gcsSourcePath, _tempDirectory); Assert.Contains("transferJobs/", transferJob.Name); Assert.True(Directory.Exists(_tempDirectory)); _transferJobName = transferJob.Name; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index 386cd12120d..f1b4780c709 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -35,9 +35,7 @@ public class StorageFixture : IDisposable, ICollectionFixture public string JobName { get; } public string SourceAgentPoolName { get; } public string SinkAgentPoolName { get; } - public string GcsSourcePath { get; } public StorageClient Storage { get; } = StorageClient.Create(); - public string ManifestObjectName { get; } = "manifest.csv"; public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create(); public StorageFixture() @@ -45,7 +43,6 @@ public StorageFixture() ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); SourceAgentPoolName = $"projects/{ProjectId}/agentPools/transfer_service_default"; SinkAgentPoolName = $"projects/{ProjectId}/agentPools/transfer_service_default"; - GcsSourcePath = "foo/bar/"; if (string.IsNullOrWhiteSpace(ProjectId)) { throw new Exception("You need to set the Environment variable 'GOOGLE_PROJECT_ID' with your Google Cloud Project's project id."); @@ -75,11 +72,6 @@ public StorageFixture() // Make the request TransferJob response = Sts.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); JobName = response.Name; - string email = Sts.GetGoogleServiceAccount(new GetGoogleServiceAccountRequest() - { - ProjectId = ProjectId - }).AccountEmail; - string memberServiceAccount = "serviceAccount:" + email; } private void CreateBucketAndGrantStsPermissions(string bucketName) @@ -128,18 +120,20 @@ private void CreateBucketAndGrantStsPermissions(string bucketName) private void UploadObjectToManifestBucket(string bucketName) { + var manifestObjectName = "manifest.csv"; var storage = StorageClient.Create(); byte[] byteArray = System.Text.Encoding.UTF8.GetBytes("flower.jpeg"); MemoryStream stream = new MemoryStream(byteArray); - storage.UploadObject(bucketName, ManifestObjectName, "application/octet-stream", stream); + storage.UploadObject(bucketName, manifestObjectName, "application/octet-stream", stream); } private void UploadObjectToPosixBucket(string bucketName) { var storage = StorageClient.Create(); + var gcsSourcePath = "foo/bar/"; byte[] byteArray = System.Text.Encoding.UTF8.GetBytes("flower.jpeg"); MemoryStream stream = new MemoryStream(byteArray); - string fileName = $"{GcsSourcePath}{DateTime.Now.ToString("yyyyMMddHHmmss")}.txt"; + string fileName = $"{gcsSourcePath}{DateTime.Now.ToString("yyyyMMddHHmmss")}.txt"; storage.UploadObject(bucketName, fileName, "application/octet-stream", stream); } internal string GetCurrentUserTempFolderPath() => System.IO.Path.GetTempPath(); diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index ddd6c5e1662..442b4c17aa4 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -25,17 +25,19 @@ public class TransferUsingManifestTest : IDisposable private readonly StorageFixture _fixture; private string _transferJobName; private readonly string _rootDirectory; + private readonly string _manifestObjectName; public TransferUsingManifestTest(StorageFixture fixture) { _fixture = fixture; _rootDirectory = fixture.GetCurrentUserTempFolderPath(); + _manifestObjectName = "manifest.csv"; } [Fact] public void TransferUsingManifest() { TransferUsingManifestSample transferUsingManifestSample = new TransferUsingManifestSample(); - var transferJob = transferUsingManifestSample.TransferUsingManifest(_fixture.ProjectId, _fixture.SourceAgentPoolName, _rootDirectory, _fixture.BucketNameManifestSource, _fixture.BucketNameSink, _fixture.ManifestObjectName); + var transferJob = transferUsingManifestSample.TransferUsingManifest(_fixture.ProjectId, _fixture.SourceAgentPoolName, _rootDirectory, _fixture.BucketNameManifestSource, _fixture.BucketNameSink, _manifestObjectName); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } From a59263c04ab6bf09111ae74d0363890e903af4ca Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 6 Feb 2025 03:36:53 -0800 Subject: [PATCH 28/58] UploadObjectToManifestBucket , UploadObjectToPosixBucket , CreateTransferJob moved to respective tests --- .../CheckLatestTransferOperationTest.cs | 27 +++++++++++- .../DownloadToPosixTest.cs | 10 ++++- .../StorageFixture.cs | 41 ------------------- .../TransferUsingManifestTest.cs | 11 ++++- 4 files changed, 45 insertions(+), 44 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs index 39d3dfce59e..a5cbd41b2aa 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs @@ -11,6 +11,8 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +using Google.Cloud.Storage.V1; +using Google.Cloud.StorageTransfer.V1; using Xunit; namespace StorageTransfer.Samples.Tests; @@ -21,17 +23,40 @@ public class CheckLatestTransferOperationTest private readonly StorageFixture _fixture; private string _jobName; + private readonly string _transferJobName; public CheckLatestTransferOperationTest(StorageFixture fixture) { _fixture = fixture; + _transferJobName = CreateTransferJob(); } [Fact] public void CheckLatestTransferOperation() { CheckLatestTransferOperationSample checkLatestTransferOperationSample = new CheckLatestTransferOperationSample(); - var transferJob = checkLatestTransferOperationSample.CheckLatestTransferOperation(_fixture.ProjectId, _fixture.JobName); + var transferJob = checkLatestTransferOperationSample.CheckLatestTransferOperation(_fixture.ProjectId, _transferJobName); Assert.Contains("transferJobs/", transferJob.Name); _jobName = transferJob.Name; } + private string CreateTransferJob() + { + // Initialize request argument(s) + TransferJob transferJob = new TransferJob + { + ProjectId = _fixture.ProjectId, + TransferSpec = new TransferSpec + { + GcsDataSink = new GcsData { BucketName = _fixture.BucketNameSource }, + GcsDataSource = new GcsData { BucketName = _fixture.BucketNameSink } + }, + Status = TransferJob.Types.Status.Enabled + }; + CreateTransferJobRequest request = new CreateTransferJobRequest + { + TransferJob = transferJob + }; + // Make the request + TransferJob response = _fixture.Sts.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + return response.Name; + } } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index 325905aa0bb..b808b2b0c2a 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -33,7 +33,8 @@ public DownloadToPosixTest(StorageFixture fixture) { _fixture = fixture; _tempDirectory = fixture.GenerateTempFolderPath(); - _gcsSourcePath = "foo/bar/"; + _gcsSourcePath = $"{Guid.NewGuid()}/{Guid.NewGuid()}/"; + UploadObjectToPosixBucket(_fixture.BucketNamePosixSource); } [Fact] @@ -46,6 +47,13 @@ public void DownloadToPosix() Assert.True(Directory.Exists(_tempDirectory)); _transferJobName = transferJob.Name; } + private void UploadObjectToPosixBucket(string bucketName) + { + byte[] byteArray = System.Text.Encoding.UTF8.GetBytes($@"{Guid.NewGuid()}.jpeg"); + MemoryStream stream = new MemoryStream(byteArray); + string fileName = $"{_gcsSourcePath}{DateTime.Now.ToString("yyyyMMddHHmmss")}.txt"; + _fixture.Storage.UploadObject(bucketName, fileName, "application/octet-stream", stream); + } public void Dispose() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index f1b4780c709..b942c5b21f6 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -32,7 +32,6 @@ public class StorageFixture : IDisposable, ICollectionFixture public string BucketNameSink { get; } = Guid.NewGuid().ToString(); public string BucketNameManifestSource { get; } = Guid.NewGuid().ToString(); public string BucketNamePosixSource { get; } = Guid.NewGuid().ToString(); - public string JobName { get; } public string SourceAgentPoolName { get; } public string SinkAgentPoolName { get; } public StorageClient Storage { get; } = StorageClient.Create(); @@ -52,28 +51,7 @@ public StorageFixture() CreateBucketAndGrantStsPermissions(BucketNameSource); CreateBucketAndGrantStsPermissions(BucketNameManifestSource); CreateBucketAndGrantStsPermissions(BucketNamePosixSource); - UploadObjectToManifestBucket(BucketNameManifestSource); - UploadObjectToPosixBucket(BucketNamePosixSource); - // Initialize request argument(s) - TransferJob transferJob = new TransferJob - { - ProjectId = ProjectId, - TransferSpec = new TransferSpec - { - GcsDataSink = new GcsData { BucketName = BucketNameSource }, - GcsDataSource = new GcsData { BucketName = BucketNameSink } - }, - Status = TransferJob.Types.Status.Enabled - }; - CreateTransferJobRequest request = new CreateTransferJobRequest - { - TransferJob = transferJob - }; - // Make the request - TransferJob response = Sts.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); - JobName = response.Name; } - private void CreateBucketAndGrantStsPermissions(string bucketName) { var bucket = Storage.CreateBucket(ProjectId, new Bucket @@ -117,25 +95,6 @@ private void CreateBucketAndGrantStsPermissions(string bucketName) policy.Bindings.Add(bucketWriterBinding); Storage.SetBucketIamPolicy(bucketName, policy); } - - private void UploadObjectToManifestBucket(string bucketName) - { - var manifestObjectName = "manifest.csv"; - var storage = StorageClient.Create(); - byte[] byteArray = System.Text.Encoding.UTF8.GetBytes("flower.jpeg"); - MemoryStream stream = new MemoryStream(byteArray); - storage.UploadObject(bucketName, manifestObjectName, "application/octet-stream", stream); - } - - private void UploadObjectToPosixBucket(string bucketName) - { - var storage = StorageClient.Create(); - var gcsSourcePath = "foo/bar/"; - byte[] byteArray = System.Text.Encoding.UTF8.GetBytes("flower.jpeg"); - MemoryStream stream = new MemoryStream(byteArray); - string fileName = $"{gcsSourcePath}{DateTime.Now.ToString("yyyyMMddHHmmss")}.txt"; - storage.UploadObject(bucketName, fileName, "application/octet-stream", stream); - } internal string GetCurrentUserTempFolderPath() => System.IO.Path.GetTempPath(); internal string GenerateTempFolderPath() => Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); public void Dispose() diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index 442b4c17aa4..b122e99f9c0 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -13,6 +13,7 @@ // limitations under the License. using System; +using System.IO; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using Xunit; @@ -30,7 +31,8 @@ public TransferUsingManifestTest(StorageFixture fixture) { _fixture = fixture; _rootDirectory = fixture.GetCurrentUserTempFolderPath(); - _manifestObjectName = "manifest.csv"; + _manifestObjectName = $@"{Guid.NewGuid()}.csv"; + UploadObjectToManifestBucket(_fixture.BucketNameManifestSource); } [Fact] @@ -42,6 +44,13 @@ public void TransferUsingManifest() _transferJobName = transferJob.Name; } + private void UploadObjectToManifestBucket(string bucketName) + { + byte[] byteArray = System.Text.Encoding.UTF8.GetBytes($@"{Guid.NewGuid()}.jpeg"); + MemoryStream stream = new MemoryStream(byteArray); + _fixture.Storage.UploadObject(bucketName, _manifestObjectName, "application/octet-stream", stream); + } + public void Dispose() { try From 7e2b4ddab872a46e73e2e0f51518c62a0b994fbe Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Mon, 10 Feb 2025 02:59:41 -0800 Subject: [PATCH 29/58] style and structure changes as per canonical sample --- .../CheckLatestTransferOperationSample.cs | 56 ++++++------ .../CreateEventDrivenGcsTransferSample.cs | 58 ++++++------ .../DownloadToPosixSample.cs | 78 ++++++++-------- .../QuickstartSample.cs | 54 ++++++------ .../TransferBetweenPosixSample.cs | 64 +++++++------- .../TransferFromPosixSample.cs | 74 ++++++++-------- .../TransferToNearlineSample.cs | 76 ++++++++-------- .../TransferUsingManifestSample.cs | 88 +++++++++---------- 8 files changed, 274 insertions(+), 274 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs index f17b6f604b1..7ef3dab7d55 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -17,44 +17,44 @@ using System; using Google.Cloud.StorageTransfer.V1; - public class CheckLatestTransferOperationSample - { - // Checks the latest transfer operation for a given transfer job - public TransferJob CheckLatestTransferOperation( +public class CheckLatestTransferOperationSample +{ + // Checks the latest transfer operation for a given transfer job + public TransferJob CheckLatestTransferOperation( // Your Google Cloud Project ID string projectId = "my-project-id", // The name of the job to check string jobName = "transferJobs/1234567890") + { + if (string.IsNullOrEmpty(jobName)) { - if (string.IsNullOrEmpty(jobName)) - { - throw new Exception("JobName can not be null or empty"); - } + throw new Exception("JobName can not be null or empty"); + } - // Create a Transfer Service client - StorageTransferServiceClient storageTransfer = StorageTransferServiceClient.Create(); - GetTransferJobRequest getTransferJobRequest = new GetTransferJobRequest { ProjectId = projectId, JobName = jobName }; - try - { - // Get Transfer job - TransferJob transferJob = storageTransfer.GetTransferJob(getTransferJobRequest); - // Get Latest operation name from transfer job - string latestOperationName = transferJob.LatestOperationName; + // Create a Transfer Service client + StorageTransferServiceClient storageTransfer = StorageTransferServiceClient.Create(); + GetTransferJobRequest getTransferJobRequest = new GetTransferJobRequest { ProjectId = projectId, JobName = jobName }; + try + { + // Get Transfer job + TransferJob transferJob = storageTransfer.GetTransferJob(getTransferJobRequest); + // Get Latest operation name from transfer job + string latestOperationName = transferJob.LatestOperationName; - if (!string.IsNullOrEmpty(latestOperationName)) - { - Console.WriteLine("The latest operation for transfer job " + jobName + " is: " + latestOperationName + ""); - } - else - { - Console.WriteLine("Transfer job " + jobName + " hasn't run yet, try again after the job has started running."); - } - return transferJob; + if (!string.IsNullOrEmpty(latestOperationName)) + { + Console.WriteLine("The latest operation for transfer job " + jobName + " is: " + latestOperationName + ""); } - catch (Exception) + else { - throw new Exception("Failed to get transfer job " + jobName + ""); + Console.WriteLine("Transfer job " + jobName + " hasn't run yet, try again after the job has started running."); } + return transferJob; + } + catch (Exception) + { + throw new Exception("Failed to get transfer job " + jobName + ""); } } +} // [END storagetransfer_get_latest_transfer_operation] diff --git a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs index de783bdf219..d57407d18dc 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs @@ -19,39 +19,39 @@ using System; using Google.Cloud.StorageTransfer.V1; - public class CreateEventDrivenGcsTransferSample +public class CreateEventDrivenGcsTransferSample +{ + public TransferJob CreateEventDrivenGcsTransfer( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The GCS bucket to transfer data from + string sourceBucket = "my-source-bucket", + // The GCS bucket to transfer data to + string sinkBucket = "my-sink-bucket", + // The subscription ID to a Pubsub queue to track + string pubSubId = "projects/PROJECT_NAME/subscriptions/SUBSCRIPTION_ID") { - public TransferJob CreateEventDrivenGcsTransfer( - // Your Google Cloud Project ID - string projectId = "my-project-id", - // The GCS bucket to transfer data from - string sourceBucket = "my-source-bucket", - // The GCS bucket to transfer data to - string sinkBucket = "my-sink-bucket", - // The subscription ID to a Pubsub queue to track - string pubSubId = "projects/PROJECT_NAME/subscriptions/SUBSCRIPTION_ID") - { - // A useful description for your transfer job - string jobDescription = $"Event driven gcs data transfer from {sourceBucket} to {sinkBucket} subscribed to {pubSubId} "; + // A useful description for your transfer job + string jobDescription = $"Event driven gcs data transfer from {sourceBucket} to {sinkBucket} subscribed to {pubSubId} "; - TransferJob transferJob = new TransferJob + TransferJob transferJob = new TransferJob + { + ProjectId = projectId, + Description = jobDescription, + TransferSpec = new TransferSpec { - ProjectId = projectId, - Description = jobDescription, - TransferSpec = new TransferSpec - { - GcsDataSink = new GcsData { BucketName = sinkBucket }, - GcsDataSource = new GcsData { BucketName = sourceBucket }, - }, - Status = TransferJob.Types.Status.Enabled, - EventStream = new EventStream { Name = pubSubId } - }; + GcsDataSink = new GcsData { BucketName = sinkBucket }, + GcsDataSource = new GcsData { BucketName = sourceBucket }, + }, + Status = TransferJob.Types.Status.Enabled, + EventStream = new EventStream { Name = pubSubId } + }; - StorageTransferServiceClient client = StorageTransferServiceClient.Create(); - TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); - Console.WriteLine($"Created an event driven transfer job from {sourceBucket} to {sinkBucket} subscribed to {pubSubId} with name {response.Name}"); - return response; - } + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + Console.WriteLine($"Created an event driven transfer job from {sourceBucket} to {sinkBucket} subscribed to {pubSubId} with name {response.Name}"); + return response; } +} // [END storagetransfer_create_event_driven_gcs_transfer] diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index 5c70ee2b6a9..5e15ee5b5de 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -17,53 +17,53 @@ using System; using Google.Cloud.StorageTransfer.V1; - public class DownloadToPosixSample +public class DownloadToPosixSample +{ + // Create a transfer from a GCS bucket to a POSIX file system + public TransferJob DownloadToPosix( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent + string sinkAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", + // Your GCS source bucket name + string gcsSourceBucket = "my-gcs-source-bucket", + // An optional path on the Google Cloud Storage bucket to download from + string gcsSourcePath = "foo/bar/", + // The root directory path on the source filesystem + string rootDirectory = "/tmp/uploads") { - // Create a transfer from a GCS bucket to a POSIX file system - public TransferJob DownloadToPosix( - // Your Google Cloud Project ID - string projectId = "my-project-id", - // The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent - string sinkAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", - // Your GCS source bucket name - string gcsSourceBucket = "my-gcs-source-bucket", - // An optional path on the Google Cloud Storage bucket to download from - string gcsSourcePath = "foo/bar/", - // The root directory path on the source filesystem - string rootDirectory = "/tmp/uploads") - { - // A useful description for your transfer job - string jobDescription = $"Download objects from a GCS source bucket ({gcsSourceBucket}/{gcsSourcePath}) to the root directory of POSIX file system"; + // A useful description for your transfer job + string jobDescription = $"Download objects from a GCS source bucket ({gcsSourceBucket}/{gcsSourcePath}) to the root directory of POSIX file system"; - TransferJob transferJob = new TransferJob + TransferJob transferJob = new TransferJob + { + ProjectId = projectId, + Description = jobDescription, + TransferSpec = new TransferSpec { - ProjectId = projectId, - Description = jobDescription, - TransferSpec = new TransferSpec - { - GcsDataSource = new GcsData { BucketName = gcsSourceBucket, Path = gcsSourcePath }, - SinkAgentPoolName = sinkAgentPoolName, - PosixDataSink = new PosixFilesystem { RootDirectory = rootDirectory } - }, - Status = TransferJob.Types.Status.Enabled, - }; + GcsDataSource = new GcsData { BucketName = gcsSourceBucket, Path = gcsSourcePath }, + SinkAgentPoolName = sinkAgentPoolName, + PosixDataSink = new PosixFilesystem { RootDirectory = rootDirectory } + }, + Status = TransferJob.Types.Status.Enabled, + }; - // Create a Transfer Service client - StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + // Create a Transfer Service client + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); - // Create a Transfer job - TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + // Create a Transfer job + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); - client.RunTransferJob(new RunTransferJobRequest - { - JobName = response.Name, - ProjectId = projectId - }); + client.RunTransferJob(new RunTransferJobRequest + { + JobName = response.Name, + ProjectId = projectId + }); - Console.WriteLine($"Created and ran transfer job from ({gcsSourceBucket}/{gcsSourcePath}) to {rootDirectory} with the name {response.Name}"); - return response; - } + Console.WriteLine($"Created and ran transfer job from ({gcsSourceBucket}/{gcsSourcePath}) to {rootDirectory} with the name {response.Name}"); + return response; } +} //[END storagetransfer_download_to_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs index 76295086cbf..9726dfb777c 100644 --- a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs @@ -19,37 +19,37 @@ using System; using Google.Cloud.StorageTransfer.V1; - public class QuickstartSample +public class QuickstartSample +{ + public TransferJob Quickstart( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The GCS bucket to transfer data from + string sourceBucket = "my-source-bucket", + // The GCS bucket to transfer data to + string sinkBucket = "my-sink-bucket") { - public TransferJob Quickstart( - // Your Google Cloud Project ID - string projectId = "my-project-id", - // The GCS bucket to transfer data from - string sourceBucket = "my-source-bucket", - // The GCS bucket to transfer data to - string sinkBucket = "my-sink-bucket") + TransferJob transferJob = new TransferJob { - TransferJob transferJob = new TransferJob + ProjectId = projectId, + TransferSpec = new TransferSpec { - ProjectId = projectId, - TransferSpec = new TransferSpec - { - GcsDataSink = new GcsData { BucketName = sinkBucket }, - GcsDataSource = new GcsData { BucketName = sourceBucket } - }, - Status = TransferJob.Types.Status.Enabled - }; + GcsDataSink = new GcsData { BucketName = sinkBucket }, + GcsDataSource = new GcsData { BucketName = sourceBucket } + }, + Status = TransferJob.Types.Status.Enabled + }; - StorageTransferServiceClient client = StorageTransferServiceClient.Create(); - TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); - client.RunTransferJob(new RunTransferJobRequest - { - JobName = response.Name, - ProjectId = projectId - }); + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + client.RunTransferJob(new RunTransferJobRequest + { + JobName = response.Name, + ProjectId = projectId + }); - Console.WriteLine($"Created and ran transfer job from {sourceBucket} to {sinkBucket} with name {response.Name}"); - return response; - } + Console.WriteLine($"Created and ran transfer job from {sourceBucket} to {sinkBucket} with name {response.Name}"); + return response; } +} // [END storagetransfer_quickstart] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs index 807e8d998f1..b07b0bdb118 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -17,10 +17,10 @@ using System; using Google.Cloud.StorageTransfer.V1; - public class TransferBetweenPosixSample - { - // Creates a transfer between POSIX file systems - public TransferJob TransferBetweenPosix( +public class TransferBetweenPosixSample +{ + // Creates a transfer between POSIX file systems + public TransferJob TransferBetweenPosix( // Your Google Cloud Project ID string projectId = "my-project-id", // The agent pool associated with the POSIX data source. If not provided, defaults to the default agent @@ -33,41 +33,41 @@ public TransferJob TransferBetweenPosix( string destinationDirectory = "/directory/to/transfer/sink", // The name of GCS bucket for intermediate storage string intermediate_bucket = "my-intermediate-bucket") - { - // A useful description for your transfer job - string jobDescription = $"Transfer objects from {rootDirectory} to the {destinationDirectory} between POSIX file system"; + { + // A useful description for your transfer job + string jobDescription = $"Transfer objects from {rootDirectory} to the {destinationDirectory} between POSIX file system"; - TransferJob transferJob = new TransferJob + TransferJob transferJob = new TransferJob + { + ProjectId = projectId, + Description = jobDescription, + TransferSpec = new TransferSpec { - ProjectId = projectId, - Description = jobDescription, - TransferSpec = new TransferSpec - { - SourceAgentPoolName = sourceAgentPoolName, - SinkAgentPoolName = sinkAgentPoolName, - PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory }, - PosixDataSink = new PosixFilesystem { RootDirectory = destinationDirectory }, - GcsIntermediateDataLocation = new GcsData { BucketName = intermediate_bucket } - }, - Status = TransferJob.Types.Status.Enabled, - }; + SourceAgentPoolName = sourceAgentPoolName, + SinkAgentPoolName = sinkAgentPoolName, + PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory }, + PosixDataSink = new PosixFilesystem { RootDirectory = destinationDirectory }, + GcsIntermediateDataLocation = new GcsData { BucketName = intermediate_bucket } + }, + Status = TransferJob.Types.Status.Enabled, + }; - // Create a Transfer Service client - StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + // Create a Transfer Service client + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); - // Create a Transfer job - TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + // Create a Transfer job + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); - client.RunTransferJob(new RunTransferJobRequest - { - JobName = response.Name, - ProjectId = projectId - }); + client.RunTransferJob(new RunTransferJobRequest + { + JobName = response.Name, + ProjectId = projectId + }); - Console.WriteLine($"Created and ran transfer job from {rootDirectory} to {destinationDirectory} with the name {response.Name}"); - return response; - } + Console.WriteLine($"Created and ran transfer job from {rootDirectory} to {destinationDirectory} with the name {response.Name}"); + return response; } +} //[END storagetransfer_transfer_posix_to_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs index b46df62cf89..60e6f29af2f 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs @@ -17,51 +17,51 @@ using System; using Google.Cloud.StorageTransfer.V1; - public class TransferFromPosixSample +public class TransferFromPosixSample +{ + // Create a transfer from a POSIX file system to a GCS sink bucket + public TransferJob TransferFromPosix( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The agent pool associated with the POSIX data source. If not provided, defaults to the default agent + string sourceAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", + // The root directory path on the source filesystem + string rootDirectory = "/tmp/uploads", + // The GCS bucket to transfer data to + string sinkBucket = "my-sink-bucket") { - // Create a transfer from a POSIX file system to a GCS sink bucket - public TransferJob TransferFromPosix( - // Your Google Cloud Project ID - string projectId = "my-project-id", - // The agent pool associated with the POSIX data source. If not provided, defaults to the default agent - string sourceAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", - // The root directory path on the source filesystem - string rootDirectory = "/tmp/uploads", - // The GCS bucket to transfer data to - string sinkBucket = "my-sink-bucket") - { - // A useful description for your transfer job - string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket})"; + // A useful description for your transfer job + string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket})"; - TransferJob transferJob = new TransferJob + TransferJob transferJob = new TransferJob + { + ProjectId = projectId, + Description = jobDescription, + TransferSpec = new TransferSpec { - ProjectId = projectId, - Description = jobDescription, - TransferSpec = new TransferSpec - { - GcsDataSink = new GcsData { BucketName = sinkBucket }, - SourceAgentPoolName = sourceAgentPoolName, - PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory } - }, - Status = TransferJob.Types.Status.Enabled, - }; + GcsDataSink = new GcsData { BucketName = sinkBucket }, + SourceAgentPoolName = sourceAgentPoolName, + PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory } + }, + Status = TransferJob.Types.Status.Enabled, + }; - // Create a Transfer Service client - StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + // Create a Transfer Service client + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); - // Create a Transfer job - TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + // Create a Transfer job + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); - client.RunTransferJob(new RunTransferJobRequest - { - JobName = response.Name, - ProjectId = projectId - }); + client.RunTransferJob(new RunTransferJobRequest + { + JobName = response.Name, + ProjectId = projectId + }); - Console.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} with the name {response.Name}"); - return response; - } + Console.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} with the name {response.Name}"); + return response; } +} // [END storagetransfer_transfer_from_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index 5d97f7204a8..f9bcfd443cb 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -18,49 +18,49 @@ using Google.Cloud.StorageTransfer.V1; using Google.Protobuf.WellKnownTypes; - public class TransferToNearlineSample +public class TransferToNearlineSample +{ + // Creates a one-off transfer job that transfers objects from a standard GCS bucket that are more + // than 30 days old to a Nearline GCS bucket + public TransferJob TransferToNearline( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The GCS bucket to transfer objects from + string sourceBucket = "my-source-bucket", + // The GCS Nearline bucket to transfer old objects to + string sinkBucket = "my-sink-bucket") { - // Creates a one-off transfer job that transfers objects from a standard GCS bucket that are more - // than 30 days old to a Nearline GCS bucket - public TransferJob TransferToNearline( - // Your Google Cloud Project ID - string projectId = "my-project-id", - // The GCS bucket to transfer objects from - string sourceBucket = "my-source-bucket", - // The GCS Nearline bucket to transfer old objects to - string sinkBucket = "my-sink-bucket") - { - // A description of this job - string jobDescription = $"Transfers old objects from standard bucket ({sourceBucket}) that haven't been modified in the last 30 days to a Nearline bucket ({sinkBucket})"; + // A description of this job + string jobDescription = $"Transfers old objects from standard bucket ({sourceBucket}) that haven't been modified in the last 30 days to a Nearline bucket ({sinkBucket})"; - TransferJob transferJob = new TransferJob + TransferJob transferJob = new TransferJob + { + ProjectId = projectId, + Description = jobDescription, + TransferSpec = new TransferSpec { - ProjectId = projectId, - Description = jobDescription, - TransferSpec = new TransferSpec - { - GcsDataSink = new GcsData { BucketName = sinkBucket }, - GcsDataSource = new GcsData { BucketName = sourceBucket }, - ObjectConditions = new ObjectConditions { MinTimeElapsedSinceLastModification = Duration.FromTimeSpan(TimeSpan.FromSeconds(2592000)) }, - TransferOptions = new TransferOptions { DeleteObjectsFromSourceAfterTransfer = true }, - }, - Status = TransferJob.Types.Status.Enabled, - Schedule = new Schedule { ScheduleStartDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)), ScheduleEndDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)) } - }; + GcsDataSink = new GcsData { BucketName = sinkBucket }, + GcsDataSource = new GcsData { BucketName = sourceBucket }, + ObjectConditions = new ObjectConditions { MinTimeElapsedSinceLastModification = Duration.FromTimeSpan(TimeSpan.FromSeconds(2592000)) }, + TransferOptions = new TransferOptions { DeleteObjectsFromSourceAfterTransfer = true }, + }, + Status = TransferJob.Types.Status.Enabled, + Schedule = new Schedule { ScheduleStartDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)), ScheduleEndDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)) } + }; - // Create a Transfer Service client - StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + // Create a Transfer Service client + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); - // Create a Transfer job - TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); - client.RunTransferJob(new RunTransferJobRequest - { - JobName = response.Name, - ProjectId = projectId - }); + // Create a Transfer job + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + client.RunTransferJob(new RunTransferJobRequest + { + JobName = response.Name, + ProjectId = projectId + }); - Console.WriteLine($"Created one-off transfer job from standard bucket {sourceBucket} to Nearline bucket {sinkBucket} with the name {response.Name}"); - return response; - } + Console.WriteLine($"Created one-off transfer job from standard bucket {sourceBucket} to Nearline bucket {sinkBucket} with the name {response.Name}"); + return response; } +} // [END storagetransfer_transfer_to_nearline] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index 73fab93c6fd..da490aa4b96 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -17,57 +17,57 @@ using System; using Google.Cloud.StorageTransfer.V1; - public class TransferUsingManifestSample +public class TransferUsingManifestSample +{ + // Create a transfer from a POSIX file system to a GCS bucket using a manifest file + public TransferJob TransferUsingManifest( + // Your Google Cloud Project ID + string projectId = "my-project-id", + // The agent pool associated with the POSIX data source. If not provided, defaults to the default agent + string sourceAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", + // The root directory path on the source filesystem + string rootDirectory = "/tmp/uploads", + // The GCS bucket which has your manifest file + string manifestBucket = "my-source-bucket", + // The GCS bucket to transfer data to + string sinkBucket = "my-sink-bucket", + // The name of the manifest file in manifestBucket that specifies which objects to transfer + string manifestObjectName = "path/to/manifest.csv") { - // Create a transfer from a POSIX file system to a GCS bucket using a manifest file - public TransferJob TransferUsingManifest( - // Your Google Cloud Project ID - string projectId = "my-project-id", - // The agent pool associated with the POSIX data source. If not provided, defaults to the default agent - string sourceAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", - // The root directory path on the source filesystem - string rootDirectory = "/tmp/uploads", - // The GCS bucket which has your manifest file - string manifestBucket = "my-source-bucket", - // The GCS bucket to transfer data to - string sinkBucket = "my-sink-bucket", - // The name of the manifest file in manifestBucket that specifies which objects to transfer - string manifestObjectName = "path/to/manifest.csv") - { - string manifestLocation = $"gs://{manifestBucket}/{manifestObjectName}"; + string manifestLocation = $"gs://{manifestBucket}/{manifestObjectName}"; - // A useful description for your transfer job - string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket}) using manifest file"; + // A useful description for your transfer job + string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket}) using manifest file"; - TransferJob transferJob = new TransferJob + TransferJob transferJob = new TransferJob + { + ProjectId = projectId, + Description = jobDescription, + TransferSpec = new TransferSpec { - ProjectId = projectId, - Description = jobDescription, - TransferSpec = new TransferSpec - { - GcsDataSink = new GcsData { BucketName = sinkBucket }, - GcsDataSource = new GcsData { BucketName = manifestBucket }, - SourceAgentPoolName = sourceAgentPoolName, - PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory }, - TransferManifest = new TransferManifest { Location = manifestLocation } - }, - Status = TransferJob.Types.Status.Enabled, - }; + GcsDataSink = new GcsData { BucketName = sinkBucket }, + GcsDataSource = new GcsData { BucketName = manifestBucket }, + SourceAgentPoolName = sourceAgentPoolName, + PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory }, + TransferManifest = new TransferManifest { Location = manifestLocation } + }, + Status = TransferJob.Types.Status.Enabled, + }; - // Create a Transfer Service client - StorageTransferServiceClient client = StorageTransferServiceClient.Create(); + // Create a Transfer Service client + StorageTransferServiceClient client = StorageTransferServiceClient.Create(); - // Create a Transfer job - TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); + // Create a Transfer job + TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); - client.RunTransferJob(new RunTransferJobRequest - { - JobName = response.Name, - ProjectId = projectId - }); + client.RunTransferJob(new RunTransferJobRequest + { + JobName = response.Name, + ProjectId = projectId + }); - Console.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} using manifest file {manifestLocation} with the name {response.Name}"); - return response; - } + Console.WriteLine($"Created and ran transfer job from {rootDirectory} to {sinkBucket} using manifest file {manifestLocation} with the name {response.Name}"); + return response; } +} // [END storagetransfer_manifest_request] From 332f78d72a13b0b75bb73da4dc2922bc84f266a9 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Tue, 11 Feb 2025 03:24:44 -0800 Subject: [PATCH 30/58] XML documentation is added to describe the sample methods --- .../CheckLatestTransferOperationSample.cs | 4 +++- .../CreateEventDrivenGcsTransferSample.cs | 3 +++ .../api/StorageTransfer.Samples/DownloadToPosixSample.cs | 4 +++- .../StorageTransfer.Samples/TransferBetweenPosixSample.cs | 4 +++- .../api/StorageTransfer.Samples/TransferFromPosixSample.cs | 4 +++- .../api/StorageTransfer.Samples/TransferToNearlineSample.cs | 6 ++++-- .../StorageTransfer.Samples/TransferUsingManifestSample.cs | 4 +++- 7 files changed, 22 insertions(+), 7 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs index 7ef3dab7d55..d3ab8445472 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -17,9 +17,11 @@ using System; using Google.Cloud.StorageTransfer.V1; +/// +/// Checks the latest transfer operation for a given transfer job. +/// public class CheckLatestTransferOperationSample { - // Checks the latest transfer operation for a given transfer job public TransferJob CheckLatestTransferOperation( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs index d57407d18dc..25f0549a8d9 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs @@ -19,6 +19,9 @@ using System; using Google.Cloud.StorageTransfer.V1; +/// +/// Create an event driven gcs data transfer from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. +/// public class CreateEventDrivenGcsTransferSample { public TransferJob CreateEventDrivenGcsTransfer( diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index 5e15ee5b5de..0d6aef6867e 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -17,9 +17,11 @@ using System; using Google.Cloud.StorageTransfer.V1; +/// +/// Create a transfer for downloading objects from a gcs bucket to the root directory of POSIX file system. +/// public class DownloadToPosixSample { - // Create a transfer from a GCS bucket to a POSIX file system public TransferJob DownloadToPosix( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs index b07b0bdb118..cfefc2751bf 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -17,9 +17,11 @@ using System; using Google.Cloud.StorageTransfer.V1; +/// +/// Create a transfer to transfer objects from root directory to the destination directory between POSIX file systems. +/// public class TransferBetweenPosixSample { - // Creates a transfer between POSIX file systems public TransferJob TransferBetweenPosix( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs index 60e6f29af2f..4f484dde1de 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs @@ -17,9 +17,11 @@ using System; using Google.Cloud.StorageTransfer.V1; +/// +/// Create a transfer to transfer objects from a POSIX file system to a gcs sink bucket. +/// public class TransferFromPosixSample { - // Create a transfer from a POSIX file system to a GCS sink bucket public TransferJob TransferFromPosix( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index f9bcfd443cb..282723133e3 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -18,10 +18,12 @@ using Google.Cloud.StorageTransfer.V1; using Google.Protobuf.WellKnownTypes; +/// +/// Creates a one-off transfer job that transfers objects from a standard gcs bucket that are more +/// than 30 days old to a nearline gcs bucket. +/// public class TransferToNearlineSample { - // Creates a one-off transfer job that transfers objects from a standard GCS bucket that are more - // than 30 days old to a Nearline GCS bucket public TransferJob TransferToNearline( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index da490aa4b96..27db28527a5 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -17,9 +17,11 @@ using System; using Google.Cloud.StorageTransfer.V1; +/// +/// Create a transfer to transfer objects from a POSIX file system to a gcs sink bucket using a manifest file. +/// public class TransferUsingManifestSample { - // Create a transfer from a POSIX file system to a GCS bucket using a manifest file public TransferJob TransferUsingManifest( // Your Google Cloud Project ID string projectId = "my-project-id", From 522b48a813f57d63f720dda30f689db420c023c9 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 19 Feb 2025 03:59:19 -0800 Subject: [PATCH 31/58] samples and test linter changes --- .../CheckLatestTransferOperationTest.cs | 2 +- .../CreateEventDrivenGcsTransferTest.cs | 2 +- .../api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs | 2 +- .../api/StorageTransfer.Samples.Tests/QuickstartTest.cs | 2 +- .../api/StorageTransfer.Samples.Tests/StorageFixture.cs | 2 +- .../StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs | 2 +- .../StorageTransfer.Samples.Tests/TransferFromPosixTest.cs | 2 +- .../StorageTransfer.Samples.Tests/TransferToNearlineTest.cs | 2 +- .../TransferUsingManifestTest.cs | 2 +- .../CheckLatestTransferOperationSample.cs | 2 +- .../CreateEventDrivenGcsTransferSample.cs | 3 +-- .../api/StorageTransfer.Samples/DownloadToPosixSample.cs | 4 +--- .../api/StorageTransfer.Samples/QuickstartSample.cs | 2 +- .../api/StorageTransfer.Samples/TransferBetweenPosixSample.cs | 4 +--- .../api/StorageTransfer.Samples/TransferFromPosixSample.cs | 4 +--- .../api/StorageTransfer.Samples/TransferToNearlineSample.cs | 2 +- .../StorageTransfer.Samples/TransferUsingManifestSample.cs | 2 +- 17 files changed, 17 insertions(+), 24 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs index a5cbd41b2aa..0b1c68cb130 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs @@ -59,4 +59,4 @@ private string CreateTransferJob() TransferJob response = _fixture.Sts.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); return response.Name; } -} +} \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs index 60833043c26..161c7d69063 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -107,4 +107,4 @@ public void Dispose() } } } -} +} \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index b808b2b0c2a..00c71e1d74a 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -76,4 +76,4 @@ public void Dispose() // Do nothing, we delete on a best effort basis. } } -} +} \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs index 33fa4dd687d..3a3687db9ad 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs @@ -61,4 +61,4 @@ public void Dispose() } } } -} +} \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index b942c5b21f6..fb767111962 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -117,4 +117,4 @@ public void Dispose() } } } -} +} \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index 4185a094b4f..3a387cd6c5a 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -81,4 +81,4 @@ public void Dispose() // Do nothing, we delete on a best effort basis. } } -} +} \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs index 0cfeab95a4a..2bbadffca52 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs @@ -59,4 +59,4 @@ public void Dispose() // Do nothing, we delete on a best effort basis. } } -} +} \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs index 6fe8e41abcb..49d8e64e813 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -62,4 +62,4 @@ public void Dispose() // Do nothing, we delete on a best effort basis. } } -} +} \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index b122e99f9c0..6df7c3f1336 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -71,4 +71,4 @@ public void Dispose() // Do nothing, we delete on a best effort basis. } } -} +} \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs index d3ab8445472..611d4751998 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -59,4 +59,4 @@ public TransferJob CheckLatestTransferOperation( } } } -// [END storagetransfer_get_latest_transfer_operation] +// [END storagetransfer_get_latest_transfer_operation] \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs index 25f0549a8d9..9eac88090f4 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs @@ -56,5 +56,4 @@ public TransferJob CreateEventDrivenGcsTransfer( return response; } } -// [END storagetransfer_create_event_driven_gcs_transfer] - +// [END storagetransfer_create_event_driven_gcs_transfer] \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index 0d6aef6867e..83172e16f99 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -66,6 +66,4 @@ public TransferJob DownloadToPosix( return response; } } -//[END storagetransfer_download_to_posix] - - +//[END storagetransfer_download_to_posix] \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs index 9726dfb777c..d963a11a4c7 100644 --- a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs @@ -52,4 +52,4 @@ public TransferJob Quickstart( return response; } } -// [END storagetransfer_quickstart] +// [END storagetransfer_quickstart] \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs index cfefc2751bf..202c8fd3b29 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -70,6 +70,4 @@ public TransferJob TransferBetweenPosix( return response; } } -//[END storagetransfer_transfer_posix_to_posix] - - +//[END storagetransfer_transfer_posix_to_posix] \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs index 4f484dde1de..9bfc347ee27 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs @@ -64,6 +64,4 @@ public TransferJob TransferFromPosix( return response; } } -// [END storagetransfer_transfer_from_posix] - - +// [END storagetransfer_transfer_from_posix] \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index 282723133e3..be0f2176778 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -65,4 +65,4 @@ public TransferJob TransferToNearline( return response; } } -// [END storagetransfer_transfer_to_nearline] +// [END storagetransfer_transfer_to_nearline] \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index 27db28527a5..454836d1b81 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -72,4 +72,4 @@ public TransferJob TransferUsingManifest( return response; } } -// [END storagetransfer_manifest_request] +// [END storagetransfer_manifest_request] \ No newline at end of file From 8642fbf93f75661c81f3bbfba9dda0649925732d Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 20 Feb 2025 05:38:57 -0800 Subject: [PATCH 32/58] Structural and linter changes in tests --- .../CheckLatestTransferOperationTest.cs | 9 +- .../CreateEventDrivenGcsTransferTest.cs | 143 +++++++++--------- .../DownloadToPosixTest.cs | 15 +- .../QuickstartTest.cs | 4 + .../StorageFixture.cs | 17 +-- .../TransferBetweenPosixTest.cs | 6 +- .../TransferFromPosixTest.cs | 4 +- .../TransferToNearlineTest.cs | 5 + .../TransferUsingManifestTest.cs | 10 +- 9 files changed, 119 insertions(+), 94 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs index 0b1c68cb130..9462c17b69f 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -using Google.Cloud.Storage.V1; + using Google.Cloud.StorageTransfer.V1; using Xunit; @@ -20,13 +20,17 @@ namespace StorageTransfer.Samples.Tests; [Collection(nameof(StorageFixture))] public class CheckLatestTransferOperationTest { - private readonly StorageFixture _fixture; private string _jobName; private readonly string _transferJobName; + public CheckLatestTransferOperationTest(StorageFixture fixture) { _fixture = fixture; + _fixture.BucketNameSource = _fixture.GenerateBucketName(); + _fixture.BucketNameSink = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSource); + _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); _transferJobName = CreateTransferJob(); } @@ -38,6 +42,7 @@ public void CheckLatestTransferOperation() Assert.Contains("transferJobs/", transferJob.Name); _jobName = transferJob.Name; } + private string CreateTransferJob() { // Initialize request argument(s) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs index 161c7d69063..14bc4f1b267 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -16,95 +16,98 @@ using System; using Google.Cloud.PubSub.V1; -using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using Xunit; -namespace StorageTransfer.Samples.Tests +namespace StorageTransfer.Samples.Tests; + +[Collection(nameof(StorageFixture))] +public class CreateEventDrivenGcsTransferTest : IDisposable { - [Collection(nameof(StorageFixture))] - public class CreateEventDrivenGcsTransferTest : IDisposable + private readonly StorageFixture _fixture; + private readonly string _pubSubId; + private string _transferJobName; + private string TopicId { get; } = "DotNetTopic" + Guid.NewGuid().ToString(); + private string SubscriptionId { get; } = "DotNetSubscription" + Guid.NewGuid().ToString(); + private SubscriberServiceApiClient SubscriberClient { get; } = SubscriberServiceApiClient.Create(); + private PublisherServiceApiClient PublisherClient { get; } = PublisherServiceApiClient.Create(); + + public CreateEventDrivenGcsTransferTest(StorageFixture fixture) { - private readonly StorageFixture _fixture; - private readonly string _pubSubId; - private string _transferJobName; - private string TopicId { get; } = "DotNetTopic" + Guid.NewGuid().ToString(); - private string SubscriptionId { get; } = "DotNetSubscription" + Guid.NewGuid().ToString(); - private SubscriberServiceApiClient SubscriberClient { get; } = SubscriberServiceApiClient.Create(); - private PublisherServiceApiClient PublisherClient { get; } = PublisherServiceApiClient.Create(); - public CreateEventDrivenGcsTransferTest(StorageFixture fixture) + _fixture = fixture; + _fixture.BucketNameSource = _fixture.GenerateBucketName(); + _fixture.BucketNameSink = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSource); + _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); + _pubSubId = $"projects/{_fixture.ProjectId}/subscriptions/{SubscriptionId}"; + string email = _fixture.Sts.GetGoogleServiceAccount(new GetGoogleServiceAccountRequest() { - _fixture = fixture; - _pubSubId = $"projects/{_fixture.ProjectId}/subscriptions/{SubscriptionId}"; - string email = _fixture.Sts.GetGoogleServiceAccount(new GetGoogleServiceAccountRequest() - { - ProjectId = _fixture.ProjectId - }).AccountEmail; + ProjectId = _fixture.ProjectId + }).AccountEmail; - string memberServiceAccount = "serviceAccount:" + email; - // Create subscription name - SubscriptionName subscriptionName = new SubscriptionName(_fixture.ProjectId, SubscriptionId); - // Create topic name - TopicName topicName = new TopicName(_fixture.ProjectId, TopicId); - // Create topic - PublisherClient.CreateTopic(topicName); - // Create subscription. - SubscriberClient.CreateSubscription(subscriptionName, topicName, pushConfig: null, ackDeadlineSeconds: 500); + string memberServiceAccount = "serviceAccount:" + email; + // Create subscription name + SubscriptionName subscriptionName = new SubscriptionName(_fixture.ProjectId, SubscriptionId); + // Create topic name + TopicName topicName = new TopicName(_fixture.ProjectId, TopicId); + // Create topic + PublisherClient.CreateTopic(topicName); + // Create subscription. + SubscriberClient.CreateSubscription(subscriptionName, topicName, pushConfig: null, ackDeadlineSeconds: 500); - var policyIamPolicyTopic = new Google.Cloud.Iam.V1.Policy(); + var policyIamPolicyTopic = new Google.Cloud.Iam.V1.Policy(); - policyIamPolicyTopic.AddRoleMember("roles/pubsub.publisher", memberServiceAccount); - - PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest - { - ResourceAsResourceName = topicName, - Policy = policyIamPolicyTopic - }); + policyIamPolicyTopic.AddRoleMember("roles/pubsub.publisher", memberServiceAccount); - var policyIamPolicySubscriber = new Google.Cloud.Iam.V1.Policy(); + PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest + { + ResourceAsResourceName = topicName, + Policy = policyIamPolicyTopic + }); - policyIamPolicySubscriber.AddRoleMember("roles/pubsub.subscriber", memberServiceAccount); + var policyIamPolicySubscriber = new Google.Cloud.Iam.V1.Policy(); - PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest - { - ResourceAsResourceName = subscriptionName, - Policy = policyIamPolicySubscriber - }); - } + policyIamPolicySubscriber.AddRoleMember("roles/pubsub.subscriber", memberServiceAccount); - [Fact] - public void CreateEventDrivenGcsTransfer() + PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest { - CreateEventDrivenGcsTransferSample createEventDrivenGcsTransferSample = new CreateEventDrivenGcsTransferSample(); - var transferJob = createEventDrivenGcsTransferSample.CreateEventDrivenGcsTransfer(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink, _pubSubId); - Assert.Contains("transferJobs/", transferJob.Name); - _transferJobName = transferJob.Name; - } + ResourceAsResourceName = subscriptionName, + Policy = policyIamPolicySubscriber + }); + } + + [Fact] + public void CreateEventDrivenGcsTransfer() + { + CreateEventDrivenGcsTransferSample createEventDrivenGcsTransferSample = new CreateEventDrivenGcsTransferSample(); + var transferJob = createEventDrivenGcsTransferSample.CreateEventDrivenGcsTransfer(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink, _pubSubId); + Assert.Contains("transferJobs/", transferJob.Name); + _transferJobName = transferJob.Name; + } - public void Dispose() + public void Dispose() + { + try { - try + _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest() { - _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest() + ProjectId = _fixture.ProjectId, + JobName = _transferJobName, + TransferJob = new TransferJob() { - ProjectId = _fixture.ProjectId, - JobName = _transferJobName, - TransferJob = new TransferJob() - { - Name = _transferJobName, - Status = TransferJob.Types.Status.Deleted - } - }); + Name = _transferJobName, + Status = TransferJob.Types.Status.Deleted + } + }); - TopicName topicName = TopicName.FromProjectTopic(_fixture.ProjectId, TopicId); - PublisherClient.DeleteTopic(topicName); - SubscriptionName subscriptionName = SubscriptionName.FromProjectSubscription(_fixture.ProjectId, SubscriptionId); - SubscriberClient.DeleteSubscription(subscriptionName); - } - catch (Exception) - { - // Do nothing, we delete on a best effort basis. - } + TopicName topicName = TopicName.FromProjectTopic(_fixture.ProjectId, TopicId); + PublisherClient.DeleteTopic(topicName); + SubscriptionName subscriptionName = SubscriptionName.FromProjectSubscription(_fixture.ProjectId, SubscriptionId); + SubscriberClient.DeleteSubscription(subscriptionName); + } + catch (Exception) + { + // Do nothing, we delete on a best effort basis. } } } \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index 00c71e1d74a..75c4f77918d 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -14,14 +14,12 @@ using System; using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using Xunit; namespace StorageTransfer.Samples.Tests; + [Collection(nameof(StorageFixture))] public class DownloadToPosixTest : IDisposable { @@ -29,12 +27,15 @@ public class DownloadToPosixTest : IDisposable private string _transferJobName; private readonly string _tempDirectory; private readonly string _gcsSourcePath; + private readonly string _bucketNamePosixSource; public DownloadToPosixTest(StorageFixture fixture) { _fixture = fixture; - _tempDirectory = fixture.GenerateTempFolderPath(); + _bucketNamePosixSource = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_bucketNamePosixSource); + _tempDirectory = _fixture.GenerateTempFolderPath(); _gcsSourcePath = $"{Guid.NewGuid()}/{Guid.NewGuid()}/"; - UploadObjectToPosixBucket(_fixture.BucketNamePosixSource); + UploadObjectToPosixBucket(_bucketNamePosixSource); } [Fact] @@ -42,11 +43,12 @@ public void DownloadToPosix() { DownloadToPosixSample downloadToPosixSample = new DownloadToPosixSample(); Directory.CreateDirectory(_tempDirectory); - var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId, _fixture.SinkAgentPoolName, _fixture.BucketNamePosixSource, _gcsSourcePath, _tempDirectory); + var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId, _fixture.SinkAgentPoolName, _bucketNamePosixSource, _gcsSourcePath, _tempDirectory); Assert.Contains("transferJobs/", transferJob.Name); Assert.True(Directory.Exists(_tempDirectory)); _transferJobName = transferJob.Name; } + private void UploadObjectToPosixBucket(string bucketName) { byte[] byteArray = System.Text.Encoding.UTF8.GetBytes($@"{Guid.NewGuid()}.jpeg"); @@ -70,6 +72,7 @@ public void Dispose() } }); Directory.Delete(_tempDirectory, true); + _fixture.Storage.DeleteBucket(_bucketNamePosixSource, new DeleteBucketOptions { DeleteObjects = true }); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs index 3a3687db9ad..f8f67687ef7 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs @@ -29,6 +29,10 @@ public class QuickstartTest : IDisposable public QuickstartTest(StorageFixture fixture) { _fixture = fixture; + _fixture.BucketNameSource = _fixture.GenerateBucketName(); + _fixture.BucketNameSink = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSource); + _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); } [Fact] diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index fb767111962..43aff5dbbf4 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -28,10 +28,8 @@ namespace StorageTransfer.Samples.Tests public class StorageFixture : IDisposable, ICollectionFixture { public string ProjectId { get; } - public string BucketNameSource { get; } = Guid.NewGuid().ToString(); - public string BucketNameSink { get; } = Guid.NewGuid().ToString(); - public string BucketNameManifestSource { get; } = Guid.NewGuid().ToString(); - public string BucketNamePosixSource { get; } = Guid.NewGuid().ToString(); + public string BucketNameSource { get; set; } + public string BucketNameSink { get; set; } public string SourceAgentPoolName { get; } public string SinkAgentPoolName { get; } public StorageClient Storage { get; } = StorageClient.Create(); @@ -46,13 +44,9 @@ public StorageFixture() { throw new Exception("You need to set the Environment variable 'GOOGLE_PROJECT_ID' with your Google Cloud Project's project id."); } - - CreateBucketAndGrantStsPermissions(BucketNameSink); - CreateBucketAndGrantStsPermissions(BucketNameSource); - CreateBucketAndGrantStsPermissions(BucketNameManifestSource); - CreateBucketAndGrantStsPermissions(BucketNamePosixSource); } - private void CreateBucketAndGrantStsPermissions(string bucketName) + + internal void CreateBucketAndGrantStsPermissions(string bucketName) { var bucket = Storage.CreateBucket(ProjectId, new Bucket { @@ -95,8 +89,11 @@ private void CreateBucketAndGrantStsPermissions(string bucketName) policy.Bindings.Add(bucketWriterBinding); Storage.SetBucketIamPolicy(bucketName, policy); } + + internal string GenerateBucketName() => Guid.NewGuid().ToString(); internal string GetCurrentUserTempFolderPath() => System.IO.Path.GetTempPath(); internal string GenerateTempFolderPath() => Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + public void Dispose() { try diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index 3a387cd6c5a..be9980c3927 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -14,13 +14,11 @@ using System; using System.IO; -using System.Linq; -using System.Xml.Linq; -using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using Xunit; namespace StorageTransfer.Samples.Tests; + [Collection(nameof(StorageFixture))] public class TransferBetweenPosixTest : IDisposable { @@ -31,6 +29,8 @@ public class TransferBetweenPosixTest : IDisposable public TransferBetweenPosixTest(StorageFixture fixture) { _fixture = fixture; + _fixture.BucketNameSource = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSource); _tempDirectory = fixture.GenerateTempFolderPath(); _tempDestinationDirectory = fixture.GenerateTempFolderPath(); } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs index 2bbadffca52..49e23e5a7dc 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs @@ -13,11 +13,11 @@ // limitations under the License. using System; -using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using Xunit; namespace StorageTransfer.Samples.Tests; + [Collection(nameof(StorageFixture))] public class TransferFromPosixTest : IDisposable { @@ -27,6 +27,8 @@ public class TransferFromPosixTest : IDisposable public TransferFromPosixTest(StorageFixture fixture) { _fixture = fixture; + _fixture.BucketNameSink = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); _rootDirectory = fixture.GetCurrentUserTempFolderPath(); } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs index 49d8e64e813..dd2617e5000 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -18,6 +18,7 @@ using Xunit; namespace StorageTransfer.Samples.Tests; + [Collection(nameof(StorageFixture))] public class TransferToNearlineTest : IDisposable { @@ -26,6 +27,10 @@ public class TransferToNearlineTest : IDisposable public TransferToNearlineTest(StorageFixture fixture) { _fixture = fixture; + _fixture.BucketNameSource = _fixture.GenerateBucketName(); + _fixture.BucketNameSink = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSource); + _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); } [Fact] diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index 6df7c3f1336..74a3b3ccb18 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -27,19 +27,24 @@ public class TransferUsingManifestTest : IDisposable private string _transferJobName; private readonly string _rootDirectory; private readonly string _manifestObjectName; + private readonly string _bucketNameManifestSource; public TransferUsingManifestTest(StorageFixture fixture) { _fixture = fixture; + _bucketNameManifestSource = _fixture.GenerateBucketName(); + _fixture.BucketNameSink = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_bucketNameManifestSource); + _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); _rootDirectory = fixture.GetCurrentUserTempFolderPath(); _manifestObjectName = $@"{Guid.NewGuid()}.csv"; - UploadObjectToManifestBucket(_fixture.BucketNameManifestSource); + UploadObjectToManifestBucket(_bucketNameManifestSource); } [Fact] public void TransferUsingManifest() { TransferUsingManifestSample transferUsingManifestSample = new TransferUsingManifestSample(); - var transferJob = transferUsingManifestSample.TransferUsingManifest(_fixture.ProjectId, _fixture.SourceAgentPoolName, _rootDirectory, _fixture.BucketNameManifestSource, _fixture.BucketNameSink, _manifestObjectName); + var transferJob = transferUsingManifestSample.TransferUsingManifest(_fixture.ProjectId, _fixture.SourceAgentPoolName, _rootDirectory, _bucketNameManifestSource, _fixture.BucketNameSink, _manifestObjectName); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } @@ -65,6 +70,7 @@ public void Dispose() Status = TransferJob.Types.Status.Deleted } }); + _fixture.Storage.DeleteBucket(_bucketNameManifestSource, new DeleteBucketOptions { DeleteObjects = true }); } catch (Exception) { From 72872613914e196cd5053431d4330de4207da7e1 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Fri, 21 Feb 2025 02:56:11 -0800 Subject: [PATCH 33/58] linter changes in tests --- .../api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs | 1 + .../StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs | 1 + .../api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs | 1 + .../api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs | 1 + .../StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs | 1 + 5 files changed, 5 insertions(+) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index 75c4f77918d..0313f5f3ebe 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -28,6 +28,7 @@ public class DownloadToPosixTest : IDisposable private readonly string _tempDirectory; private readonly string _gcsSourcePath; private readonly string _bucketNamePosixSource; + public DownloadToPosixTest(StorageFixture fixture) { _fixture = fixture; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index be9980c3927..7aad45caf56 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -26,6 +26,7 @@ public class TransferBetweenPosixTest : IDisposable private string _transferJobName; private readonly string _tempDirectory; private readonly string _tempDestinationDirectory; + public TransferBetweenPosixTest(StorageFixture fixture) { _fixture = fixture; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs index 49e23e5a7dc..e8f416e80da 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs @@ -24,6 +24,7 @@ public class TransferFromPosixTest : IDisposable private readonly StorageFixture _fixture; private string _transferJobName; private readonly string _rootDirectory; + public TransferFromPosixTest(StorageFixture fixture) { _fixture = fixture; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs index dd2617e5000..d9bfbefb88b 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -24,6 +24,7 @@ public class TransferToNearlineTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; + public TransferToNearlineTest(StorageFixture fixture) { _fixture = fixture; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index 74a3b3ccb18..bef0970d34a 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -28,6 +28,7 @@ public class TransferUsingManifestTest : IDisposable private readonly string _rootDirectory; private readonly string _manifestObjectName; private readonly string _bucketNameManifestSource; + public TransferUsingManifestTest(StorageFixture fixture) { _fixture = fixture; From 904c9890789cbdf4b2870a47a2b49a45875a2d5a Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Fri, 21 Feb 2025 03:35:54 -0800 Subject: [PATCH 34/58] reuse of existing fixture's storage client in transfer to nearline test --- .../StorageTransfer.Samples.Tests/TransferToNearlineTest.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs index d9bfbefb88b..c782bf237f5 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -38,11 +38,11 @@ public TransferToNearlineTest(StorageFixture fixture) public void TestTransferToNearline() { TransferToNearlineSample transferToNearlineSample = new TransferToNearlineSample(); - var storage = StorageClient.Create(); + var storage = _fixture.Storage; var bucket = storage.GetBucket(_fixture.BucketNameSink); string storageClass = StorageClasses.Nearline; bucket.StorageClass = storageClass; - bucket = storage.UpdateBucket(bucket); + storage.UpdateBucket(bucket); var transferJob = transferToNearlineSample.TransferToNearline(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; From 1b0d4faa29e9ee6edf23ae8e8173246b2a518ed8 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 26 Feb 2025 05:11:30 -0800 Subject: [PATCH 35/58] linter changes as per .editorconfig file at the root of repository --- .../CheckLatestTransferOperationTest.cs | 2 +- .../CreateEventDrivenGcsTransferTest.cs | 4 ++-- .../StorageTransfer.Samples.Tests/DownloadToPosixTest.cs | 6 +++--- .../api/StorageTransfer.Samples.Tests/QuickstartTest.cs | 4 ++-- .../api/StorageTransfer.Samples.Tests/StorageFixture.cs | 8 ++++---- .../TransferBetweenPosixTest.cs | 4 ++-- .../TransferFromPosixTest.cs | 4 ++-- .../TransferToNearlineTest.cs | 4 ++-- .../TransferUsingManifestTest.cs | 6 +++--- .../CheckLatestTransferOperationSample.cs | 4 ++-- .../CreateEventDrivenGcsTransferSample.cs | 4 ++-- .../api/StorageTransfer.Samples/DownloadToPosixSample.cs | 4 ++-- .../api/StorageTransfer.Samples/QuickstartSample.cs | 4 ++-- .../StorageTransfer.Samples/TransferBetweenPosixSample.cs | 4 ++-- .../StorageTransfer.Samples/TransferFromPosixSample.cs | 4 ++-- .../StorageTransfer.Samples/TransferToNearlineSample.cs | 4 ++-- .../TransferUsingManifestSample.cs | 4 ++-- 17 files changed, 37 insertions(+), 37 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs index 9462c17b69f..723f566098d 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs @@ -64,4 +64,4 @@ private string CreateTransferJob() TransferJob response = _fixture.Sts.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); return response.Name; } -} \ No newline at end of file +} diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs index 14bc4f1b267..cba7c2b99fd 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -14,9 +14,9 @@ * limitations under the License. */ -using System; using Google.Cloud.PubSub.V1; using Google.Cloud.StorageTransfer.V1; +using System; using Xunit; namespace StorageTransfer.Samples.Tests; @@ -110,4 +110,4 @@ public void Dispose() // Do nothing, we delete on a best effort basis. } } -} \ No newline at end of file +} diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index 0313f5f3ebe..ac9116f3abc 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; -using System.IO; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; +using System; +using System.IO; using Xunit; namespace StorageTransfer.Samples.Tests; @@ -80,4 +80,4 @@ public void Dispose() // Do nothing, we delete on a best effort basis. } } -} \ No newline at end of file +} diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs index f8f67687ef7..9efbb3334a1 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs @@ -14,8 +14,8 @@ * limitations under the License. */ -using System; using Google.Cloud.StorageTransfer.V1; +using System; using Xunit; namespace StorageTransfer.Samples.Tests @@ -65,4 +65,4 @@ public void Dispose() } } } -} \ No newline at end of file +} diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index 43aff5dbbf4..fbd53117768 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -14,12 +14,12 @@ * limitations under the License. */ -using System; -using System.Collections.Generic; -using System.IO; using Google.Apis.Storage.v1.Data; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; +using System; +using System.Collections.Generic; +using System.IO; using Xunit; namespace StorageTransfer.Samples.Tests @@ -114,4 +114,4 @@ public void Dispose() } } } -} \ No newline at end of file +} diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index 7aad45caf56..f267930f566 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +using Google.Cloud.StorageTransfer.V1; using System; using System.IO; -using Google.Cloud.StorageTransfer.V1; using Xunit; namespace StorageTransfer.Samples.Tests; @@ -82,4 +82,4 @@ public void Dispose() // Do nothing, we delete on a best effort basis. } } -} \ No newline at end of file +} diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs index e8f416e80da..88f763d25fc 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; using Google.Cloud.StorageTransfer.V1; +using System; using Xunit; namespace StorageTransfer.Samples.Tests; @@ -62,4 +62,4 @@ public void Dispose() // Do nothing, we delete on a best effort basis. } } -} \ No newline at end of file +} diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs index c782bf237f5..eacce91d255 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; +using System; using Xunit; namespace StorageTransfer.Samples.Tests; @@ -68,4 +68,4 @@ public void Dispose() // Do nothing, we delete on a best effort basis. } } -} \ No newline at end of file +} diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index bef0970d34a..58711707643 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -using System; -using System.IO; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; +using System; +using System.IO; using Xunit; namespace StorageTransfer.Samples.Tests; @@ -78,4 +78,4 @@ public void Dispose() // Do nothing, we delete on a best effort basis. } } -} \ No newline at end of file +} diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs index 611d4751998..4005ef76341 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -14,8 +14,8 @@ // [START storagetransfer_get_latest_transfer_operation] -using System; using Google.Cloud.StorageTransfer.V1; +using System; /// /// Checks the latest transfer operation for a given transfer job. @@ -59,4 +59,4 @@ public TransferJob CheckLatestTransferOperation( } } } -// [END storagetransfer_get_latest_transfer_operation] \ No newline at end of file +// [END storagetransfer_get_latest_transfer_operation] diff --git a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs index 9eac88090f4..49465a395f3 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs @@ -16,8 +16,8 @@ // [START storagetransfer_create_event_driven_gcs_transfer] -using System; using Google.Cloud.StorageTransfer.V1; +using System; /// /// Create an event driven gcs data transfer from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. @@ -56,4 +56,4 @@ public TransferJob CreateEventDrivenGcsTransfer( return response; } } -// [END storagetransfer_create_event_driven_gcs_transfer] \ No newline at end of file +// [END storagetransfer_create_event_driven_gcs_transfer] diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index 83172e16f99..cd92b85e2c3 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -14,8 +14,8 @@ // [START storagetransfer_download_to_posix] -using System; using Google.Cloud.StorageTransfer.V1; +using System; /// /// Create a transfer for downloading objects from a gcs bucket to the root directory of POSIX file system. @@ -66,4 +66,4 @@ public TransferJob DownloadToPosix( return response; } } -//[END storagetransfer_download_to_posix] \ No newline at end of file +//[END storagetransfer_download_to_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs index d963a11a4c7..fe08f6082c2 100644 --- a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs @@ -16,8 +16,8 @@ // [START storagetransfer_quickstart] -using System; using Google.Cloud.StorageTransfer.V1; +using System; public class QuickstartSample { @@ -52,4 +52,4 @@ public TransferJob Quickstart( return response; } } -// [END storagetransfer_quickstart] \ No newline at end of file +// [END storagetransfer_quickstart] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs index 202c8fd3b29..5d8abed0ce5 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -14,8 +14,8 @@ // [START storagetransfer_transfer_posix_to_posix] -using System; using Google.Cloud.StorageTransfer.V1; +using System; /// /// Create a transfer to transfer objects from root directory to the destination directory between POSIX file systems. @@ -70,4 +70,4 @@ public TransferJob TransferBetweenPosix( return response; } } -//[END storagetransfer_transfer_posix_to_posix] \ No newline at end of file +//[END storagetransfer_transfer_posix_to_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs index 9bfc347ee27..bc82f8f74dc 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs @@ -14,8 +14,8 @@ // [START storagetransfer_transfer_from_posix] -using System; using Google.Cloud.StorageTransfer.V1; +using System; /// /// Create a transfer to transfer objects from a POSIX file system to a gcs sink bucket. @@ -64,4 +64,4 @@ public TransferJob TransferFromPosix( return response; } } -// [END storagetransfer_transfer_from_posix] \ No newline at end of file +// [END storagetransfer_transfer_from_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index be0f2176778..98dcfb0406d 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -14,9 +14,9 @@ // [START storagetransfer_transfer_to_nearline] -using System; using Google.Cloud.StorageTransfer.V1; using Google.Protobuf.WellKnownTypes; +using System; /// /// Creates a one-off transfer job that transfers objects from a standard gcs bucket that are more @@ -65,4 +65,4 @@ public TransferJob TransferToNearline( return response; } } -// [END storagetransfer_transfer_to_nearline] \ No newline at end of file +// [END storagetransfer_transfer_to_nearline] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index 454836d1b81..c92860b2174 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -14,8 +14,8 @@ // [START storagetransfer_manifest_request] -using System; using Google.Cloud.StorageTransfer.V1; +using System; /// /// Create a transfer to transfer objects from a POSIX file system to a gcs sink bucket using a manifest file. @@ -72,4 +72,4 @@ public TransferJob TransferUsingManifest( return response; } } -// [END storagetransfer_manifest_request] \ No newline at end of file +// [END storagetransfer_manifest_request] From dda9b52823e268e7dd39707219e393bcd90dfed4 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 12 Mar 2025 00:49:13 -0700 Subject: [PATCH 36/58] exception handling removed from CheckLatestTransferOperation --- .../CheckLatestTransferOperationTest.cs | 24 +++++++++++++- .../CheckLatestTransferOperationSample.cs | 33 +++++++------------ 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs index 723f566098d..f9107e38d98 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs @@ -13,12 +13,13 @@ // limitations under the License. using Google.Cloud.StorageTransfer.V1; +using System; using Xunit; namespace StorageTransfer.Samples.Tests; [Collection(nameof(StorageFixture))] -public class CheckLatestTransferOperationTest +public class CheckLatestTransferOperationTest : IDisposable { private readonly StorageFixture _fixture; private string _jobName; @@ -64,4 +65,25 @@ private string CreateTransferJob() TransferJob response = _fixture.Sts.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); return response.Name; } + + public void Dispose() + { + try + { + _fixture.Sts.UpdateTransferJob(new UpdateTransferJobRequest() + { + ProjectId = _fixture.ProjectId, + JobName = _transferJobName, + TransferJob = new TransferJob() + { + Name = _transferJobName, + Status = TransferJob.Types.Status.Deleted + } + }); + } + catch (Exception) + { + // Do nothing, we delete on a best effort basis. + } + } } diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs index 4005ef76341..f0f188b04e2 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -28,35 +28,24 @@ public TransferJob CheckLatestTransferOperation( // The name of the job to check string jobName = "transferJobs/1234567890") { - if (string.IsNullOrEmpty(jobName)) - { - throw new Exception("JobName can not be null or empty"); - } - // Create a Transfer Service client StorageTransferServiceClient storageTransfer = StorageTransferServiceClient.Create(); GetTransferJobRequest getTransferJobRequest = new GetTransferJobRequest { ProjectId = projectId, JobName = jobName }; - try - { - // Get Transfer job - TransferJob transferJob = storageTransfer.GetTransferJob(getTransferJobRequest); - // Get Latest operation name from transfer job - string latestOperationName = transferJob.LatestOperationName; - if (!string.IsNullOrEmpty(latestOperationName)) - { - Console.WriteLine("The latest operation for transfer job " + jobName + " is: " + latestOperationName + ""); - } - else - { - Console.WriteLine("Transfer job " + jobName + " hasn't run yet, try again after the job has started running."); - } - return transferJob; + // Get Transfer job + TransferJob transferJob = storageTransfer.GetTransferJob(getTransferJobRequest); + // Get Latest operation name from transfer job + string latestOperationName = transferJob.LatestOperationName; + + if (!string.IsNullOrEmpty(latestOperationName)) + { + Console.WriteLine("The latest operation for transfer job " + jobName + " is: " + latestOperationName + ""); } - catch (Exception) + else { - throw new Exception("Failed to get transfer job " + jobName + ""); + Console.WriteLine("Transfer job " + jobName + " hasn't run yet, try again after the job has started running."); } + return transferJob; } } // [END storagetransfer_get_latest_transfer_operation] From f1b05b37942aa131ccff60fdd9084adfedd39260 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 27 Mar 2025 03:51:36 -0700 Subject: [PATCH 37/58] Comments added in tests and sample description modified --- storagetransfer/api/.editorconfig | 281 ++++++++++++++++++ .../CheckLatestTransferOperationTest.cs | 2 + .../CreateEventDrivenGcsTransferTest.cs | 1 + .../DownloadToPosixTest.cs | 2 + .../QuickstartTest.cs | 1 + .../TransferBetweenPosixTest.cs | 1 + .../TransferFromPosixTest.cs | 1 + .../TransferToNearlineTest.cs | 1 + .../TransferUsingManifestTest.cs | 2 + .../CheckLatestTransferOperationSample.cs | 8 +- .../CreateEventDrivenGcsTransferSample.cs | 10 +- .../DownloadToPosixSample.cs | 11 +- .../QuickstartSample.cs | 6 + .../TransferBetweenPosixSample.cs | 18 +- .../TransferFromPosixSample.cs | 10 +- .../TransferToNearlineSample.cs | 11 +- .../TransferUsingManifestSample.cs | 13 +- testutil/TestBucket.cs | 4 +- testutil/TestUtil.cs | 26 +- 19 files changed, 369 insertions(+), 40 deletions(-) create mode 100644 storagetransfer/api/.editorconfig diff --git a/storagetransfer/api/.editorconfig b/storagetransfer/api/.editorconfig new file mode 100644 index 00000000000..0ec4d4be40e --- /dev/null +++ b/storagetransfer/api/.editorconfig @@ -0,0 +1,281 @@ +# To learn more about .editorconfig see https://aka.ms/editorconfigdocs +# top-most EditorConfig file +root = true +############################### +# Core EditorConfig Options # +############################### +# All files +[*] +insert_final_newline = true +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true + +# Generated code +[*{_AssemblyInfo.cs,.g.cs}] +generated_code = true + +# XML project files +[*.{csproj}] +indent_size = 2 + +# Code files + +############################### +# .NET Coding Conventions # +############################### +[*.{cs}] + +# Organize usings +dotnet_sort_system_directives_first = false + +# this. preferences +dotnet_style_qualification_for_field = false:suggestion +dotnet_style_qualification_for_property = false:suggestion +dotnet_style_qualification_for_method = false:suggestion +dotnet_style_qualification_for_event = false:suggestion + +# Language keywords vs BCL types preferences +dotnet_style_predefined_type_for_locals_parameters_members = true:suggestion +dotnet_style_predefined_type_for_member_access = true:suggestion + +# Parentheses preferences +dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_other_operators = never_if_unnecessary:silent + +# Modifier preferences +csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:warning +dotnet_style_require_accessibility_modifiers = for_non_interface_members:warning +dotnet_style_readonly_field = true:warning + +# Expression-level preferences +dotnet_style_object_initializer = true:suggestion +dotnet_style_collection_initializer = true:suggestion +dotnet_style_explicit_tuple_names = true:suggestion +dotnet_style_null_propagation = true:suggestion +dotnet_style_coalesce_expression = true:suggestion +dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion +dotnet_style_prefer_inferred_tuple_names = true:suggestion +dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion +dotnet_style_prefer_auto_properties = true:suggestion +dotnet_style_prefer_conditional_expression_over_assignment = true:silent +dotnet_style_prefer_conditional_expression_over_return = true:silent +csharp_prefer_simple_default_expression = true:suggestion + +############################### +# Naming Conventions # +############################### +# Style Definitions +dotnet_naming_style.pascal_case_style.capitalization = pascal_case + +# Use PascalCase for constant fields +dotnet_naming_rule.constant_fields_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.constant_fields_should_be_pascal_case.symbols = constant_fields +dotnet_naming_rule.constant_fields_should_be_pascal_case.style = pascal_case_style +dotnet_naming_symbols.constant_fields.applicable_kinds = field +dotnet_naming_symbols.constant_fields.applicable_accessibilities = * +dotnet_naming_symbols.constant_fields.required_modifiers = const +dotnet_naming_style.pascal_case_style.capitalization = pascal_case + +# static fields should have s_ prefix +dotnet_naming_rule.static_fields_should_have_prefix.severity = suggestion +dotnet_naming_rule.static_fields_should_have_prefix.symbols = static_fields +dotnet_naming_rule.static_fields_should_have_prefix.style = static_prefix_style +dotnet_naming_symbols.static_fields.applicable_kinds = field +dotnet_naming_symbols.static_fields.required_modifiers = static +dotnet_naming_symbols.static_fields.applicable_accessibilities = private, internal, private_protected +dotnet_naming_style.static_prefix_style.required_prefix = s_ +dotnet_naming_style.static_prefix_style.capitalization = camel_case + +# internal and private fields should be _camelCase +dotnet_naming_rule.camel_case_for_private_internal_fields.severity = suggestion +dotnet_naming_rule.camel_case_for_private_internal_fields.symbols = private_internal_fields +dotnet_naming_rule.camel_case_for_private_internal_fields.style = camel_case_underscore_style +dotnet_naming_symbols.private_internal_fields.applicable_kinds = field +dotnet_naming_symbols.private_internal_fields.applicable_accessibilities = private, internal +dotnet_naming_style.camel_case_underscore_style.required_prefix = _ +dotnet_naming_style.camel_case_underscore_style.capitalization = camel_case + +# Locals and parameters are camelCase +dotnet_naming_rule.locals_should_be_camel_case.severity = suggestion +dotnet_naming_rule.locals_should_be_camel_case.symbols = locals_and_parameters +dotnet_naming_rule.locals_should_be_camel_case.style = camel_case_style +dotnet_naming_symbols.locals_and_parameters.applicable_kinds = parameter, local +dotnet_naming_style.camel_case_style.capitalization = camel_case + +# Public property and methods should be PascalCase +dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.symbols = public_methods_and_properties +dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.style = pascal_case +dotnet_naming_symbols.public_methods_and_properties.applicable_kinds = method, property +dotnet_naming_symbols.public_methods_and_properties.applicable_accessibilities = public +dotnet_naming_style.pascal_case.capitalization = pascal_case + +# Private and internal methods should be PascalCase +dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.symbols = private_and_internal_methods +dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.style = pascal_case +dotnet_naming_symbols.private_and_internal_methods.applicable_kinds = method +dotnet_naming_symbols.private_and_internal_methods.applicable_accessibilities = private, internal +############################### +# C# Coding Conventions # +############################### +[*.cs] +# var preferences +csharp_style_var_for_built_in_types = true:silent +csharp_style_var_when_type_is_apparent = true:silent +csharp_style_var_elsewhere = true:silent +dotnet_style_predefined_type_for_locals_parameters_members = true:silent +dotnet_style_predefined_type_for_member_access = true:silent + +csharp_using_directive_placement = outside_namespace:warning +csharp_prefer_braces = true:suggestion +csharp_preserve_single_line_blocks = true:none +csharp_preserve_single_line_statements = false:none +csharp_prefer_static_local_function = true:suggestion +csharp_prefer_simple_using_statement = true:suggestion +csharp_style_prefer_switch_expression = true:suggestion + +# Expression-bodied members +csharp_style_expression_bodied_methods = true:suggestion +csharp_style_expression_bodied_constructors = true:suggestion +csharp_style_expression_bodied_operators = true:suggestion +csharp_style_expression_bodied_properties = true:suggestion +csharp_style_expression_bodied_indexers = true:suggestion +csharp_style_expression_bodied_accessors = true:suggestion +csharp_style_expression_bodied_lambdas = true:suggestion +csharp_style_expression_bodied_local_functions = true:suggestion + +# Pattern matching preferences +csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion +csharp_style_pattern_matching_over_as_with_null_check = true:suggestion +csharp_style_inlined_variable_declaration = true:suggestion + +# Null-checking preferences +csharp_style_throw_expression = true:suggestion +csharp_style_conditional_delegate_call = true:suggestion + +# Other features +csharp_style_prefer_index_operator = true:suggestion +csharp_style_prefer_range_operator = true:suggestion +csharp_style_pattern_local_over_anonymous_function = false:none + +############################### +# C# Formatting Rules # +############################### +indent_size = 4 +insert_final_newline = true +charset = utf-8 + +# New line preferences +csharp_new_line_before_open_brace = all +csharp_new_line_before_else = true +csharp_new_line_before_catch = true +csharp_new_line_before_finally = true +csharp_new_line_before_members_in_object_initializers = true +csharp_new_line_before_members_in_anonymous_types = true +csharp_new_line_between_query_expression_clauses = true + +# Indentation preferences +csharp_indent_block_contents = true +csharp_indent_braces = false +csharp_indent_case_contents = true +csharp_indent_case_contents_when_block = true +csharp_indent_switch_labels = true +csharp_indent_labels = one_less_than_current + +# Space preferences +csharp_space_after_cast = true +csharp_space_after_colon_in_inheritance_clause = true +csharp_space_after_comma = true +csharp_space_after_dot = false +csharp_space_after_keywords_in_control_flow_statements = true +csharp_space_after_semicolon_in_for_statement = true +csharp_space_between_method_call_parameter_list_parentheses = false +csharp_space_between_method_declaration_parameter_list_parentheses = false +csharp_space_between_parentheses = false +csharp_space_before_comma = false +csharp_space_before_dot = false +csharp_space_before_open_square_brackets = false +csharp_space_before_semicolon_in_for_statement = false +csharp_space_between_empty_square_brackets = false +csharp_space_around_declaration_statements = do_not_ignore +csharp_space_before_colon_in_inheritance_clause = true +csharp_space_after_colon_in_inheritance_clause = true +csharp_space_around_binary_operators = before_and_after +csharp_space_between_method_declaration_empty_parameter_list_parentheses = false +csharp_space_between_method_call_name_and_opening_parenthesis = false +csharp_space_between_method_call_empty_parameter_list_parentheses = false +csharp_space_between_method_declaration_name_and_open_parenthesis = false +csharp_space_between_square_brackets = false + +# Wrapping preferences +csharp_preserve_single_line_statements = true +csharp_preserve_single_line_blocks = true + +# License header +file_header_template = Copyright 2024 Google LLC\n\nLicensed under the Apache License, Version 2.0 (the "License").\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at \n\nhttps://www.apache.org/licenses/LICENSE-2.0 \n\nUnless required by applicable law or agreed to in writing, software \ndistributed under the License is distributed on an "AS IS" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and \nlimitations under the License. +csharp_style_namespace_declarations = file_scoped:suggestion +csharp_style_prefer_null_check_over_type_check = true:suggestion +csharp_style_prefer_local_over_anonymous_function = true:suggestion +csharp_style_prefer_tuple_swap = true:suggestion +csharp_style_implicit_object_creation_when_type_is_apparent = true:suggestion +csharp_style_unused_value_assignment_preference = discard_variable:suggestion +csharp_style_unused_value_expression_statement_preference = discard_variable:suggestion +csharp_style_prefer_pattern_matching = true:suggestion +csharp_style_prefer_not_pattern = true:suggestion +csharp_style_prefer_extended_property_pattern = true:suggestion +csharp_style_prefer_parameter_null_checking = true:suggestion +csharp_style_prefer_method_group_conversion = true:suggestion + +# Change the severity to `error` for the rules that must not be violated, to fail the build. + +# IDE0005: Remove unnecessary using directives. +dotnet_diagnostic.IDE0005.severity = suggestion + +# IDE0011: Add braces +dotnet_diagnostic.IDE0011.severity = suggestion + +# IDE0017: Object initialization can be simplified. +dotnet_diagnostic.IDE0017.severity = suggestion + +# IDE0019: Use pattern matching. +dotnet_diagnostic.IDE0019.severity = suggestion + +# IDE0021: Use expression body for constructor. +dotnet_diagnostic.IDE0021.severity = suggestion + +# IDE0028: Collection initialization can be simplified. +dotnet_diagnostic.IDE0028.severity = suggestion + +# IDE0035: Remove unreachable code. +dotnet_diagnostic.IDE0035.severity = suggestion + +# IDE0040: Accessibility modifiers required. +dotnet_diagnostic.IDE0040.severity = suggestion + +# IDE0052: Remove unread private member +dotnet_diagnostic.IDE0052.severity = suggestion + +# IDE0053: Use expression body for lambda expression. +dotnet_diagnostic.IDE0053.severity = suggestion + +# IDE0055: All C# and .NET formatting rules. +dotnet_diagnostic.IDE0055.severity = suggestion + +# IDE0060: Remove unused parameter +dotnet_diagnostic.IDE0060.severity = suggestion + +# IDE0063: Using statement can be simplified +dotnet_diagnostic.IDE0063.severity = suggestion + +# IDE0065: Using statement can be simplified +dotnet_diagnostic.IDE0065.severity = suggestion + +# IDE0071: Interpolation can be simplified. +dotnet_diagnostic.IDE0071.severity = suggestion + +# IDE0075: Simplify conditional expression. +dotnet_diagnostic.IDE0075.severity = suggestion \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs index f9107e38d98..f798dc5b882 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs @@ -35,6 +35,7 @@ public CheckLatestTransferOperationTest(StorageFixture fixture) _transferJobName = CreateTransferJob(); } + // Checks the latest transfer operation for a created transfer job. [Fact] public void CheckLatestTransferOperation() { @@ -44,6 +45,7 @@ public void CheckLatestTransferOperation() _jobName = transferJob.Name; } + // Creates one-time transfer job from a Google cloud storage bucket to another bucket. private string CreateTransferJob() { // Initialize request argument(s) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs index cba7c2b99fd..54a85805ea3 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -76,6 +76,7 @@ public CreateEventDrivenGcsTransferTest(StorageFixture fixture) }); } + // Transfers object from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. [Fact] public void CreateEventDrivenGcsTransfer() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index ac9116f3abc..bc3c3247fdf 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -39,6 +39,7 @@ public DownloadToPosixTest(StorageFixture fixture) UploadObjectToPosixBucket(_bucketNamePosixSource); } + // Downloads an object from a GCS bucket to a POSIX file system. [Fact] public void DownloadToPosix() { @@ -50,6 +51,7 @@ public void DownloadToPosix() _transferJobName = transferJob.Name; } + // Uploads an object to the POSIX bucket. private void UploadObjectToPosixBucket(string bucketName) { byte[] byteArray = System.Text.Encoding.UTF8.GetBytes($@"{Guid.NewGuid()}.jpeg"); diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs index 9efbb3334a1..68f5868e5ae 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs @@ -35,6 +35,7 @@ public QuickstartTest(StorageFixture fixture) _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); } + // Creates a one-time transfer job from a Google cloud storage bucket to another bucket [Fact] public void TestQuickstart() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index f267930f566..1b5d8045cfe 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -36,6 +36,7 @@ public TransferBetweenPosixTest(StorageFixture fixture) _tempDestinationDirectory = fixture.GenerateTempFolderPath(); } + // Transfers objects from a POSIX file system to another POSIX file system. [Fact] public void TransferBetweenPosix() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs index 88f763d25fc..b81eeb7689b 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs @@ -33,6 +33,7 @@ public TransferFromPosixTest(StorageFixture fixture) _rootDirectory = fixture.GetCurrentUserTempFolderPath(); } + // Transfers objects from a POSIX file system to a Cloud Storage bucket. [Fact] public void TransferFromPosix() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs index eacce91d255..d0eaa9a6f00 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -34,6 +34,7 @@ public TransferToNearlineTest(StorageFixture fixture) _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); } + // Transfers objects from a Google Cloud Storage bucket to another bucket with Nearline storage class. [Fact] public void TestTransferToNearline() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index 58711707643..dde52fc6548 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -41,6 +41,7 @@ public TransferUsingManifestTest(StorageFixture fixture) UploadObjectToManifestBucket(_bucketNameManifestSource); } + // Transfers objects from a POSIX file system to a sink bucket using a manifest file. [Fact] public void TransferUsingManifest() { @@ -50,6 +51,7 @@ public void TransferUsingManifest() _transferJobName = transferJob.Name; } + // Uploads an object to the manifest bucket. private void UploadObjectToManifestBucket(string bucketName) { byte[] byteArray = System.Text.Encoding.UTF8.GetBytes($@"{Guid.NewGuid()}.jpeg"); diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs index f0f188b04e2..04df26cec05 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -17,11 +17,13 @@ using Google.Cloud.StorageTransfer.V1; using System; -/// -/// Checks the latest transfer operation for a given transfer job. -/// public class CheckLatestTransferOperationSample { + /// + /// Sample that checks the latest transfer operation for a given transfer job. + /// + /// Your Google Cloud Project ID. + /// The name of the job to check. public TransferJob CheckLatestTransferOperation( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs index 49465a395f3..3619408aca0 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs @@ -19,11 +19,15 @@ using Google.Cloud.StorageTransfer.V1; using System; -/// -/// Create an event driven gcs data transfer from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. -/// public class CreateEventDrivenGcsTransferSample { + /// + /// Sample that creates an event driven gcs data transfer from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. + /// + /// Your Google Cloud Project ID. + /// The GCS bucket to transfer data from. + /// The GCS bucket to transfer data to. + /// The subscription ID to a Pubsub queue to track. public TransferJob CreateEventDrivenGcsTransfer( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index cd92b85e2c3..e0d78bf742b 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -17,11 +17,16 @@ using Google.Cloud.StorageTransfer.V1; using System; -/// -/// Create a transfer for downloading objects from a gcs bucket to the root directory of POSIX file system. -/// public class DownloadToPosixSample { + /// + /// Sample that creates a transfer for downloading objects from a gcs bucket to the root directory of POSIX file system. + /// + /// Your Google Cloud Project ID. + /// The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent. + /// Your GCS source bucket name. + /// An optional path on the Google Cloud Storage bucket to download from. + /// The root directory path on the source filesystem. public TransferJob DownloadToPosix( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs index fe08f6082c2..a2a5bef3d84 100644 --- a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs @@ -21,6 +21,12 @@ public class QuickstartSample { + /// + /// Sample that creates a one-time transfer job from a Google cloud storage bucket to another bucket. + /// + /// Your Google Cloud Project ID. + /// The GCS bucket to transfer data from. + /// The GCS bucket to transfer data to. public TransferJob Quickstart( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs index 5d8abed0ce5..ba00450564e 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -17,11 +17,17 @@ using Google.Cloud.StorageTransfer.V1; using System; -/// -/// Create a transfer to transfer objects from root directory to the destination directory between POSIX file systems. -/// public class TransferBetweenPosixSample { + /// + /// Sample that creates a transfer to transfer objects from root directory to the destination directory between POSIX file systems. + /// + /// The ID of the project. + /// The agent pool associated with the POSIX data source. If not provided, defaults to the default agent. + /// The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent. + /// The root directory path on the source filesystem. + /// The root directory path on the sink filesystem. + /// The name of GCS bucket for intermediate storage. public TransferJob TransferBetweenPosix( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -34,7 +40,7 @@ public TransferJob TransferBetweenPosix( // The root directory path on the sink filesystem string destinationDirectory = "/directory/to/transfer/sink", // The name of GCS bucket for intermediate storage - string intermediate_bucket = "my-intermediate-bucket") + string intermediateBucket = "my-intermediate-bucket") { // A useful description for your transfer job string jobDescription = $"Transfer objects from {rootDirectory} to the {destinationDirectory} between POSIX file system"; @@ -49,7 +55,7 @@ public TransferJob TransferBetweenPosix( SinkAgentPoolName = sinkAgentPoolName, PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory }, PosixDataSink = new PosixFilesystem { RootDirectory = destinationDirectory }, - GcsIntermediateDataLocation = new GcsData { BucketName = intermediate_bucket } + GcsIntermediateDataLocation = new GcsData { BucketName = intermediateBucket } }, Status = TransferJob.Types.Status.Enabled, }; @@ -70,4 +76,4 @@ public TransferJob TransferBetweenPosix( return response; } } -//[END storagetransfer_transfer_posix_to_posix] +// [END storagetransfer_transfer_posix_to_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs index bc82f8f74dc..1307acff0b3 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs @@ -17,11 +17,15 @@ using Google.Cloud.StorageTransfer.V1; using System; -/// -/// Create a transfer to transfer objects from a POSIX file system to a gcs sink bucket. -/// public class TransferFromPosixSample { + /// + /// Sample that creates a transfer job to transfer objects from a POSIX file system to a gcs sink bucket. + /// + /// The ID of the project. + /// The agent pool associated with the POSIX data source. If not provided, defaults to the default agent. + /// The root directory path on the source filesystem. + /// The GCS bucket to transfer data to. public TransferJob TransferFromPosix( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index 98dcfb0406d..deaeca3fe3e 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -18,12 +18,15 @@ using Google.Protobuf.WellKnownTypes; using System; -/// -/// Creates a one-off transfer job that transfers objects from a standard gcs bucket that are more -/// than 30 days old to a nearline gcs bucket. -/// public class TransferToNearlineSample { + /// + /// Sample that creates a one-off transfer job that transfers objects from a standard gcs bucket that are more + /// than 30 days old to a nearline gcs bucket. + /// + /// Your Google Cloud Project ID. + /// The GCS bucket to transfer objects from. + /// The GCS Nearline bucket to transfer old objects to. public TransferJob TransferToNearline( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index c92860b2174..8154a36605c 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -17,11 +17,18 @@ using Google.Cloud.StorageTransfer.V1; using System; -/// -/// Create a transfer to transfer objects from a POSIX file system to a gcs sink bucket using a manifest file. -/// public class TransferUsingManifestSample { + /// + /// Sample that creates a transfer job to transfer objects from a POSIX file system to a gcs sink bucket using a manifest file. + /// + /// The ID of the project. + /// The agent pool associated with the POSIX data source. If not provided , + /// defaults to the default agent + /// The root directory path on the source filesystem. + /// The GCS bucket which has your manifest file. + /// The GCS bucket to transfer data to. + /// The name of the manifest file in manifestBucket that specifies which objects to transfer. public TransferJob TransferUsingManifest( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/testutil/TestBucket.cs b/testutil/TestBucket.cs index c5afc307d6d..f07f6406574 100644 --- a/testutil/TestBucket.cs +++ b/testutil/TestBucket.cs @@ -1,4 +1,4 @@ -// Copyright(c) 2017 Google Inc. +// Copyright(c) 2017 Google Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); you may not // use this file except in compliance with the License. You may obtain a copy of @@ -146,4 +146,4 @@ public void Dispose() _garbage.Clear(); } } -} \ No newline at end of file +} diff --git a/testutil/TestUtil.cs b/testutil/TestUtil.cs index 4b366d196eb..763ff37a2cf 100644 --- a/testutil/TestUtil.cs +++ b/testutil/TestUtil.cs @@ -1,4 +1,4 @@ -/* +/* * Copyright (c) 2016 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not @@ -38,8 +38,8 @@ public static string RandomName() while (nextChar < randomChars.Length) { rng.GetBytes(randomByte); - if (legalChars.Contains((char)randomByte[0])) - randomChars[nextChar++] = (char)randomByte[0]; + if (legalChars.Contains((char) randomByte[0])) + randomChars[nextChar++] = (char) randomByte[0]; } return new string(randomChars); } @@ -76,21 +76,21 @@ public T Eventually(Func func) catch (Exception e) when (ShouldCatch(e) && i < MaxTryCount) { int jitteredDelayMs; - lock(_lock) + lock (_lock) { - jitteredDelayMs = delayMs/2 + (int)(_random.NextDouble() * delayMs); + jitteredDelayMs = delayMs / 2 + (int) (_random.NextDouble() * delayMs); } Thread.Sleep(jitteredDelayMs); - delayMs *= (int)DelayMultiplier; + delayMs *= (int) DelayMultiplier; } } } public void Eventually(Action action) => Eventually(() => - { + { action(); - return 0; + return 0; }); @@ -108,10 +108,10 @@ public async Task Eventually(Func> asyncFunc) int jitteredDelayMs; lock (_lock) { - jitteredDelayMs = delayMs / 2 + (int)(_random.NextDouble() * delayMs); + jitteredDelayMs = delayMs / 2 + (int) (_random.NextDouble() * delayMs); } await Task.Delay(jitteredDelayMs); - delayMs *= (int)DelayMultiplier; + delayMs *= (int) DelayMultiplier; } } } @@ -119,9 +119,9 @@ public async Task Eventually(Func> asyncFunc) public async Task Eventually(Func action) { await Eventually(async () => - { + { await action(); - return 0; + return 0; }); } @@ -265,4 +265,4 @@ public override string ToString() } } -} \ No newline at end of file +} From b05a6bc42cf5f474ee2370cfac46a4c78260ce15 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 27 Mar 2025 03:58:21 -0700 Subject: [PATCH 38/58] Revert "Comments added in tests and sample description modified" This reverts commit f1b05b37942aa131ccff60fdd9084adfedd39260. --- storagetransfer/api/.editorconfig | 281 ------------------ .../CheckLatestTransferOperationTest.cs | 2 - .../CreateEventDrivenGcsTransferTest.cs | 1 - .../DownloadToPosixTest.cs | 2 - .../QuickstartTest.cs | 1 - .../TransferBetweenPosixTest.cs | 1 - .../TransferFromPosixTest.cs | 1 - .../TransferToNearlineTest.cs | 1 - .../TransferUsingManifestTest.cs | 2 - .../CheckLatestTransferOperationSample.cs | 8 +- .../CreateEventDrivenGcsTransferSample.cs | 10 +- .../DownloadToPosixSample.cs | 11 +- .../QuickstartSample.cs | 6 - .../TransferBetweenPosixSample.cs | 18 +- .../TransferFromPosixSample.cs | 10 +- .../TransferToNearlineSample.cs | 11 +- .../TransferUsingManifestSample.cs | 13 +- testutil/TestBucket.cs | 4 +- testutil/TestUtil.cs | 26 +- 19 files changed, 40 insertions(+), 369 deletions(-) delete mode 100644 storagetransfer/api/.editorconfig diff --git a/storagetransfer/api/.editorconfig b/storagetransfer/api/.editorconfig deleted file mode 100644 index 0ec4d4be40e..00000000000 --- a/storagetransfer/api/.editorconfig +++ /dev/null @@ -1,281 +0,0 @@ -# To learn more about .editorconfig see https://aka.ms/editorconfigdocs -# top-most EditorConfig file -root = true -############################### -# Core EditorConfig Options # -############################### -# All files -[*] -insert_final_newline = true -indent_style = space -indent_size = 4 -trim_trailing_whitespace = true - -# Generated code -[*{_AssemblyInfo.cs,.g.cs}] -generated_code = true - -# XML project files -[*.{csproj}] -indent_size = 2 - -# Code files - -############################### -# .NET Coding Conventions # -############################### -[*.{cs}] - -# Organize usings -dotnet_sort_system_directives_first = false - -# this. preferences -dotnet_style_qualification_for_field = false:suggestion -dotnet_style_qualification_for_property = false:suggestion -dotnet_style_qualification_for_method = false:suggestion -dotnet_style_qualification_for_event = false:suggestion - -# Language keywords vs BCL types preferences -dotnet_style_predefined_type_for_locals_parameters_members = true:suggestion -dotnet_style_predefined_type_for_member_access = true:suggestion - -# Parentheses preferences -dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:silent -dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:silent -dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:silent -dotnet_style_parentheses_in_other_operators = never_if_unnecessary:silent - -# Modifier preferences -csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:warning -dotnet_style_require_accessibility_modifiers = for_non_interface_members:warning -dotnet_style_readonly_field = true:warning - -# Expression-level preferences -dotnet_style_object_initializer = true:suggestion -dotnet_style_collection_initializer = true:suggestion -dotnet_style_explicit_tuple_names = true:suggestion -dotnet_style_null_propagation = true:suggestion -dotnet_style_coalesce_expression = true:suggestion -dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion -dotnet_style_prefer_inferred_tuple_names = true:suggestion -dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion -dotnet_style_prefer_auto_properties = true:suggestion -dotnet_style_prefer_conditional_expression_over_assignment = true:silent -dotnet_style_prefer_conditional_expression_over_return = true:silent -csharp_prefer_simple_default_expression = true:suggestion - -############################### -# Naming Conventions # -############################### -# Style Definitions -dotnet_naming_style.pascal_case_style.capitalization = pascal_case - -# Use PascalCase for constant fields -dotnet_naming_rule.constant_fields_should_be_pascal_case.severity = suggestion -dotnet_naming_rule.constant_fields_should_be_pascal_case.symbols = constant_fields -dotnet_naming_rule.constant_fields_should_be_pascal_case.style = pascal_case_style -dotnet_naming_symbols.constant_fields.applicable_kinds = field -dotnet_naming_symbols.constant_fields.applicable_accessibilities = * -dotnet_naming_symbols.constant_fields.required_modifiers = const -dotnet_naming_style.pascal_case_style.capitalization = pascal_case - -# static fields should have s_ prefix -dotnet_naming_rule.static_fields_should_have_prefix.severity = suggestion -dotnet_naming_rule.static_fields_should_have_prefix.symbols = static_fields -dotnet_naming_rule.static_fields_should_have_prefix.style = static_prefix_style -dotnet_naming_symbols.static_fields.applicable_kinds = field -dotnet_naming_symbols.static_fields.required_modifiers = static -dotnet_naming_symbols.static_fields.applicable_accessibilities = private, internal, private_protected -dotnet_naming_style.static_prefix_style.required_prefix = s_ -dotnet_naming_style.static_prefix_style.capitalization = camel_case - -# internal and private fields should be _camelCase -dotnet_naming_rule.camel_case_for_private_internal_fields.severity = suggestion -dotnet_naming_rule.camel_case_for_private_internal_fields.symbols = private_internal_fields -dotnet_naming_rule.camel_case_for_private_internal_fields.style = camel_case_underscore_style -dotnet_naming_symbols.private_internal_fields.applicable_kinds = field -dotnet_naming_symbols.private_internal_fields.applicable_accessibilities = private, internal -dotnet_naming_style.camel_case_underscore_style.required_prefix = _ -dotnet_naming_style.camel_case_underscore_style.capitalization = camel_case - -# Locals and parameters are camelCase -dotnet_naming_rule.locals_should_be_camel_case.severity = suggestion -dotnet_naming_rule.locals_should_be_camel_case.symbols = locals_and_parameters -dotnet_naming_rule.locals_should_be_camel_case.style = camel_case_style -dotnet_naming_symbols.locals_and_parameters.applicable_kinds = parameter, local -dotnet_naming_style.camel_case_style.capitalization = camel_case - -# Public property and methods should be PascalCase -dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.severity = suggestion -dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.symbols = public_methods_and_properties -dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.style = pascal_case -dotnet_naming_symbols.public_methods_and_properties.applicable_kinds = method, property -dotnet_naming_symbols.public_methods_and_properties.applicable_accessibilities = public -dotnet_naming_style.pascal_case.capitalization = pascal_case - -# Private and internal methods should be PascalCase -dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.severity = suggestion -dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.symbols = private_and_internal_methods -dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.style = pascal_case -dotnet_naming_symbols.private_and_internal_methods.applicable_kinds = method -dotnet_naming_symbols.private_and_internal_methods.applicable_accessibilities = private, internal -############################### -# C# Coding Conventions # -############################### -[*.cs] -# var preferences -csharp_style_var_for_built_in_types = true:silent -csharp_style_var_when_type_is_apparent = true:silent -csharp_style_var_elsewhere = true:silent -dotnet_style_predefined_type_for_locals_parameters_members = true:silent -dotnet_style_predefined_type_for_member_access = true:silent - -csharp_using_directive_placement = outside_namespace:warning -csharp_prefer_braces = true:suggestion -csharp_preserve_single_line_blocks = true:none -csharp_preserve_single_line_statements = false:none -csharp_prefer_static_local_function = true:suggestion -csharp_prefer_simple_using_statement = true:suggestion -csharp_style_prefer_switch_expression = true:suggestion - -# Expression-bodied members -csharp_style_expression_bodied_methods = true:suggestion -csharp_style_expression_bodied_constructors = true:suggestion -csharp_style_expression_bodied_operators = true:suggestion -csharp_style_expression_bodied_properties = true:suggestion -csharp_style_expression_bodied_indexers = true:suggestion -csharp_style_expression_bodied_accessors = true:suggestion -csharp_style_expression_bodied_lambdas = true:suggestion -csharp_style_expression_bodied_local_functions = true:suggestion - -# Pattern matching preferences -csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion -csharp_style_pattern_matching_over_as_with_null_check = true:suggestion -csharp_style_inlined_variable_declaration = true:suggestion - -# Null-checking preferences -csharp_style_throw_expression = true:suggestion -csharp_style_conditional_delegate_call = true:suggestion - -# Other features -csharp_style_prefer_index_operator = true:suggestion -csharp_style_prefer_range_operator = true:suggestion -csharp_style_pattern_local_over_anonymous_function = false:none - -############################### -# C# Formatting Rules # -############################### -indent_size = 4 -insert_final_newline = true -charset = utf-8 - -# New line preferences -csharp_new_line_before_open_brace = all -csharp_new_line_before_else = true -csharp_new_line_before_catch = true -csharp_new_line_before_finally = true -csharp_new_line_before_members_in_object_initializers = true -csharp_new_line_before_members_in_anonymous_types = true -csharp_new_line_between_query_expression_clauses = true - -# Indentation preferences -csharp_indent_block_contents = true -csharp_indent_braces = false -csharp_indent_case_contents = true -csharp_indent_case_contents_when_block = true -csharp_indent_switch_labels = true -csharp_indent_labels = one_less_than_current - -# Space preferences -csharp_space_after_cast = true -csharp_space_after_colon_in_inheritance_clause = true -csharp_space_after_comma = true -csharp_space_after_dot = false -csharp_space_after_keywords_in_control_flow_statements = true -csharp_space_after_semicolon_in_for_statement = true -csharp_space_between_method_call_parameter_list_parentheses = false -csharp_space_between_method_declaration_parameter_list_parentheses = false -csharp_space_between_parentheses = false -csharp_space_before_comma = false -csharp_space_before_dot = false -csharp_space_before_open_square_brackets = false -csharp_space_before_semicolon_in_for_statement = false -csharp_space_between_empty_square_brackets = false -csharp_space_around_declaration_statements = do_not_ignore -csharp_space_before_colon_in_inheritance_clause = true -csharp_space_after_colon_in_inheritance_clause = true -csharp_space_around_binary_operators = before_and_after -csharp_space_between_method_declaration_empty_parameter_list_parentheses = false -csharp_space_between_method_call_name_and_opening_parenthesis = false -csharp_space_between_method_call_empty_parameter_list_parentheses = false -csharp_space_between_method_declaration_name_and_open_parenthesis = false -csharp_space_between_square_brackets = false - -# Wrapping preferences -csharp_preserve_single_line_statements = true -csharp_preserve_single_line_blocks = true - -# License header -file_header_template = Copyright 2024 Google LLC\n\nLicensed under the Apache License, Version 2.0 (the "License").\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at \n\nhttps://www.apache.org/licenses/LICENSE-2.0 \n\nUnless required by applicable law or agreed to in writing, software \ndistributed under the License is distributed on an "AS IS" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and \nlimitations under the License. -csharp_style_namespace_declarations = file_scoped:suggestion -csharp_style_prefer_null_check_over_type_check = true:suggestion -csharp_style_prefer_local_over_anonymous_function = true:suggestion -csharp_style_prefer_tuple_swap = true:suggestion -csharp_style_implicit_object_creation_when_type_is_apparent = true:suggestion -csharp_style_unused_value_assignment_preference = discard_variable:suggestion -csharp_style_unused_value_expression_statement_preference = discard_variable:suggestion -csharp_style_prefer_pattern_matching = true:suggestion -csharp_style_prefer_not_pattern = true:suggestion -csharp_style_prefer_extended_property_pattern = true:suggestion -csharp_style_prefer_parameter_null_checking = true:suggestion -csharp_style_prefer_method_group_conversion = true:suggestion - -# Change the severity to `error` for the rules that must not be violated, to fail the build. - -# IDE0005: Remove unnecessary using directives. -dotnet_diagnostic.IDE0005.severity = suggestion - -# IDE0011: Add braces -dotnet_diagnostic.IDE0011.severity = suggestion - -# IDE0017: Object initialization can be simplified. -dotnet_diagnostic.IDE0017.severity = suggestion - -# IDE0019: Use pattern matching. -dotnet_diagnostic.IDE0019.severity = suggestion - -# IDE0021: Use expression body for constructor. -dotnet_diagnostic.IDE0021.severity = suggestion - -# IDE0028: Collection initialization can be simplified. -dotnet_diagnostic.IDE0028.severity = suggestion - -# IDE0035: Remove unreachable code. -dotnet_diagnostic.IDE0035.severity = suggestion - -# IDE0040: Accessibility modifiers required. -dotnet_diagnostic.IDE0040.severity = suggestion - -# IDE0052: Remove unread private member -dotnet_diagnostic.IDE0052.severity = suggestion - -# IDE0053: Use expression body for lambda expression. -dotnet_diagnostic.IDE0053.severity = suggestion - -# IDE0055: All C# and .NET formatting rules. -dotnet_diagnostic.IDE0055.severity = suggestion - -# IDE0060: Remove unused parameter -dotnet_diagnostic.IDE0060.severity = suggestion - -# IDE0063: Using statement can be simplified -dotnet_diagnostic.IDE0063.severity = suggestion - -# IDE0065: Using statement can be simplified -dotnet_diagnostic.IDE0065.severity = suggestion - -# IDE0071: Interpolation can be simplified. -dotnet_diagnostic.IDE0071.severity = suggestion - -# IDE0075: Simplify conditional expression. -dotnet_diagnostic.IDE0075.severity = suggestion \ No newline at end of file diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs index f798dc5b882..f9107e38d98 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs @@ -35,7 +35,6 @@ public CheckLatestTransferOperationTest(StorageFixture fixture) _transferJobName = CreateTransferJob(); } - // Checks the latest transfer operation for a created transfer job. [Fact] public void CheckLatestTransferOperation() { @@ -45,7 +44,6 @@ public void CheckLatestTransferOperation() _jobName = transferJob.Name; } - // Creates one-time transfer job from a Google cloud storage bucket to another bucket. private string CreateTransferJob() { // Initialize request argument(s) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs index 54a85805ea3..cba7c2b99fd 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -76,7 +76,6 @@ public CreateEventDrivenGcsTransferTest(StorageFixture fixture) }); } - // Transfers object from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. [Fact] public void CreateEventDrivenGcsTransfer() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index bc3c3247fdf..ac9116f3abc 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -39,7 +39,6 @@ public DownloadToPosixTest(StorageFixture fixture) UploadObjectToPosixBucket(_bucketNamePosixSource); } - // Downloads an object from a GCS bucket to a POSIX file system. [Fact] public void DownloadToPosix() { @@ -51,7 +50,6 @@ public void DownloadToPosix() _transferJobName = transferJob.Name; } - // Uploads an object to the POSIX bucket. private void UploadObjectToPosixBucket(string bucketName) { byte[] byteArray = System.Text.Encoding.UTF8.GetBytes($@"{Guid.NewGuid()}.jpeg"); diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs index 68f5868e5ae..9efbb3334a1 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs @@ -35,7 +35,6 @@ public QuickstartTest(StorageFixture fixture) _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); } - // Creates a one-time transfer job from a Google cloud storage bucket to another bucket [Fact] public void TestQuickstart() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index 1b5d8045cfe..f267930f566 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -36,7 +36,6 @@ public TransferBetweenPosixTest(StorageFixture fixture) _tempDestinationDirectory = fixture.GenerateTempFolderPath(); } - // Transfers objects from a POSIX file system to another POSIX file system. [Fact] public void TransferBetweenPosix() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs index b81eeb7689b..88f763d25fc 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs @@ -33,7 +33,6 @@ public TransferFromPosixTest(StorageFixture fixture) _rootDirectory = fixture.GetCurrentUserTempFolderPath(); } - // Transfers objects from a POSIX file system to a Cloud Storage bucket. [Fact] public void TransferFromPosix() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs index d0eaa9a6f00..eacce91d255 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -34,7 +34,6 @@ public TransferToNearlineTest(StorageFixture fixture) _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); } - // Transfers objects from a Google Cloud Storage bucket to another bucket with Nearline storage class. [Fact] public void TestTransferToNearline() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index dde52fc6548..58711707643 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -41,7 +41,6 @@ public TransferUsingManifestTest(StorageFixture fixture) UploadObjectToManifestBucket(_bucketNameManifestSource); } - // Transfers objects from a POSIX file system to a sink bucket using a manifest file. [Fact] public void TransferUsingManifest() { @@ -51,7 +50,6 @@ public void TransferUsingManifest() _transferJobName = transferJob.Name; } - // Uploads an object to the manifest bucket. private void UploadObjectToManifestBucket(string bucketName) { byte[] byteArray = System.Text.Encoding.UTF8.GetBytes($@"{Guid.NewGuid()}.jpeg"); diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs index 04df26cec05..f0f188b04e2 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -17,13 +17,11 @@ using Google.Cloud.StorageTransfer.V1; using System; +/// +/// Checks the latest transfer operation for a given transfer job. +/// public class CheckLatestTransferOperationSample { - /// - /// Sample that checks the latest transfer operation for a given transfer job. - /// - /// Your Google Cloud Project ID. - /// The name of the job to check. public TransferJob CheckLatestTransferOperation( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs index 3619408aca0..49465a395f3 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs @@ -19,15 +19,11 @@ using Google.Cloud.StorageTransfer.V1; using System; +/// +/// Create an event driven gcs data transfer from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. +/// public class CreateEventDrivenGcsTransferSample { - /// - /// Sample that creates an event driven gcs data transfer from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. - /// - /// Your Google Cloud Project ID. - /// The GCS bucket to transfer data from. - /// The GCS bucket to transfer data to. - /// The subscription ID to a Pubsub queue to track. public TransferJob CreateEventDrivenGcsTransfer( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index e0d78bf742b..cd92b85e2c3 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -17,16 +17,11 @@ using Google.Cloud.StorageTransfer.V1; using System; +/// +/// Create a transfer for downloading objects from a gcs bucket to the root directory of POSIX file system. +/// public class DownloadToPosixSample { - /// - /// Sample that creates a transfer for downloading objects from a gcs bucket to the root directory of POSIX file system. - /// - /// Your Google Cloud Project ID. - /// The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent. - /// Your GCS source bucket name. - /// An optional path on the Google Cloud Storage bucket to download from. - /// The root directory path on the source filesystem. public TransferJob DownloadToPosix( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs index a2a5bef3d84..fe08f6082c2 100644 --- a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs @@ -21,12 +21,6 @@ public class QuickstartSample { - /// - /// Sample that creates a one-time transfer job from a Google cloud storage bucket to another bucket. - /// - /// Your Google Cloud Project ID. - /// The GCS bucket to transfer data from. - /// The GCS bucket to transfer data to. public TransferJob Quickstart( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs index ba00450564e..5d8abed0ce5 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -17,17 +17,11 @@ using Google.Cloud.StorageTransfer.V1; using System; +/// +/// Create a transfer to transfer objects from root directory to the destination directory between POSIX file systems. +/// public class TransferBetweenPosixSample { - /// - /// Sample that creates a transfer to transfer objects from root directory to the destination directory between POSIX file systems. - /// - /// The ID of the project. - /// The agent pool associated with the POSIX data source. If not provided, defaults to the default agent. - /// The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent. - /// The root directory path on the source filesystem. - /// The root directory path on the sink filesystem. - /// The name of GCS bucket for intermediate storage. public TransferJob TransferBetweenPosix( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -40,7 +34,7 @@ public TransferJob TransferBetweenPosix( // The root directory path on the sink filesystem string destinationDirectory = "/directory/to/transfer/sink", // The name of GCS bucket for intermediate storage - string intermediateBucket = "my-intermediate-bucket") + string intermediate_bucket = "my-intermediate-bucket") { // A useful description for your transfer job string jobDescription = $"Transfer objects from {rootDirectory} to the {destinationDirectory} between POSIX file system"; @@ -55,7 +49,7 @@ public TransferJob TransferBetweenPosix( SinkAgentPoolName = sinkAgentPoolName, PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory }, PosixDataSink = new PosixFilesystem { RootDirectory = destinationDirectory }, - GcsIntermediateDataLocation = new GcsData { BucketName = intermediateBucket } + GcsIntermediateDataLocation = new GcsData { BucketName = intermediate_bucket } }, Status = TransferJob.Types.Status.Enabled, }; @@ -76,4 +70,4 @@ public TransferJob TransferBetweenPosix( return response; } } -// [END storagetransfer_transfer_posix_to_posix] +//[END storagetransfer_transfer_posix_to_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs index 1307acff0b3..bc82f8f74dc 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs @@ -17,15 +17,11 @@ using Google.Cloud.StorageTransfer.V1; using System; +/// +/// Create a transfer to transfer objects from a POSIX file system to a gcs sink bucket. +/// public class TransferFromPosixSample { - /// - /// Sample that creates a transfer job to transfer objects from a POSIX file system to a gcs sink bucket. - /// - /// The ID of the project. - /// The agent pool associated with the POSIX data source. If not provided, defaults to the default agent. - /// The root directory path on the source filesystem. - /// The GCS bucket to transfer data to. public TransferJob TransferFromPosix( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index deaeca3fe3e..98dcfb0406d 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -18,15 +18,12 @@ using Google.Protobuf.WellKnownTypes; using System; +/// +/// Creates a one-off transfer job that transfers objects from a standard gcs bucket that are more +/// than 30 days old to a nearline gcs bucket. +/// public class TransferToNearlineSample { - /// - /// Sample that creates a one-off transfer job that transfers objects from a standard gcs bucket that are more - /// than 30 days old to a nearline gcs bucket. - /// - /// Your Google Cloud Project ID. - /// The GCS bucket to transfer objects from. - /// The GCS Nearline bucket to transfer old objects to. public TransferJob TransferToNearline( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index 8154a36605c..c92860b2174 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -17,18 +17,11 @@ using Google.Cloud.StorageTransfer.V1; using System; +/// +/// Create a transfer to transfer objects from a POSIX file system to a gcs sink bucket using a manifest file. +/// public class TransferUsingManifestSample { - /// - /// Sample that creates a transfer job to transfer objects from a POSIX file system to a gcs sink bucket using a manifest file. - /// - /// The ID of the project. - /// The agent pool associated with the POSIX data source. If not provided , - /// defaults to the default agent - /// The root directory path on the source filesystem. - /// The GCS bucket which has your manifest file. - /// The GCS bucket to transfer data to. - /// The name of the manifest file in manifestBucket that specifies which objects to transfer. public TransferJob TransferUsingManifest( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/testutil/TestBucket.cs b/testutil/TestBucket.cs index f07f6406574..c5afc307d6d 100644 --- a/testutil/TestBucket.cs +++ b/testutil/TestBucket.cs @@ -1,4 +1,4 @@ -// Copyright(c) 2017 Google Inc. +// Copyright(c) 2017 Google Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); you may not // use this file except in compliance with the License. You may obtain a copy of @@ -146,4 +146,4 @@ public void Dispose() _garbage.Clear(); } } -} +} \ No newline at end of file diff --git a/testutil/TestUtil.cs b/testutil/TestUtil.cs index 763ff37a2cf..4b366d196eb 100644 --- a/testutil/TestUtil.cs +++ b/testutil/TestUtil.cs @@ -1,4 +1,4 @@ -/* +/* * Copyright (c) 2016 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not @@ -38,8 +38,8 @@ public static string RandomName() while (nextChar < randomChars.Length) { rng.GetBytes(randomByte); - if (legalChars.Contains((char) randomByte[0])) - randomChars[nextChar++] = (char) randomByte[0]; + if (legalChars.Contains((char)randomByte[0])) + randomChars[nextChar++] = (char)randomByte[0]; } return new string(randomChars); } @@ -76,21 +76,21 @@ public T Eventually(Func func) catch (Exception e) when (ShouldCatch(e) && i < MaxTryCount) { int jitteredDelayMs; - lock (_lock) + lock(_lock) { - jitteredDelayMs = delayMs / 2 + (int) (_random.NextDouble() * delayMs); + jitteredDelayMs = delayMs/2 + (int)(_random.NextDouble() * delayMs); } Thread.Sleep(jitteredDelayMs); - delayMs *= (int) DelayMultiplier; + delayMs *= (int)DelayMultiplier; } } } public void Eventually(Action action) => Eventually(() => - { + { action(); - return 0; + return 0; }); @@ -108,10 +108,10 @@ public async Task Eventually(Func> asyncFunc) int jitteredDelayMs; lock (_lock) { - jitteredDelayMs = delayMs / 2 + (int) (_random.NextDouble() * delayMs); + jitteredDelayMs = delayMs / 2 + (int)(_random.NextDouble() * delayMs); } await Task.Delay(jitteredDelayMs); - delayMs *= (int) DelayMultiplier; + delayMs *= (int)DelayMultiplier; } } } @@ -119,9 +119,9 @@ public async Task Eventually(Func> asyncFunc) public async Task Eventually(Func action) { await Eventually(async () => - { + { await action(); - return 0; + return 0; }); } @@ -265,4 +265,4 @@ public override string ToString() } } -} +} \ No newline at end of file From 526ddade33e7445a3a2079dd00e52564a834003d Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 27 Mar 2025 04:39:48 -0700 Subject: [PATCH 39/58] Comments are added and sample description modified --- .../CheckLatestTransferOperationTest.cs | 2 ++ .../CreateEventDrivenGcsTransferTest.cs | 1 + .../DownloadToPosixTest.cs | 2 ++ .../TransferBetweenPosixTest.cs | 1 + .../TransferFromPosixTest.cs | 1 + .../TransferToNearlineTest.cs | 1 + .../TransferUsingManifestTest.cs | 2 ++ .../CheckLatestTransferOperationSample.cs | 8 +++++--- .../CreateEventDrivenGcsTransferSample.cs | 10 +++++++--- .../DownloadToPosixSample.cs | 11 ++++++++--- .../TransferBetweenPosixSample.cs | 18 ++++++++++++------ .../TransferFromPosixSample.cs | 10 +++++++--- .../TransferToNearlineSample.cs | 11 +++++++---- .../TransferUsingManifestSample.cs | 13 ++++++++++--- 14 files changed, 66 insertions(+), 25 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs index f9107e38d98..f798dc5b882 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs @@ -35,6 +35,7 @@ public CheckLatestTransferOperationTest(StorageFixture fixture) _transferJobName = CreateTransferJob(); } + // Checks the latest transfer operation for a created transfer job. [Fact] public void CheckLatestTransferOperation() { @@ -44,6 +45,7 @@ public void CheckLatestTransferOperation() _jobName = transferJob.Name; } + // Creates one-time transfer job from a Google cloud storage bucket to another bucket. private string CreateTransferJob() { // Initialize request argument(s) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs index cba7c2b99fd..54a85805ea3 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -76,6 +76,7 @@ public CreateEventDrivenGcsTransferTest(StorageFixture fixture) }); } + // Transfers object from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. [Fact] public void CreateEventDrivenGcsTransfer() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index ac9116f3abc..bc3c3247fdf 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -39,6 +39,7 @@ public DownloadToPosixTest(StorageFixture fixture) UploadObjectToPosixBucket(_bucketNamePosixSource); } + // Downloads an object from a GCS bucket to a POSIX file system. [Fact] public void DownloadToPosix() { @@ -50,6 +51,7 @@ public void DownloadToPosix() _transferJobName = transferJob.Name; } + // Uploads an object to the POSIX bucket. private void UploadObjectToPosixBucket(string bucketName) { byte[] byteArray = System.Text.Encoding.UTF8.GetBytes($@"{Guid.NewGuid()}.jpeg"); diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index f267930f566..1b5d8045cfe 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -36,6 +36,7 @@ public TransferBetweenPosixTest(StorageFixture fixture) _tempDestinationDirectory = fixture.GenerateTempFolderPath(); } + // Transfers objects from a POSIX file system to another POSIX file system. [Fact] public void TransferBetweenPosix() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs index 88f763d25fc..b81eeb7689b 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs @@ -33,6 +33,7 @@ public TransferFromPosixTest(StorageFixture fixture) _rootDirectory = fixture.GetCurrentUserTempFolderPath(); } + // Transfers objects from a POSIX file system to a Cloud Storage bucket. [Fact] public void TransferFromPosix() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs index eacce91d255..d0eaa9a6f00 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -34,6 +34,7 @@ public TransferToNearlineTest(StorageFixture fixture) _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); } + // Transfers objects from a Google Cloud Storage bucket to another bucket with Nearline storage class. [Fact] public void TestTransferToNearline() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index 58711707643..dde52fc6548 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -41,6 +41,7 @@ public TransferUsingManifestTest(StorageFixture fixture) UploadObjectToManifestBucket(_bucketNameManifestSource); } + // Transfers objects from a POSIX file system to a sink bucket using a manifest file. [Fact] public void TransferUsingManifest() { @@ -50,6 +51,7 @@ public void TransferUsingManifest() _transferJobName = transferJob.Name; } + // Uploads an object to the manifest bucket. private void UploadObjectToManifestBucket(string bucketName) { byte[] byteArray = System.Text.Encoding.UTF8.GetBytes($@"{Guid.NewGuid()}.jpeg"); diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs index f0f188b04e2..04df26cec05 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -17,11 +17,13 @@ using Google.Cloud.StorageTransfer.V1; using System; -/// -/// Checks the latest transfer operation for a given transfer job. -/// public class CheckLatestTransferOperationSample { + /// + /// Sample that checks the latest transfer operation for a given transfer job. + /// + /// Your Google Cloud Project ID. + /// The name of the job to check. public TransferJob CheckLatestTransferOperation( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs index 49465a395f3..3619408aca0 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs @@ -19,11 +19,15 @@ using Google.Cloud.StorageTransfer.V1; using System; -/// -/// Create an event driven gcs data transfer from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. -/// public class CreateEventDrivenGcsTransferSample { + /// + /// Sample that creates an event driven gcs data transfer from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. + /// + /// Your Google Cloud Project ID. + /// The GCS bucket to transfer data from. + /// The GCS bucket to transfer data to. + /// The subscription ID to a Pubsub queue to track. public TransferJob CreateEventDrivenGcsTransfer( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index cd92b85e2c3..e0d78bf742b 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -17,11 +17,16 @@ using Google.Cloud.StorageTransfer.V1; using System; -/// -/// Create a transfer for downloading objects from a gcs bucket to the root directory of POSIX file system. -/// public class DownloadToPosixSample { + /// + /// Sample that creates a transfer for downloading objects from a gcs bucket to the root directory of POSIX file system. + /// + /// Your Google Cloud Project ID. + /// The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent. + /// Your GCS source bucket name. + /// An optional path on the Google Cloud Storage bucket to download from. + /// The root directory path on the source filesystem. public TransferJob DownloadToPosix( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs index 5d8abed0ce5..ba00450564e 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -17,11 +17,17 @@ using Google.Cloud.StorageTransfer.V1; using System; -/// -/// Create a transfer to transfer objects from root directory to the destination directory between POSIX file systems. -/// public class TransferBetweenPosixSample { + /// + /// Sample that creates a transfer to transfer objects from root directory to the destination directory between POSIX file systems. + /// + /// The ID of the project. + /// The agent pool associated with the POSIX data source. If not provided, defaults to the default agent. + /// The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent. + /// The root directory path on the source filesystem. + /// The root directory path on the sink filesystem. + /// The name of GCS bucket for intermediate storage. public TransferJob TransferBetweenPosix( // Your Google Cloud Project ID string projectId = "my-project-id", @@ -34,7 +40,7 @@ public TransferJob TransferBetweenPosix( // The root directory path on the sink filesystem string destinationDirectory = "/directory/to/transfer/sink", // The name of GCS bucket for intermediate storage - string intermediate_bucket = "my-intermediate-bucket") + string intermediateBucket = "my-intermediate-bucket") { // A useful description for your transfer job string jobDescription = $"Transfer objects from {rootDirectory} to the {destinationDirectory} between POSIX file system"; @@ -49,7 +55,7 @@ public TransferJob TransferBetweenPosix( SinkAgentPoolName = sinkAgentPoolName, PosixDataSource = new PosixFilesystem { RootDirectory = rootDirectory }, PosixDataSink = new PosixFilesystem { RootDirectory = destinationDirectory }, - GcsIntermediateDataLocation = new GcsData { BucketName = intermediate_bucket } + GcsIntermediateDataLocation = new GcsData { BucketName = intermediateBucket } }, Status = TransferJob.Types.Status.Enabled, }; @@ -70,4 +76,4 @@ public TransferJob TransferBetweenPosix( return response; } } -//[END storagetransfer_transfer_posix_to_posix] +// [END storagetransfer_transfer_posix_to_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs index bc82f8f74dc..1307acff0b3 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs @@ -17,11 +17,15 @@ using Google.Cloud.StorageTransfer.V1; using System; -/// -/// Create a transfer to transfer objects from a POSIX file system to a gcs sink bucket. -/// public class TransferFromPosixSample { + /// + /// Sample that creates a transfer job to transfer objects from a POSIX file system to a gcs sink bucket. + /// + /// The ID of the project. + /// The agent pool associated with the POSIX data source. If not provided, defaults to the default agent. + /// The root directory path on the source filesystem. + /// The GCS bucket to transfer data to. public TransferJob TransferFromPosix( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index 98dcfb0406d..deaeca3fe3e 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -18,12 +18,15 @@ using Google.Protobuf.WellKnownTypes; using System; -/// -/// Creates a one-off transfer job that transfers objects from a standard gcs bucket that are more -/// than 30 days old to a nearline gcs bucket. -/// public class TransferToNearlineSample { + /// + /// Sample that creates a one-off transfer job that transfers objects from a standard gcs bucket that are more + /// than 30 days old to a nearline gcs bucket. + /// + /// Your Google Cloud Project ID. + /// The GCS bucket to transfer objects from. + /// The GCS Nearline bucket to transfer old objects to. public TransferJob TransferToNearline( // Your Google Cloud Project ID string projectId = "my-project-id", diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index c92860b2174..8154a36605c 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -17,11 +17,18 @@ using Google.Cloud.StorageTransfer.V1; using System; -/// -/// Create a transfer to transfer objects from a POSIX file system to a gcs sink bucket using a manifest file. -/// public class TransferUsingManifestSample { + /// + /// Sample that creates a transfer job to transfer objects from a POSIX file system to a gcs sink bucket using a manifest file. + /// + /// The ID of the project. + /// The agent pool associated with the POSIX data source. If not provided , + /// defaults to the default agent + /// The root directory path on the source filesystem. + /// The GCS bucket which has your manifest file. + /// The GCS bucket to transfer data to. + /// The name of the manifest file in manifestBucket that specifies which objects to transfer. public TransferJob TransferUsingManifest( // Your Google Cloud Project ID string projectId = "my-project-id", From c6950d8b00ece45343e86e257b58c3131148d557 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Thu, 27 Mar 2025 22:18:18 -0700 Subject: [PATCH 40/58] Quickstart test and sample changes --- .../api/StorageTransfer.Samples.Tests/QuickstartTest.cs | 1 + .../api/StorageTransfer.Samples/QuickstartSample.cs | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs index 9efbb3334a1..3c6fb011cf7 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs @@ -35,6 +35,7 @@ public QuickstartTest(StorageFixture fixture) _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); } + // Creates a one-time transfer job from a Google cloud storage bucket to another bucket. [Fact] public void TestQuickstart() { diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs index fe08f6082c2..a2a5bef3d84 100644 --- a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs @@ -21,6 +21,12 @@ public class QuickstartSample { + /// + /// Sample that creates a one-time transfer job from a Google cloud storage bucket to another bucket. + /// + /// Your Google Cloud Project ID. + /// The GCS bucket to transfer data from. + /// The GCS bucket to transfer data to. public TransferJob Quickstart( // Your Google Cloud Project ID string projectId = "my-project-id", From d043b9e8358030305693f6f101601822ba180b20 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Fri, 28 Mar 2025 00:22:23 -0700 Subject: [PATCH 41/58] minor changes --- .../api/StorageTransfer.Samples/DownloadToPosixSample.cs | 2 +- .../api/StorageTransfer.Samples/TransferBetweenPosixSample.cs | 2 +- .../StorageTransfer.Samples/TransferUsingManifestSample.cs | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index e0d78bf742b..31e1937309f 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -71,4 +71,4 @@ public TransferJob DownloadToPosix( return response; } } -//[END storagetransfer_download_to_posix] +// [END storagetransfer_download_to_posix] diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs index ba00450564e..78973c0d919 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -20,7 +20,7 @@ public class TransferBetweenPosixSample { /// - /// Sample that creates a transfer to transfer objects from root directory to the destination directory between POSIX file systems. + /// Sample that creates a transfer that transfer objects from root directory to the destination directory between POSIX file systems. /// /// The ID of the project. /// The agent pool associated with the POSIX data source. If not provided, defaults to the default agent. diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index 8154a36605c..cf75fb9a2e4 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -20,10 +20,10 @@ public class TransferUsingManifestSample { /// - /// Sample that creates a transfer job to transfer objects from a POSIX file system to a gcs sink bucket using a manifest file. + /// Sample that creates a transfer job that transfer objects from a POSIX file system to a gcs sink bucket using a manifest file. /// /// The ID of the project. - /// The agent pool associated with the POSIX data source. If not provided , + /// The agent pool associated with the POSIX data source. If not provided, /// defaults to the default agent /// The root directory path on the source filesystem. /// The GCS bucket which has your manifest file. From 869274aae549912b2667a51eb4fee47aef0d4a3a Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Mon, 14 Apr 2025 03:39:05 -0700 Subject: [PATCH 42/58] Sample and Test changes --- .../CheckLatestTransferOperationTest.cs | 5 +-- .../CreateEventDrivenGcsTransferTest.cs | 36 ++++++++--------- .../DownloadToPosixTest.cs | 2 - .../QuickstartTest.cs | 1 - .../StorageFixture.cs | 40 ++++++++++++++----- .../TransferBetweenPosixTest.cs | 1 - .../TransferFromPosixTest.cs | 1 - .../TransferToNearlineTest.cs | 3 +- .../TransferUsingManifestTest.cs | 2 - .../CheckLatestTransferOperationSample.cs | 11 ++--- .../CreateEventDrivenGcsTransferSample.cs | 11 ++--- .../DownloadToPosixSample.cs | 14 +------ .../QuickstartSample.cs | 7 +--- .../TransferBetweenPosixSample.cs | 15 +------ .../TransferFromPosixSample.cs | 13 +----- .../TransferToNearlineSample.cs | 11 +---- .../TransferUsingManifestSample.cs | 15 +------ 17 files changed, 66 insertions(+), 122 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs index f798dc5b882..b5f3e934a31 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs @@ -35,7 +35,6 @@ public CheckLatestTransferOperationTest(StorageFixture fixture) _transferJobName = CreateTransferJob(); } - // Checks the latest transfer operation for a created transfer job. [Fact] public void CheckLatestTransferOperation() { @@ -45,10 +44,8 @@ public void CheckLatestTransferOperation() _jobName = transferJob.Name; } - // Creates one-time transfer job from a Google cloud storage bucket to another bucket. private string CreateTransferJob() { - // Initialize request argument(s) TransferJob transferJob = new TransferJob { ProjectId = _fixture.ProjectId, @@ -63,7 +60,7 @@ private string CreateTransferJob() { TransferJob = transferJob }; - // Make the request + TransferJob response = _fixture.Sts.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); return response.Name; } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs index 54a85805ea3..c8687af21d7 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -27,8 +27,8 @@ public class CreateEventDrivenGcsTransferTest : IDisposable private readonly StorageFixture _fixture; private readonly string _pubSubId; private string _transferJobName; - private string TopicId { get; } = "DotNetTopic" + Guid.NewGuid().ToString(); - private string SubscriptionId { get; } = "DotNetSubscription" + Guid.NewGuid().ToString(); + private string TopicId { get; } = Guid.NewGuid().ToString(); + private string SubscriptionId { get; } = Guid.NewGuid().ToString(); private SubscriberServiceApiClient SubscriberClient { get; } = SubscriberServiceApiClient.Create(); private PublisherServiceApiClient PublisherClient { get; } = PublisherServiceApiClient.Create(); @@ -40,25 +40,33 @@ public CreateEventDrivenGcsTransferTest(StorageFixture fixture) _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSource); _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); _pubSubId = $"projects/{_fixture.ProjectId}/subscriptions/{SubscriptionId}"; + CreatePubSubResourcesAndGrantStsPermissions(); + } + + [Fact] + public void CreateEventDrivenGcsTransfer() + { + CreateEventDrivenGcsTransferSample createEventDrivenGcsTransferSample = new CreateEventDrivenGcsTransferSample(); + var transferJob = createEventDrivenGcsTransferSample.CreateEventDrivenGcsTransfer(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink, _pubSubId); + Assert.Contains("transferJobs/", transferJob.Name); + _transferJobName = transferJob.Name; + } + + private void CreatePubSubResourcesAndGrantStsPermissions() + { string email = _fixture.Sts.GetGoogleServiceAccount(new GetGoogleServiceAccountRequest() { ProjectId = _fixture.ProjectId }).AccountEmail; string memberServiceAccount = "serviceAccount:" + email; - // Create subscription name SubscriptionName subscriptionName = new SubscriptionName(_fixture.ProjectId, SubscriptionId); - // Create topic name TopicName topicName = new TopicName(_fixture.ProjectId, TopicId); - // Create topic PublisherClient.CreateTopic(topicName); - // Create subscription. SubscriberClient.CreateSubscription(subscriptionName, topicName, pushConfig: null, ackDeadlineSeconds: 500); var policyIamPolicyTopic = new Google.Cloud.Iam.V1.Policy(); - policyIamPolicyTopic.AddRoleMember("roles/pubsub.publisher", memberServiceAccount); - PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest { ResourceAsResourceName = topicName, @@ -66,9 +74,7 @@ public CreateEventDrivenGcsTransferTest(StorageFixture fixture) }); var policyIamPolicySubscriber = new Google.Cloud.Iam.V1.Policy(); - policyIamPolicySubscriber.AddRoleMember("roles/pubsub.subscriber", memberServiceAccount); - PublisherClient.IAMPolicyClient.SetIamPolicy(new Google.Cloud.Iam.V1.SetIamPolicyRequest { ResourceAsResourceName = subscriptionName, @@ -76,16 +82,6 @@ public CreateEventDrivenGcsTransferTest(StorageFixture fixture) }); } - // Transfers object from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. - [Fact] - public void CreateEventDrivenGcsTransfer() - { - CreateEventDrivenGcsTransferSample createEventDrivenGcsTransferSample = new CreateEventDrivenGcsTransferSample(); - var transferJob = createEventDrivenGcsTransferSample.CreateEventDrivenGcsTransfer(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink, _pubSubId); - Assert.Contains("transferJobs/", transferJob.Name); - _transferJobName = transferJob.Name; - } - public void Dispose() { try diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index bc3c3247fdf..ac9116f3abc 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -39,7 +39,6 @@ public DownloadToPosixTest(StorageFixture fixture) UploadObjectToPosixBucket(_bucketNamePosixSource); } - // Downloads an object from a GCS bucket to a POSIX file system. [Fact] public void DownloadToPosix() { @@ -51,7 +50,6 @@ public void DownloadToPosix() _transferJobName = transferJob.Name; } - // Uploads an object to the POSIX bucket. private void UploadObjectToPosixBucket(string bucketName) { byte[] byteArray = System.Text.Encoding.UTF8.GetBytes($@"{Guid.NewGuid()}.jpeg"); diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs index 3c6fb011cf7..9efbb3334a1 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs @@ -35,7 +35,6 @@ public QuickstartTest(StorageFixture fixture) _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); } - // Creates a one-time transfer job from a Google cloud storage bucket to another bucket. [Fact] public void TestQuickstart() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index fbd53117768..df85646e2eb 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -14,12 +14,15 @@ * limitations under the License. */ +using Google; using Google.Apis.Storage.v1.Data; using Google.Cloud.Storage.V1; using Google.Cloud.StorageTransfer.V1; using System; using System.Collections.Generic; using System.IO; +using System.Threading; +using System.Xml.Linq; using Xunit; namespace StorageTransfer.Samples.Tests @@ -32,11 +35,14 @@ public class StorageFixture : IDisposable, ICollectionFixture public string BucketNameSink { get; set; } public string SourceAgentPoolName { get; } public string SinkAgentPoolName { get; } + public StorageClient Client { get; } public StorageClient Storage { get; } = StorageClient.Create(); public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create(); + private readonly List _bucketsToDelete = []; public StorageFixture() { + Client = StorageClient.Create(); ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); SourceAgentPoolName = $"projects/{ProjectId}/agentPools/transfer_service_default"; SinkAgentPoolName = $"projects/{ProjectId}/agentPools/transfer_service_default"; @@ -46,7 +52,7 @@ public StorageFixture() } } - internal void CreateBucketAndGrantStsPermissions(string bucketName) + internal void CreateBucketAndGrantStsPermissions(string bucketName, bool registerForDeletion = true) { var bucket = Storage.CreateBucket(ProjectId, new Bucket { @@ -88,30 +94,44 @@ internal void CreateBucketAndGrantStsPermissions(string bucketName) policy.Bindings.Add(bucketReaderBinding); policy.Bindings.Add(bucketWriterBinding); Storage.SetBucketIamPolicy(bucketName, policy); + SleepAfterBucketCreateDelete(); + if (registerForDeletion) + { + RegisterBucketToDelete(bucketName); + } } internal string GenerateBucketName() => Guid.NewGuid().ToString(); internal string GetCurrentUserTempFolderPath() => System.IO.Path.GetTempPath(); internal string GenerateTempFolderPath() => Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + /// + /// Bucket creation/deletion is rate-limited. To avoid making the tests flaky, we sleep after each operation. + /// + internal static void SleepAfterBucketCreateDelete() => Thread.Sleep(2000); + + internal void RegisterBucketToDelete(string bucket) => _bucketsToDelete.Add(bucket); + public void Dispose() { - try + foreach (var bucket in _bucketsToDelete) { - Storage.DeleteBucket(BucketNameSink, new DeleteBucketOptions { DeleteObjects = true }); - } - catch (Exception) - { - // Do nothing, we delete on a best effort basis. + DeleteBucket(Client, bucket, null); } + } + + private void DeleteBucket(StorageClient client, string bucket, string userProject) + { try { - Storage.DeleteBucket(BucketNameSource, new DeleteBucketOptions { DeleteObjects = true }); + client.DeleteBucket(bucket, new DeleteBucketOptions { UserProject = userProject, DeleteObjects = true }); } - catch (Exception) + catch (GoogleApiException) { - // Do nothing, we delete on a best effort basis. + // Some tests fail to delete buckets due to object retention locks etc. + // They can be cleaned up later. } + SleepAfterBucketCreateDelete(); } } } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index 1b5d8045cfe..f267930f566 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -36,7 +36,6 @@ public TransferBetweenPosixTest(StorageFixture fixture) _tempDestinationDirectory = fixture.GenerateTempFolderPath(); } - // Transfers objects from a POSIX file system to another POSIX file system. [Fact] public void TransferBetweenPosix() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs index b81eeb7689b..88f763d25fc 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs @@ -33,7 +33,6 @@ public TransferFromPosixTest(StorageFixture fixture) _rootDirectory = fixture.GetCurrentUserTempFolderPath(); } - // Transfers objects from a POSIX file system to a Cloud Storage bucket. [Fact] public void TransferFromPosix() { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs index d0eaa9a6f00..b239ad868eb 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -34,9 +34,8 @@ public TransferToNearlineTest(StorageFixture fixture) _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); } - // Transfers objects from a Google Cloud Storage bucket to another bucket with Nearline storage class. [Fact] - public void TestTransferToNearline() + public void TransferToNearline() { TransferToNearlineSample transferToNearlineSample = new TransferToNearlineSample(); var storage = _fixture.Storage; diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index dde52fc6548..58711707643 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -41,7 +41,6 @@ public TransferUsingManifestTest(StorageFixture fixture) UploadObjectToManifestBucket(_bucketNameManifestSource); } - // Transfers objects from a POSIX file system to a sink bucket using a manifest file. [Fact] public void TransferUsingManifest() { @@ -51,7 +50,6 @@ public void TransferUsingManifest() _transferJobName = transferJob.Name; } - // Uploads an object to the manifest bucket. private void UploadObjectToManifestBucket(string bucketName) { byte[] byteArray = System.Text.Encoding.UTF8.GetBytes($@"{Guid.NewGuid()}.jpeg"); diff --git a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs index 04df26cec05..da9a1bdeaaa 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CheckLatestTransferOperationSample.cs @@ -20,23 +20,18 @@ public class CheckLatestTransferOperationSample { /// - /// Sample that checks the latest transfer operation for a given transfer job. + /// Checks the latest transfer operation for a given transfer job. /// - /// Your Google Cloud Project ID. - /// The name of the job to check. + /// The ID of the Google Cloud project. + /// The name of the transfer job. public TransferJob CheckLatestTransferOperation( - // Your Google Cloud Project ID string projectId = "my-project-id", - // The name of the job to check string jobName = "transferJobs/1234567890") { - // Create a Transfer Service client StorageTransferServiceClient storageTransfer = StorageTransferServiceClient.Create(); GetTransferJobRequest getTransferJobRequest = new GetTransferJobRequest { ProjectId = projectId, JobName = jobName }; - // Get Transfer job TransferJob transferJob = storageTransfer.GetTransferJob(getTransferJobRequest); - // Get Latest operation name from transfer job string latestOperationName = transferJob.LatestOperationName; if (!string.IsNullOrEmpty(latestOperationName)) diff --git a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs index 3619408aca0..34c57b3afbf 100644 --- a/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/CreateEventDrivenGcsTransferSample.cs @@ -22,23 +22,18 @@ public class CreateEventDrivenGcsTransferSample { /// - /// Sample that creates an event driven gcs data transfer from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. + /// Creates an event driven gcs data transfer from source gcs bucket to sink gcs bucket subscribed to pub/sub subscription id. /// - /// Your Google Cloud Project ID. + /// The ID of the Google Cloud project. /// The GCS bucket to transfer data from. /// The GCS bucket to transfer data to. - /// The subscription ID to a Pubsub queue to track. + /// The subscription ID to a PubSub queue to track. public TransferJob CreateEventDrivenGcsTransfer( - // Your Google Cloud Project ID string projectId = "my-project-id", - // The GCS bucket to transfer data from string sourceBucket = "my-source-bucket", - // The GCS bucket to transfer data to string sinkBucket = "my-sink-bucket", - // The subscription ID to a Pubsub queue to track string pubSubId = "projects/PROJECT_NAME/subscriptions/SUBSCRIPTION_ID") { - // A useful description for your transfer job string jobDescription = $"Event driven gcs data transfer from {sourceBucket} to {sinkBucket} subscribed to {pubSubId} "; TransferJob transferJob = new TransferJob diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index 31e1937309f..5b554a759e2 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -20,26 +20,20 @@ public class DownloadToPosixSample { /// - /// Sample that creates a transfer for downloading objects from a gcs bucket to the root directory of POSIX file system. + /// Creates a transfer for downloading objects from a gcs bucket to the root directory of POSIX file system. /// - /// Your Google Cloud Project ID. + /// The ID of the Google Cloud project. /// The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent. /// Your GCS source bucket name. /// An optional path on the Google Cloud Storage bucket to download from. /// The root directory path on the source filesystem. public TransferJob DownloadToPosix( - // Your Google Cloud Project ID string projectId = "my-project-id", - // The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent string sinkAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", - // Your GCS source bucket name string gcsSourceBucket = "my-gcs-source-bucket", - // An optional path on the Google Cloud Storage bucket to download from string gcsSourcePath = "foo/bar/", - // The root directory path on the source filesystem string rootDirectory = "/tmp/uploads") { - // A useful description for your transfer job string jobDescription = $"Download objects from a GCS source bucket ({gcsSourceBucket}/{gcsSourcePath}) to the root directory of POSIX file system"; TransferJob transferJob = new TransferJob @@ -55,12 +49,8 @@ public TransferJob DownloadToPosix( Status = TransferJob.Types.Status.Enabled, }; - // Create a Transfer Service client StorageTransferServiceClient client = StorageTransferServiceClient.Create(); - - // Create a Transfer job TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); - client.RunTransferJob(new RunTransferJobRequest { JobName = response.Name, diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs index a2a5bef3d84..276d6177c10 100644 --- a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs @@ -22,17 +22,14 @@ public class QuickstartSample { /// - /// Sample that creates a one-time transfer job from a Google cloud storage bucket to another bucket. + /// Creates an one-time transfer job from a Google Cloud storage bucket to another bucket. /// - /// Your Google Cloud Project ID. + /// The ID of the Google Cloud project. /// The GCS bucket to transfer data from. /// The GCS bucket to transfer data to. public TransferJob Quickstart( - // Your Google Cloud Project ID string projectId = "my-project-id", - // The GCS bucket to transfer data from string sourceBucket = "my-source-bucket", - // The GCS bucket to transfer data to string sinkBucket = "my-sink-bucket") { TransferJob transferJob = new TransferJob diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs index 78973c0d919..b1410a2a92f 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferBetweenPosixSample.cs @@ -20,29 +20,22 @@ public class TransferBetweenPosixSample { /// - /// Sample that creates a transfer that transfer objects from root directory to the destination directory between POSIX file systems. + /// Creates a transfer that transfer objects from root directory to the destination directory between POSIX file systems. /// - /// The ID of the project. + /// The ID of the Google Cloud project. /// The agent pool associated with the POSIX data source. If not provided, defaults to the default agent. /// The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent. /// The root directory path on the source filesystem. /// The root directory path on the sink filesystem. /// The name of GCS bucket for intermediate storage. public TransferJob TransferBetweenPosix( - // Your Google Cloud Project ID string projectId = "my-project-id", - // The agent pool associated with the POSIX data source. If not provided, defaults to the default agent string sourceAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", - // The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent string sinkAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", - // The root directory path on the source filesystem string rootDirectory = "/tmp/uploads", - // The root directory path on the sink filesystem string destinationDirectory = "/directory/to/transfer/sink", - // The name of GCS bucket for intermediate storage string intermediateBucket = "my-intermediate-bucket") { - // A useful description for your transfer job string jobDescription = $"Transfer objects from {rootDirectory} to the {destinationDirectory} between POSIX file system"; TransferJob transferJob = new TransferJob @@ -60,12 +53,8 @@ public TransferJob TransferBetweenPosix( Status = TransferJob.Types.Status.Enabled, }; - // Create a Transfer Service client StorageTransferServiceClient client = StorageTransferServiceClient.Create(); - - // Create a Transfer job TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); - client.RunTransferJob(new RunTransferJobRequest { JobName = response.Name, diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs index 1307acff0b3..8d03df85a69 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferFromPosixSample.cs @@ -20,23 +20,18 @@ public class TransferFromPosixSample { /// - /// Sample that creates a transfer job to transfer objects from a POSIX file system to a gcs sink bucket. + /// Creates a transfer job to transfer objects from a POSIX file system to a gcs sink bucket. /// - /// The ID of the project. + /// The ID of the Google Cloud project. /// The agent pool associated with the POSIX data source. If not provided, defaults to the default agent. /// The root directory path on the source filesystem. /// The GCS bucket to transfer data to. public TransferJob TransferFromPosix( - // Your Google Cloud Project ID string projectId = "my-project-id", - // The agent pool associated with the POSIX data source. If not provided, defaults to the default agent string sourceAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", - // The root directory path on the source filesystem string rootDirectory = "/tmp/uploads", - // The GCS bucket to transfer data to string sinkBucket = "my-sink-bucket") { - // A useful description for your transfer job string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket})"; TransferJob transferJob = new TransferJob @@ -52,12 +47,8 @@ public TransferJob TransferFromPosix( Status = TransferJob.Types.Status.Enabled, }; - // Create a Transfer Service client StorageTransferServiceClient client = StorageTransferServiceClient.Create(); - - // Create a Transfer job TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); - client.RunTransferJob(new RunTransferJobRequest { JobName = response.Name, diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs index deaeca3fe3e..0e32ed5f087 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferToNearlineSample.cs @@ -21,21 +21,17 @@ public class TransferToNearlineSample { /// - /// Sample that creates a one-off transfer job that transfers objects from a standard gcs bucket that are more + /// Creates an one-off transfer job that transfers objects from a standard gcs bucket that are more /// than 30 days old to a nearline gcs bucket. /// - /// Your Google Cloud Project ID. + /// The ID of the Google Cloud project. /// The GCS bucket to transfer objects from. /// The GCS Nearline bucket to transfer old objects to. public TransferJob TransferToNearline( - // Your Google Cloud Project ID string projectId = "my-project-id", - // The GCS bucket to transfer objects from string sourceBucket = "my-source-bucket", - // The GCS Nearline bucket to transfer old objects to string sinkBucket = "my-sink-bucket") { - // A description of this job string jobDescription = $"Transfers old objects from standard bucket ({sourceBucket}) that haven't been modified in the last 30 days to a Nearline bucket ({sinkBucket})"; TransferJob transferJob = new TransferJob @@ -53,10 +49,7 @@ public TransferJob TransferToNearline( Schedule = new Schedule { ScheduleStartDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)), ScheduleEndDate = Google.Type.Date.FromDateTime(System.DateTime.UtcNow.Date.AddMonths(1)) } }; - // Create a Transfer Service client StorageTransferServiceClient client = StorageTransferServiceClient.Create(); - - // Create a Transfer job TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); client.RunTransferJob(new RunTransferJobRequest { diff --git a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs index cf75fb9a2e4..f21fb6d6faa 100644 --- a/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/TransferUsingManifestSample.cs @@ -20,9 +20,9 @@ public class TransferUsingManifestSample { /// - /// Sample that creates a transfer job that transfer objects from a POSIX file system to a gcs sink bucket using a manifest file. + /// Creates a transfer job that transfer objects from a POSIX file system to a gcs sink bucket using a manifest file. /// - /// The ID of the project. + /// The ID of the Google Cloud project. /// The agent pool associated with the POSIX data source. If not provided, /// defaults to the default agent /// The root directory path on the source filesystem. @@ -30,22 +30,15 @@ public class TransferUsingManifestSample /// The GCS bucket to transfer data to. /// The name of the manifest file in manifestBucket that specifies which objects to transfer. public TransferJob TransferUsingManifest( - // Your Google Cloud Project ID string projectId = "my-project-id", - // The agent pool associated with the POSIX data source. If not provided, defaults to the default agent string sourceAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", - // The root directory path on the source filesystem string rootDirectory = "/tmp/uploads", - // The GCS bucket which has your manifest file string manifestBucket = "my-source-bucket", - // The GCS bucket to transfer data to string sinkBucket = "my-sink-bucket", - // The name of the manifest file in manifestBucket that specifies which objects to transfer string manifestObjectName = "path/to/manifest.csv") { string manifestLocation = $"gs://{manifestBucket}/{manifestObjectName}"; - // A useful description for your transfer job string jobDescription = $"Transfers objects from a POSIX file system to a sink bucket ({sinkBucket}) using manifest file"; TransferJob transferJob = new TransferJob @@ -63,12 +56,8 @@ public TransferJob TransferUsingManifest( Status = TransferJob.Types.Status.Enabled, }; - // Create a Transfer Service client StorageTransferServiceClient client = StorageTransferServiceClient.Create(); - - // Create a Transfer job TransferJob response = client.CreateTransferJob(new CreateTransferJobRequest { TransferJob = transferJob }); - client.RunTransferJob(new RunTransferJobRequest { JobName = response.Name, From dece92adce3a1f2f41196adc3b5da12294ad0aac Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 16 Apr 2025 02:07:48 -0700 Subject: [PATCH 43/58] minor changes --- .../api/StorageTransfer.Samples.Tests/StorageFixture.cs | 4 +--- .../TransferBetweenPosixTest.cs | 5 ++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index df85646e2eb..83897fd8628 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -35,14 +35,12 @@ public class StorageFixture : IDisposable, ICollectionFixture public string BucketNameSink { get; set; } public string SourceAgentPoolName { get; } public string SinkAgentPoolName { get; } - public StorageClient Client { get; } public StorageClient Storage { get; } = StorageClient.Create(); public StorageTransferServiceClient Sts { get; } = StorageTransferServiceClient.Create(); private readonly List _bucketsToDelete = []; public StorageFixture() { - Client = StorageClient.Create(); ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); SourceAgentPoolName = $"projects/{ProjectId}/agentPools/transfer_service_default"; SinkAgentPoolName = $"projects/{ProjectId}/agentPools/transfer_service_default"; @@ -116,7 +114,7 @@ public void Dispose() { foreach (var bucket in _bucketsToDelete) { - DeleteBucket(Client, bucket, null); + DeleteBucket(Storage, bucket, null); } } diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index f267930f566..43489ba8223 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -42,15 +42,14 @@ public void TransferBetweenPosix() TransferBetweenPosixSample transferBetweenPosixSample = new TransferBetweenPosixSample(); Directory.CreateDirectory(_tempDirectory); Directory.CreateDirectory(_tempDestinationDirectory); - string fileName = Path.Combine(_tempDirectory, "test.txt"); - // Check if file already exists. If yes, delete it. + string fileName = Path.Combine(_tempDirectory, $"{Guid.NewGuid().ToString()}.txt"); if (File.Exists(fileName)) { File.Delete(fileName); } using (StreamWriter sw = new StreamWriter(fileName)) { - sw.WriteLine("test message"); + sw.WriteLine(Guid.NewGuid().ToString()); } var transferJob = transferBetweenPosixSample.TransferBetweenPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.SinkAgentPoolName, _tempDirectory, _tempDestinationDirectory, _fixture.BucketNameSource); Assert.Contains("transferJobs/", transferJob.Name); From 40d28d14c90ce219b4efb2f38934314836277da8 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 16 Apr 2025 03:15:49 -0700 Subject: [PATCH 44/58] individual bucket creation and deletion test changes --- .../CheckLatestTransferOperationTest.cs | 16 ++++++++++------ .../CreateEventDrivenGcsTransferTest.cs | 14 +++++++++----- .../QuickstartTest.cs | 14 +++++++++----- .../StorageFixture.cs | 2 -- .../TransferBetweenPosixTest.cs | 8 +++++--- .../TransferFromPosixTest.cs | 8 +++++--- .../TransferToNearlineTest.cs | 16 ++++++++++------ .../TransferUsingManifestTest.cs | 18 ++++++++++-------- .../DownloadToPosixSample.cs | 10 +++++----- 9 files changed, 63 insertions(+), 43 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs index b5f3e934a31..adaf2f838da 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CheckLatestTransferOperationTest.cs @@ -24,14 +24,16 @@ public class CheckLatestTransferOperationTest : IDisposable private readonly StorageFixture _fixture; private string _jobName; private readonly string _transferJobName; + private readonly string _sourceBucket; + private readonly string _sinkBucket; public CheckLatestTransferOperationTest(StorageFixture fixture) { _fixture = fixture; - _fixture.BucketNameSource = _fixture.GenerateBucketName(); - _fixture.BucketNameSink = _fixture.GenerateBucketName(); - _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSource); - _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); + _sourceBucket = _fixture.GenerateBucketName(); + _sinkBucket = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_sourceBucket); + _fixture.CreateBucketAndGrantStsPermissions(_sinkBucket); _transferJobName = CreateTransferJob(); } @@ -51,8 +53,8 @@ private string CreateTransferJob() ProjectId = _fixture.ProjectId, TransferSpec = new TransferSpec { - GcsDataSink = new GcsData { BucketName = _fixture.BucketNameSource }, - GcsDataSource = new GcsData { BucketName = _fixture.BucketNameSink } + GcsDataSink = new GcsData { BucketName = _sinkBucket }, + GcsDataSource = new GcsData { BucketName = _sourceBucket } }, Status = TransferJob.Types.Status.Enabled }; @@ -79,6 +81,8 @@ public void Dispose() Status = TransferJob.Types.Status.Deleted } }); + _fixture.Storage.DeleteBucket(_sourceBucket); + _fixture.Storage.DeleteBucket(_sinkBucket); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs index c8687af21d7..8e65a07b7fa 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -27,6 +27,8 @@ public class CreateEventDrivenGcsTransferTest : IDisposable private readonly StorageFixture _fixture; private readonly string _pubSubId; private string _transferJobName; + private readonly string _sourceBucket; + private readonly string _sinkBucket; private string TopicId { get; } = Guid.NewGuid().ToString(); private string SubscriptionId { get; } = Guid.NewGuid().ToString(); private SubscriberServiceApiClient SubscriberClient { get; } = SubscriberServiceApiClient.Create(); @@ -35,10 +37,10 @@ public class CreateEventDrivenGcsTransferTest : IDisposable public CreateEventDrivenGcsTransferTest(StorageFixture fixture) { _fixture = fixture; - _fixture.BucketNameSource = _fixture.GenerateBucketName(); - _fixture.BucketNameSink = _fixture.GenerateBucketName(); - _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSource); - _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); + _sourceBucket = _fixture.GenerateBucketName(); + _sinkBucket = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_sourceBucket); + _fixture.CreateBucketAndGrantStsPermissions(_sinkBucket); _pubSubId = $"projects/{_fixture.ProjectId}/subscriptions/{SubscriptionId}"; CreatePubSubResourcesAndGrantStsPermissions(); } @@ -47,7 +49,7 @@ public CreateEventDrivenGcsTransferTest(StorageFixture fixture) public void CreateEventDrivenGcsTransfer() { CreateEventDrivenGcsTransferSample createEventDrivenGcsTransferSample = new CreateEventDrivenGcsTransferSample(); - var transferJob = createEventDrivenGcsTransferSample.CreateEventDrivenGcsTransfer(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink, _pubSubId); + var transferJob = createEventDrivenGcsTransferSample.CreateEventDrivenGcsTransfer(_fixture.ProjectId, _sourceBucket, _sinkBucket, _pubSubId); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } @@ -101,6 +103,8 @@ public void Dispose() PublisherClient.DeleteTopic(topicName); SubscriptionName subscriptionName = SubscriptionName.FromProjectSubscription(_fixture.ProjectId, SubscriptionId); SubscriberClient.DeleteSubscription(subscriptionName); + _fixture.Storage.DeleteBucket(_sourceBucket); + _fixture.Storage.DeleteBucket(_sinkBucket); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs index 9efbb3334a1..7f32bd64055 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs @@ -25,21 +25,23 @@ public class QuickstartTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; + private readonly string _sourceBucket; + private readonly string _sinkBucket; public QuickstartTest(StorageFixture fixture) { _fixture = fixture; - _fixture.BucketNameSource = _fixture.GenerateBucketName(); - _fixture.BucketNameSink = _fixture.GenerateBucketName(); - _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSource); - _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); + _sourceBucket = _fixture.GenerateBucketName(); + _sinkBucket = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_sourceBucket); + _fixture.CreateBucketAndGrantStsPermissions(_sinkBucket); } [Fact] public void TestQuickstart() { QuickstartSample quickstartSample = new QuickstartSample(); - var transferJob = quickstartSample.Quickstart(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink); + var transferJob = quickstartSample.Quickstart(_fixture.ProjectId, _sourceBucket, _sinkBucket); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } @@ -58,6 +60,8 @@ public void Dispose() Status = TransferJob.Types.Status.Deleted } }); + _fixture.Storage.DeleteBucket(_sourceBucket); + _fixture.Storage.DeleteBucket(_sinkBucket); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs index 83897fd8628..7c3e0588386 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageFixture.cs @@ -31,8 +31,6 @@ namespace StorageTransfer.Samples.Tests public class StorageFixture : IDisposable, ICollectionFixture { public string ProjectId { get; } - public string BucketNameSource { get; set; } - public string BucketNameSink { get; set; } public string SourceAgentPoolName { get; } public string SinkAgentPoolName { get; } public StorageClient Storage { get; } = StorageClient.Create(); diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs index 43489ba8223..42f39d043be 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferBetweenPosixTest.cs @@ -26,12 +26,13 @@ public class TransferBetweenPosixTest : IDisposable private string _transferJobName; private readonly string _tempDirectory; private readonly string _tempDestinationDirectory; + private readonly string _bucket; public TransferBetweenPosixTest(StorageFixture fixture) { _fixture = fixture; - _fixture.BucketNameSource = _fixture.GenerateBucketName(); - _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSource); + _bucket = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_bucket); _tempDirectory = fixture.GenerateTempFolderPath(); _tempDestinationDirectory = fixture.GenerateTempFolderPath(); } @@ -51,7 +52,7 @@ public void TransferBetweenPosix() { sw.WriteLine(Guid.NewGuid().ToString()); } - var transferJob = transferBetweenPosixSample.TransferBetweenPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.SinkAgentPoolName, _tempDirectory, _tempDestinationDirectory, _fixture.BucketNameSource); + var transferJob = transferBetweenPosixSample.TransferBetweenPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _fixture.SinkAgentPoolName, _tempDirectory, _tempDestinationDirectory, _bucket); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; Assert.True(Directory.Exists(_tempDirectory)); @@ -75,6 +76,7 @@ public void Dispose() }); Directory.Delete(_tempDirectory, true); Directory.Delete(_tempDestinationDirectory, true); + _fixture.Storage.DeleteBucket(_bucket); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs index 88f763d25fc..3f72ac58dc1 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferFromPosixTest.cs @@ -24,12 +24,13 @@ public class TransferFromPosixTest : IDisposable private readonly StorageFixture _fixture; private string _transferJobName; private readonly string _rootDirectory; + private readonly string _bucket; public TransferFromPosixTest(StorageFixture fixture) { _fixture = fixture; - _fixture.BucketNameSink = _fixture.GenerateBucketName(); - _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); + _bucket = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_bucket); _rootDirectory = fixture.GetCurrentUserTempFolderPath(); } @@ -37,7 +38,7 @@ public TransferFromPosixTest(StorageFixture fixture) public void TransferFromPosix() { TransferFromPosixSample transferFromPosixSample = new TransferFromPosixSample(); - var transferJob = transferFromPosixSample.TransferFromPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _rootDirectory, _fixture.BucketNameSink); + var transferJob = transferFromPosixSample.TransferFromPosix(_fixture.ProjectId, _fixture.SourceAgentPoolName, _rootDirectory, _bucket); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } @@ -56,6 +57,7 @@ public void Dispose() Status = TransferJob.Types.Status.Deleted } }); + _fixture.Storage.DeleteBucket(_bucket); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs index b239ad868eb..2ceddc570b3 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferToNearlineTest.cs @@ -24,14 +24,16 @@ public class TransferToNearlineTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; + private readonly string _sourceBucket; + private readonly string _sinkBucket; public TransferToNearlineTest(StorageFixture fixture) { _fixture = fixture; - _fixture.BucketNameSource = _fixture.GenerateBucketName(); - _fixture.BucketNameSink = _fixture.GenerateBucketName(); - _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSource); - _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); + _sourceBucket = _fixture.GenerateBucketName(); + _sinkBucket = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_sourceBucket); + _fixture.CreateBucketAndGrantStsPermissions(_sinkBucket); } [Fact] @@ -39,11 +41,11 @@ public void TransferToNearline() { TransferToNearlineSample transferToNearlineSample = new TransferToNearlineSample(); var storage = _fixture.Storage; - var bucket = storage.GetBucket(_fixture.BucketNameSink); + var bucket = storage.GetBucket(_sinkBucket); string storageClass = StorageClasses.Nearline; bucket.StorageClass = storageClass; storage.UpdateBucket(bucket); - var transferJob = transferToNearlineSample.TransferToNearline(_fixture.ProjectId, _fixture.BucketNameSource, _fixture.BucketNameSink); + var transferJob = transferToNearlineSample.TransferToNearline(_fixture.ProjectId, _sourceBucket, _sinkBucket); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } @@ -62,6 +64,8 @@ public void Dispose() Status = TransferJob.Types.Status.Deleted } }); + _fixture.Storage.DeleteBucket(_sourceBucket); + _fixture.Storage.DeleteBucket(_sinkBucket); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs index 58711707643..811acf71385 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/TransferUsingManifestTest.cs @@ -27,25 +27,26 @@ public class TransferUsingManifestTest : IDisposable private string _transferJobName; private readonly string _rootDirectory; private readonly string _manifestObjectName; - private readonly string _bucketNameManifestSource; + private readonly string _manifestBucket; + private readonly string _sinkBucket; public TransferUsingManifestTest(StorageFixture fixture) { _fixture = fixture; - _bucketNameManifestSource = _fixture.GenerateBucketName(); - _fixture.BucketNameSink = _fixture.GenerateBucketName(); - _fixture.CreateBucketAndGrantStsPermissions(_bucketNameManifestSource); - _fixture.CreateBucketAndGrantStsPermissions(_fixture.BucketNameSink); + _manifestBucket = _fixture.GenerateBucketName(); + _sinkBucket = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_manifestBucket); + _fixture.CreateBucketAndGrantStsPermissions(_sinkBucket); _rootDirectory = fixture.GetCurrentUserTempFolderPath(); _manifestObjectName = $@"{Guid.NewGuid()}.csv"; - UploadObjectToManifestBucket(_bucketNameManifestSource); + UploadObjectToManifestBucket(_manifestBucket); } [Fact] public void TransferUsingManifest() { TransferUsingManifestSample transferUsingManifestSample = new TransferUsingManifestSample(); - var transferJob = transferUsingManifestSample.TransferUsingManifest(_fixture.ProjectId, _fixture.SourceAgentPoolName, _rootDirectory, _bucketNameManifestSource, _fixture.BucketNameSink, _manifestObjectName); + var transferJob = transferUsingManifestSample.TransferUsingManifest(_fixture.ProjectId, _fixture.SourceAgentPoolName, _rootDirectory, _manifestBucket, _sinkBucket, _manifestObjectName); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } @@ -71,7 +72,8 @@ public void Dispose() Status = TransferJob.Types.Status.Deleted } }); - _fixture.Storage.DeleteBucket(_bucketNameManifestSource, new DeleteBucketOptions { DeleteObjects = true }); + _fixture.Storage.DeleteBucket(_manifestBucket, new DeleteBucketOptions { DeleteObjects = true }); + _fixture.Storage.DeleteBucket(_sinkBucket, new DeleteBucketOptions { DeleteObjects = true }); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs index 5b554a759e2..e0ddd6fa376 100644 --- a/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/DownloadToPosixSample.cs @@ -24,17 +24,17 @@ public class DownloadToPosixSample /// /// The ID of the Google Cloud project. /// The agent pool associated with the POSIX data sink. If not provided, defaults to the default agent. - /// Your GCS source bucket name. + /// Your GCS source bucket name. /// An optional path on the Google Cloud Storage bucket to download from. /// The root directory path on the source filesystem. public TransferJob DownloadToPosix( string projectId = "my-project-id", string sinkAgentPoolName = "projects/my-project-id/agentPools/transfer_service_default", - string gcsSourceBucket = "my-gcs-source-bucket", + string sourceBucket = "my-source-bucket", string gcsSourcePath = "foo/bar/", string rootDirectory = "/tmp/uploads") { - string jobDescription = $"Download objects from a GCS source bucket ({gcsSourceBucket}/{gcsSourcePath}) to the root directory of POSIX file system"; + string jobDescription = $"Download objects from a GCS source bucket ({sourceBucket}/{gcsSourcePath}) to the root directory of POSIX file system"; TransferJob transferJob = new TransferJob { @@ -42,7 +42,7 @@ public TransferJob DownloadToPosix( Description = jobDescription, TransferSpec = new TransferSpec { - GcsDataSource = new GcsData { BucketName = gcsSourceBucket, Path = gcsSourcePath }, + GcsDataSource = new GcsData { BucketName = sourceBucket, Path = gcsSourcePath }, SinkAgentPoolName = sinkAgentPoolName, PosixDataSink = new PosixFilesystem { RootDirectory = rootDirectory } }, @@ -57,7 +57,7 @@ public TransferJob DownloadToPosix( ProjectId = projectId }); - Console.WriteLine($"Created and ran transfer job from ({gcsSourceBucket}/{gcsSourcePath}) to {rootDirectory} with the name {response.Name}"); + Console.WriteLine($"Created and ran transfer job from ({sourceBucket}/{gcsSourcePath}) to {rootDirectory} with the name {response.Name}"); return response; } } From c2b42b8ce53c43c45cecb5135ea40ecfbd200576 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 16 Apr 2025 04:50:44 -0700 Subject: [PATCH 45/58] QuickStart changes --- .../DownloadToPosixTest.cs | 14 +++++++------- .../QuickstartTest.cs | 10 +++++----- .../StorageTransfer.Samples/QuickstartSample.cs | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs index ac9116f3abc..0ee2a072e2f 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/DownloadToPosixTest.cs @@ -27,16 +27,16 @@ public class DownloadToPosixTest : IDisposable private string _transferJobName; private readonly string _tempDirectory; private readonly string _gcsSourcePath; - private readonly string _bucketNamePosixSource; + private readonly string _sourceBucket; public DownloadToPosixTest(StorageFixture fixture) { _fixture = fixture; - _bucketNamePosixSource = _fixture.GenerateBucketName(); - _fixture.CreateBucketAndGrantStsPermissions(_bucketNamePosixSource); + _sourceBucket = _fixture.GenerateBucketName(); + _fixture.CreateBucketAndGrantStsPermissions(_sourceBucket); _tempDirectory = _fixture.GenerateTempFolderPath(); _gcsSourcePath = $"{Guid.NewGuid()}/{Guid.NewGuid()}/"; - UploadObjectToPosixBucket(_bucketNamePosixSource); + UploadObjectToBucket(_sourceBucket); } [Fact] @@ -44,13 +44,13 @@ public void DownloadToPosix() { DownloadToPosixSample downloadToPosixSample = new DownloadToPosixSample(); Directory.CreateDirectory(_tempDirectory); - var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId, _fixture.SinkAgentPoolName, _bucketNamePosixSource, _gcsSourcePath, _tempDirectory); + var transferJob = downloadToPosixSample.DownloadToPosix(_fixture.ProjectId, _fixture.SinkAgentPoolName, _sourceBucket, _gcsSourcePath, _tempDirectory); Assert.Contains("transferJobs/", transferJob.Name); Assert.True(Directory.Exists(_tempDirectory)); _transferJobName = transferJob.Name; } - private void UploadObjectToPosixBucket(string bucketName) + private void UploadObjectToBucket(string bucketName) { byte[] byteArray = System.Text.Encoding.UTF8.GetBytes($@"{Guid.NewGuid()}.jpeg"); MemoryStream stream = new MemoryStream(byteArray); @@ -73,7 +73,7 @@ public void Dispose() } }); Directory.Delete(_tempDirectory, true); - _fixture.Storage.DeleteBucket(_bucketNamePosixSource, new DeleteBucketOptions { DeleteObjects = true }); + _fixture.Storage.DeleteBucket(_sourceBucket, new DeleteBucketOptions { DeleteObjects = true }); } catch (Exception) { diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs index 7f32bd64055..53048c0d578 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs @@ -21,14 +21,14 @@ namespace StorageTransfer.Samples.Tests { [Collection(nameof(StorageFixture))] - public class QuickstartTest : IDisposable + public class QuickStartTest : IDisposable { private readonly StorageFixture _fixture; private string _transferJobName; private readonly string _sourceBucket; private readonly string _sinkBucket; - public QuickstartTest(StorageFixture fixture) + public QuickStartTest(StorageFixture fixture) { _fixture = fixture; _sourceBucket = _fixture.GenerateBucketName(); @@ -38,10 +38,10 @@ public QuickstartTest(StorageFixture fixture) } [Fact] - public void TestQuickstart() + public void QuickStart() { - QuickstartSample quickstartSample = new QuickstartSample(); - var transferJob = quickstartSample.Quickstart(_fixture.ProjectId, _sourceBucket, _sinkBucket); + QuickStartSample quickStartSample = new QuickStartSample(); + var transferJob = quickStartSample.QuickStart(_fixture.ProjectId, _sourceBucket, _sinkBucket); Assert.Contains("transferJobs/", transferJob.Name); _transferJobName = transferJob.Name; } diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs index 276d6177c10..115e71231da 100644 --- a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs +++ b/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs @@ -19,7 +19,7 @@ using Google.Cloud.StorageTransfer.V1; using System; -public class QuickstartSample +public class QuickStartSample { /// /// Creates an one-time transfer job from a Google Cloud storage bucket to another bucket. @@ -27,7 +27,7 @@ public class QuickstartSample /// The ID of the Google Cloud project. /// The GCS bucket to transfer data from. /// The GCS bucket to transfer data to. - public TransferJob Quickstart( + public TransferJob QuickStart( string projectId = "my-project-id", string sourceBucket = "my-source-bucket", string sinkBucket = "my-sink-bucket") From 5e9cca3270961487df5e2a2c7edae03049b43d41 Mon Sep 17 00:00:00 2001 From: mahendra-google Date: Wed, 16 Apr 2025 05:07:00 -0700 Subject: [PATCH 46/58] event driven transfer code changes --- storage/api/.editorconfig | 281 ++++++++++++++++++ storagetransfer/api/.editorconfig | 281 ++++++++++++++++++ .../CreateEventDrivenGcsTransferTest.cs | 4 +- 3 files changed, 564 insertions(+), 2 deletions(-) create mode 100644 storage/api/.editorconfig create mode 100644 storagetransfer/api/.editorconfig diff --git a/storage/api/.editorconfig b/storage/api/.editorconfig new file mode 100644 index 00000000000..a0b75fc4dff --- /dev/null +++ b/storage/api/.editorconfig @@ -0,0 +1,281 @@ +# To learn more about .editorconfig see https://aka.ms/editorconfigdocs +# top-most EditorConfig file +root = true +############################### +# Core EditorConfig Options # +############################### +# All files +[*] +insert_final_newline = true +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true + +# Generated code +[*{_AssemblyInfo.cs,.g.cs}] +generated_code = true + +# XML project files +[*.{csproj}] +indent_size = 2 + +# Code files + +############################### +# .NET Coding Conventions # +############################### +[*.{cs}] + +# Organize usings +dotnet_sort_system_directives_first = false + +# this. preferences +dotnet_style_qualification_for_field = false:suggestion +dotnet_style_qualification_for_property = false:suggestion +dotnet_style_qualification_for_method = false:suggestion +dotnet_style_qualification_for_event = false:suggestion + +# Language keywords vs BCL types preferences +dotnet_style_predefined_type_for_locals_parameters_members = true:suggestion +dotnet_style_predefined_type_for_member_access = true:suggestion + +# Parentheses preferences +dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_other_operators = never_if_unnecessary:silent + +# Modifier preferences +csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:warning +dotnet_style_require_accessibility_modifiers = for_non_interface_members:warning +dotnet_style_readonly_field = true:warning + +# Expression-level preferences +dotnet_style_object_initializer = true:suggestion +dotnet_style_collection_initializer = true:suggestion +dotnet_style_explicit_tuple_names = true:suggestion +dotnet_style_null_propagation = true:suggestion +dotnet_style_coalesce_expression = true:suggestion +dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion +dotnet_style_prefer_inferred_tuple_names = true:suggestion +dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion +dotnet_style_prefer_auto_properties = true:suggestion +dotnet_style_prefer_conditional_expression_over_assignment = true:silent +dotnet_style_prefer_conditional_expression_over_return = true:silent +csharp_prefer_simple_default_expression = true:suggestion + +############################### +# Naming Conventions # +############################### +# Style Definitions +dotnet_naming_style.pascal_case_style.capitalization = pascal_case + +# Use PascalCase for constant fields +dotnet_naming_rule.constant_fields_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.constant_fields_should_be_pascal_case.symbols = constant_fields +dotnet_naming_rule.constant_fields_should_be_pascal_case.style = pascal_case_style +dotnet_naming_symbols.constant_fields.applicable_kinds = field +dotnet_naming_symbols.constant_fields.applicable_accessibilities = * +dotnet_naming_symbols.constant_fields.required_modifiers = const +dotnet_naming_style.pascal_case_style.capitalization = pascal_case + +# static fields should have s_ prefix +dotnet_naming_rule.static_fields_should_have_prefix.severity = suggestion +dotnet_naming_rule.static_fields_should_have_prefix.symbols = static_fields +dotnet_naming_rule.static_fields_should_have_prefix.style = static_prefix_style +dotnet_naming_symbols.static_fields.applicable_kinds = field +dotnet_naming_symbols.static_fields.required_modifiers = static +dotnet_naming_symbols.static_fields.applicable_accessibilities = private, internal, private_protected +dotnet_naming_style.static_prefix_style.required_prefix = s_ +dotnet_naming_style.static_prefix_style.capitalization = camel_case + +# internal and private fields should be _camelCase +dotnet_naming_rule.camel_case_for_private_internal_fields.severity = suggestion +dotnet_naming_rule.camel_case_for_private_internal_fields.symbols = private_internal_fields +dotnet_naming_rule.camel_case_for_private_internal_fields.style = camel_case_underscore_style +dotnet_naming_symbols.private_internal_fields.applicable_kinds = field +dotnet_naming_symbols.private_internal_fields.applicable_accessibilities = private, internal +dotnet_naming_style.camel_case_underscore_style.required_prefix = _ +dotnet_naming_style.camel_case_underscore_style.capitalization = camel_case + +# Locals and parameters are camelCase +dotnet_naming_rule.locals_should_be_camel_case.severity = suggestion +dotnet_naming_rule.locals_should_be_camel_case.symbols = locals_and_parameters +dotnet_naming_rule.locals_should_be_camel_case.style = camel_case_style +dotnet_naming_symbols.locals_and_parameters.applicable_kinds = parameter, local +dotnet_naming_style.camel_case_style.capitalization = camel_case + +# Public property and methods should be PascalCase +dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.symbols = public_methods_and_properties +dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.style = pascal_case +dotnet_naming_symbols.public_methods_and_properties.applicable_kinds = method, property +dotnet_naming_symbols.public_methods_and_properties.applicable_accessibilities = public +dotnet_naming_style.pascal_case.capitalization = pascal_case + +# Private and internal methods should be PascalCase +dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.symbols = private_and_internal_methods +dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.style = pascal_case +dotnet_naming_symbols.private_and_internal_methods.applicable_kinds = method +dotnet_naming_symbols.private_and_internal_methods.applicable_accessibilities = private, internal +############################### +# C# Coding Conventions # +############################### +[*.cs] +# var preferences +csharp_style_var_for_built_in_types = true:silent +csharp_style_var_when_type_is_apparent = true:silent +csharp_style_var_elsewhere = true:silent +dotnet_style_predefined_type_for_locals_parameters_members = true:silent +dotnet_style_predefined_type_for_member_access = true:silent + +csharp_using_directive_placement = outside_namespace:warning +csharp_prefer_braces = true:suggestion +csharp_preserve_single_line_blocks = true:none +csharp_preserve_single_line_statements = false:none +csharp_prefer_static_local_function = true:suggestion +csharp_prefer_simple_using_statement = true:suggestion +csharp_style_prefer_switch_expression = true:suggestion + +# Expression-bodied members +csharp_style_expression_bodied_methods = true:suggestion +csharp_style_expression_bodied_constructors = true:suggestion +csharp_style_expression_bodied_operators = true:suggestion +csharp_style_expression_bodied_properties = true:suggestion +csharp_style_expression_bodied_indexers = true:suggestion +csharp_style_expression_bodied_accessors = true:suggestion +csharp_style_expression_bodied_lambdas = true:suggestion +csharp_style_expression_bodied_local_functions = true:suggestion + +# Pattern matching preferences +csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion +csharp_style_pattern_matching_over_as_with_null_check = true:suggestion +csharp_style_inlined_variable_declaration = true:suggestion + +# Null-checking preferences +csharp_style_throw_expression = true:suggestion +csharp_style_conditional_delegate_call = true:suggestion + +# Other features +csharp_style_prefer_index_operator = true:suggestion +csharp_style_prefer_range_operator = true:suggestion +csharp_style_pattern_local_over_anonymous_function = false:none + +############################### +# C# Formatting Rules # +############################### +indent_size = 4 +insert_final_newline = true +charset = utf-8 + +# New line preferences +csharp_new_line_before_open_brace = all +csharp_new_line_before_else = true +csharp_new_line_before_catch = true +csharp_new_line_before_finally = true +csharp_new_line_before_members_in_object_initializers = true +csharp_new_line_before_members_in_anonymous_types = true +csharp_new_line_between_query_expression_clauses = true + +# Indentation preferences +csharp_indent_block_contents = true +csharp_indent_braces = false +csharp_indent_case_contents = true +csharp_indent_case_contents_when_block = true +csharp_indent_switch_labels = true +csharp_indent_labels = one_less_than_current + +# Space preferences +csharp_space_after_cast = true +csharp_space_after_colon_in_inheritance_clause = true +csharp_space_after_comma = true +csharp_space_after_dot = false +csharp_space_after_keywords_in_control_flow_statements = true +csharp_space_after_semicolon_in_for_statement = true +csharp_space_between_method_call_parameter_list_parentheses = false +csharp_space_between_method_declaration_parameter_list_parentheses = false +csharp_space_between_parentheses = false +csharp_space_before_comma = false +csharp_space_before_dot = false +csharp_space_before_open_square_brackets = false +csharp_space_before_semicolon_in_for_statement = false +csharp_space_between_empty_square_brackets = false +csharp_space_around_declaration_statements = do_not_ignore +csharp_space_before_colon_in_inheritance_clause = true +csharp_space_after_colon_in_inheritance_clause = true +csharp_space_around_binary_operators = before_and_after +csharp_space_between_method_declaration_empty_parameter_list_parentheses = false +csharp_space_between_method_call_name_and_opening_parenthesis = false +csharp_space_between_method_call_empty_parameter_list_parentheses = false +csharp_space_between_method_declaration_name_and_open_parenthesis = false +csharp_space_between_square_brackets = false + +# Wrapping preferences +csharp_preserve_single_line_statements = true +csharp_preserve_single_line_blocks = true + +# License header +file_header_template = Copyright 2024 Google LLC\n\nLicensed under the Apache License, Version 2.0 (the "License").\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at \n\nhttps://www.apache.org/licenses/LICENSE-2.0 \n\nUnless required by applicable law or agreed to in writing, software \ndistributed under the License is distributed on an "AS IS" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and \nlimitations under the License. +csharp_style_namespace_declarations = file_scoped:suggestion +csharp_style_prefer_null_check_over_type_check = true:suggestion +csharp_style_prefer_local_over_anonymous_function = true:suggestion +csharp_style_prefer_tuple_swap = true:suggestion +csharp_style_implicit_object_creation_when_type_is_apparent = true:suggestion +csharp_style_unused_value_assignment_preference = discard_variable:suggestion +csharp_style_unused_value_expression_statement_preference = discard_variable:suggestion +csharp_style_prefer_pattern_matching = true:suggestion +csharp_style_prefer_not_pattern = true:suggestion +csharp_style_prefer_extended_property_pattern = true:suggestion +csharp_style_prefer_parameter_null_checking = true:suggestion +csharp_style_prefer_method_group_conversion = true:suggestion + +# Change the severity to `error` for the rules that must not be violated, to fail the build. + +# IDE0005: Remove unnecessary using directives. +dotnet_diagnostic.IDE0005.severity = suggestion + +# IDE0011: Add braces +dotnet_diagnostic.IDE0011.severity = suggestion + +# IDE0017: Object initialization can be simplified. +dotnet_diagnostic.IDE0017.severity = suggestion + +# IDE0019: Use pattern matching. +dotnet_diagnostic.IDE0019.severity = suggestion + +# IDE0021: Use expression body for constructor. +dotnet_diagnostic.IDE0021.severity = suggestion + +# IDE0028: Collection initialization can be simplified. +dotnet_diagnostic.IDE0028.severity = suggestion + +# IDE0035: Remove unreachable code. +dotnet_diagnostic.IDE0035.severity = suggestion + +# IDE0040: Accessibility modifiers required. +dotnet_diagnostic.IDE0040.severity = suggestion + +# IDE0052: Remove unread private member +dotnet_diagnostic.IDE0052.severity = suggestion + +# IDE0053: Use expression body for lambda expression. +dotnet_diagnostic.IDE0053.severity = suggestion + +# IDE0055: All C# and .NET formatting rules. +dotnet_diagnostic.IDE0055.severity = suggestion + +# IDE0060: Remove unused parameter +dotnet_diagnostic.IDE0060.severity = suggestion + +# IDE0063: Using statement can be simplified +dotnet_diagnostic.IDE0063.severity = suggestion + +# IDE0065: Using statement can be simplified +dotnet_diagnostic.IDE0065.severity = suggestion + +# IDE0071: Interpolation can be simplified. +dotnet_diagnostic.IDE0071.severity = suggestion + +# IDE0075: Simplify conditional expression. +dotnet_diagnostic.IDE0075.severity = suggestion diff --git a/storagetransfer/api/.editorconfig b/storagetransfer/api/.editorconfig new file mode 100644 index 00000000000..a0b75fc4dff --- /dev/null +++ b/storagetransfer/api/.editorconfig @@ -0,0 +1,281 @@ +# To learn more about .editorconfig see https://aka.ms/editorconfigdocs +# top-most EditorConfig file +root = true +############################### +# Core EditorConfig Options # +############################### +# All files +[*] +insert_final_newline = true +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true + +# Generated code +[*{_AssemblyInfo.cs,.g.cs}] +generated_code = true + +# XML project files +[*.{csproj}] +indent_size = 2 + +# Code files + +############################### +# .NET Coding Conventions # +############################### +[*.{cs}] + +# Organize usings +dotnet_sort_system_directives_first = false + +# this. preferences +dotnet_style_qualification_for_field = false:suggestion +dotnet_style_qualification_for_property = false:suggestion +dotnet_style_qualification_for_method = false:suggestion +dotnet_style_qualification_for_event = false:suggestion + +# Language keywords vs BCL types preferences +dotnet_style_predefined_type_for_locals_parameters_members = true:suggestion +dotnet_style_predefined_type_for_member_access = true:suggestion + +# Parentheses preferences +dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:silent +dotnet_style_parentheses_in_other_operators = never_if_unnecessary:silent + +# Modifier preferences +csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:warning +dotnet_style_require_accessibility_modifiers = for_non_interface_members:warning +dotnet_style_readonly_field = true:warning + +# Expression-level preferences +dotnet_style_object_initializer = true:suggestion +dotnet_style_collection_initializer = true:suggestion +dotnet_style_explicit_tuple_names = true:suggestion +dotnet_style_null_propagation = true:suggestion +dotnet_style_coalesce_expression = true:suggestion +dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion +dotnet_style_prefer_inferred_tuple_names = true:suggestion +dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion +dotnet_style_prefer_auto_properties = true:suggestion +dotnet_style_prefer_conditional_expression_over_assignment = true:silent +dotnet_style_prefer_conditional_expression_over_return = true:silent +csharp_prefer_simple_default_expression = true:suggestion + +############################### +# Naming Conventions # +############################### +# Style Definitions +dotnet_naming_style.pascal_case_style.capitalization = pascal_case + +# Use PascalCase for constant fields +dotnet_naming_rule.constant_fields_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.constant_fields_should_be_pascal_case.symbols = constant_fields +dotnet_naming_rule.constant_fields_should_be_pascal_case.style = pascal_case_style +dotnet_naming_symbols.constant_fields.applicable_kinds = field +dotnet_naming_symbols.constant_fields.applicable_accessibilities = * +dotnet_naming_symbols.constant_fields.required_modifiers = const +dotnet_naming_style.pascal_case_style.capitalization = pascal_case + +# static fields should have s_ prefix +dotnet_naming_rule.static_fields_should_have_prefix.severity = suggestion +dotnet_naming_rule.static_fields_should_have_prefix.symbols = static_fields +dotnet_naming_rule.static_fields_should_have_prefix.style = static_prefix_style +dotnet_naming_symbols.static_fields.applicable_kinds = field +dotnet_naming_symbols.static_fields.required_modifiers = static +dotnet_naming_symbols.static_fields.applicable_accessibilities = private, internal, private_protected +dotnet_naming_style.static_prefix_style.required_prefix = s_ +dotnet_naming_style.static_prefix_style.capitalization = camel_case + +# internal and private fields should be _camelCase +dotnet_naming_rule.camel_case_for_private_internal_fields.severity = suggestion +dotnet_naming_rule.camel_case_for_private_internal_fields.symbols = private_internal_fields +dotnet_naming_rule.camel_case_for_private_internal_fields.style = camel_case_underscore_style +dotnet_naming_symbols.private_internal_fields.applicable_kinds = field +dotnet_naming_symbols.private_internal_fields.applicable_accessibilities = private, internal +dotnet_naming_style.camel_case_underscore_style.required_prefix = _ +dotnet_naming_style.camel_case_underscore_style.capitalization = camel_case + +# Locals and parameters are camelCase +dotnet_naming_rule.locals_should_be_camel_case.severity = suggestion +dotnet_naming_rule.locals_should_be_camel_case.symbols = locals_and_parameters +dotnet_naming_rule.locals_should_be_camel_case.style = camel_case_style +dotnet_naming_symbols.locals_and_parameters.applicable_kinds = parameter, local +dotnet_naming_style.camel_case_style.capitalization = camel_case + +# Public property and methods should be PascalCase +dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.symbols = public_methods_and_properties +dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.style = pascal_case +dotnet_naming_symbols.public_methods_and_properties.applicable_kinds = method, property +dotnet_naming_symbols.public_methods_and_properties.applicable_accessibilities = public +dotnet_naming_style.pascal_case.capitalization = pascal_case + +# Private and internal methods should be PascalCase +dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.severity = suggestion +dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.symbols = private_and_internal_methods +dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.style = pascal_case +dotnet_naming_symbols.private_and_internal_methods.applicable_kinds = method +dotnet_naming_symbols.private_and_internal_methods.applicable_accessibilities = private, internal +############################### +# C# Coding Conventions # +############################### +[*.cs] +# var preferences +csharp_style_var_for_built_in_types = true:silent +csharp_style_var_when_type_is_apparent = true:silent +csharp_style_var_elsewhere = true:silent +dotnet_style_predefined_type_for_locals_parameters_members = true:silent +dotnet_style_predefined_type_for_member_access = true:silent + +csharp_using_directive_placement = outside_namespace:warning +csharp_prefer_braces = true:suggestion +csharp_preserve_single_line_blocks = true:none +csharp_preserve_single_line_statements = false:none +csharp_prefer_static_local_function = true:suggestion +csharp_prefer_simple_using_statement = true:suggestion +csharp_style_prefer_switch_expression = true:suggestion + +# Expression-bodied members +csharp_style_expression_bodied_methods = true:suggestion +csharp_style_expression_bodied_constructors = true:suggestion +csharp_style_expression_bodied_operators = true:suggestion +csharp_style_expression_bodied_properties = true:suggestion +csharp_style_expression_bodied_indexers = true:suggestion +csharp_style_expression_bodied_accessors = true:suggestion +csharp_style_expression_bodied_lambdas = true:suggestion +csharp_style_expression_bodied_local_functions = true:suggestion + +# Pattern matching preferences +csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion +csharp_style_pattern_matching_over_as_with_null_check = true:suggestion +csharp_style_inlined_variable_declaration = true:suggestion + +# Null-checking preferences +csharp_style_throw_expression = true:suggestion +csharp_style_conditional_delegate_call = true:suggestion + +# Other features +csharp_style_prefer_index_operator = true:suggestion +csharp_style_prefer_range_operator = true:suggestion +csharp_style_pattern_local_over_anonymous_function = false:none + +############################### +# C# Formatting Rules # +############################### +indent_size = 4 +insert_final_newline = true +charset = utf-8 + +# New line preferences +csharp_new_line_before_open_brace = all +csharp_new_line_before_else = true +csharp_new_line_before_catch = true +csharp_new_line_before_finally = true +csharp_new_line_before_members_in_object_initializers = true +csharp_new_line_before_members_in_anonymous_types = true +csharp_new_line_between_query_expression_clauses = true + +# Indentation preferences +csharp_indent_block_contents = true +csharp_indent_braces = false +csharp_indent_case_contents = true +csharp_indent_case_contents_when_block = true +csharp_indent_switch_labels = true +csharp_indent_labels = one_less_than_current + +# Space preferences +csharp_space_after_cast = true +csharp_space_after_colon_in_inheritance_clause = true +csharp_space_after_comma = true +csharp_space_after_dot = false +csharp_space_after_keywords_in_control_flow_statements = true +csharp_space_after_semicolon_in_for_statement = true +csharp_space_between_method_call_parameter_list_parentheses = false +csharp_space_between_method_declaration_parameter_list_parentheses = false +csharp_space_between_parentheses = false +csharp_space_before_comma = false +csharp_space_before_dot = false +csharp_space_before_open_square_brackets = false +csharp_space_before_semicolon_in_for_statement = false +csharp_space_between_empty_square_brackets = false +csharp_space_around_declaration_statements = do_not_ignore +csharp_space_before_colon_in_inheritance_clause = true +csharp_space_after_colon_in_inheritance_clause = true +csharp_space_around_binary_operators = before_and_after +csharp_space_between_method_declaration_empty_parameter_list_parentheses = false +csharp_space_between_method_call_name_and_opening_parenthesis = false +csharp_space_between_method_call_empty_parameter_list_parentheses = false +csharp_space_between_method_declaration_name_and_open_parenthesis = false +csharp_space_between_square_brackets = false + +# Wrapping preferences +csharp_preserve_single_line_statements = true +csharp_preserve_single_line_blocks = true + +# License header +file_header_template = Copyright 2024 Google LLC\n\nLicensed under the Apache License, Version 2.0 (the "License").\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at \n\nhttps://www.apache.org/licenses/LICENSE-2.0 \n\nUnless required by applicable law or agreed to in writing, software \ndistributed under the License is distributed on an "AS IS" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and \nlimitations under the License. +csharp_style_namespace_declarations = file_scoped:suggestion +csharp_style_prefer_null_check_over_type_check = true:suggestion +csharp_style_prefer_local_over_anonymous_function = true:suggestion +csharp_style_prefer_tuple_swap = true:suggestion +csharp_style_implicit_object_creation_when_type_is_apparent = true:suggestion +csharp_style_unused_value_assignment_preference = discard_variable:suggestion +csharp_style_unused_value_expression_statement_preference = discard_variable:suggestion +csharp_style_prefer_pattern_matching = true:suggestion +csharp_style_prefer_not_pattern = true:suggestion +csharp_style_prefer_extended_property_pattern = true:suggestion +csharp_style_prefer_parameter_null_checking = true:suggestion +csharp_style_prefer_method_group_conversion = true:suggestion + +# Change the severity to `error` for the rules that must not be violated, to fail the build. + +# IDE0005: Remove unnecessary using directives. +dotnet_diagnostic.IDE0005.severity = suggestion + +# IDE0011: Add braces +dotnet_diagnostic.IDE0011.severity = suggestion + +# IDE0017: Object initialization can be simplified. +dotnet_diagnostic.IDE0017.severity = suggestion + +# IDE0019: Use pattern matching. +dotnet_diagnostic.IDE0019.severity = suggestion + +# IDE0021: Use expression body for constructor. +dotnet_diagnostic.IDE0021.severity = suggestion + +# IDE0028: Collection initialization can be simplified. +dotnet_diagnostic.IDE0028.severity = suggestion + +# IDE0035: Remove unreachable code. +dotnet_diagnostic.IDE0035.severity = suggestion + +# IDE0040: Accessibility modifiers required. +dotnet_diagnostic.IDE0040.severity = suggestion + +# IDE0052: Remove unread private member +dotnet_diagnostic.IDE0052.severity = suggestion + +# IDE0053: Use expression body for lambda expression. +dotnet_diagnostic.IDE0053.severity = suggestion + +# IDE0055: All C# and .NET formatting rules. +dotnet_diagnostic.IDE0055.severity = suggestion + +# IDE0060: Remove unused parameter +dotnet_diagnostic.IDE0060.severity = suggestion + +# IDE0063: Using statement can be simplified +dotnet_diagnostic.IDE0063.severity = suggestion + +# IDE0065: Using statement can be simplified +dotnet_diagnostic.IDE0065.severity = suggestion + +# IDE0071: Interpolation can be simplified. +dotnet_diagnostic.IDE0071.severity = suggestion + +# IDE0075: Simplify conditional expression. +dotnet_diagnostic.IDE0075.severity = suggestion diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs index 8e65a07b7fa..8815afa51df 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/CreateEventDrivenGcsTransferTest.cs @@ -29,8 +29,8 @@ public class CreateEventDrivenGcsTransferTest : IDisposable private string _transferJobName; private readonly string _sourceBucket; private readonly string _sinkBucket; - private string TopicId { get; } = Guid.NewGuid().ToString(); - private string SubscriptionId { get; } = Guid.NewGuid().ToString(); + private string TopicId { get; } = $"Topic-{Guid.NewGuid().ToString()}"; + private string SubscriptionId { get; } = $"Subscription-{Guid.NewGuid().ToString()}"; private SubscriberServiceApiClient SubscriberClient { get; } = SubscriberServiceApiClient.Create(); private PublisherServiceApiClient PublisherClient { get; } = PublisherServiceApiClient.Create(); From c7e1a3576b0af45880b8710b53d19c21e683f2a2 Mon Sep 17 00:00:00 2001 From: Mahendra Date: Wed, 16 Apr 2025 17:38:59 +0530 Subject: [PATCH 47/58] Delete storage/api/.editorconfig --- storage/api/.editorconfig | 281 -------------------------------------- 1 file changed, 281 deletions(-) delete mode 100644 storage/api/.editorconfig diff --git a/storage/api/.editorconfig b/storage/api/.editorconfig deleted file mode 100644 index a0b75fc4dff..00000000000 --- a/storage/api/.editorconfig +++ /dev/null @@ -1,281 +0,0 @@ -# To learn more about .editorconfig see https://aka.ms/editorconfigdocs -# top-most EditorConfig file -root = true -############################### -# Core EditorConfig Options # -############################### -# All files -[*] -insert_final_newline = true -indent_style = space -indent_size = 4 -trim_trailing_whitespace = true - -# Generated code -[*{_AssemblyInfo.cs,.g.cs}] -generated_code = true - -# XML project files -[*.{csproj}] -indent_size = 2 - -# Code files - -############################### -# .NET Coding Conventions # -############################### -[*.{cs}] - -# Organize usings -dotnet_sort_system_directives_first = false - -# this. preferences -dotnet_style_qualification_for_field = false:suggestion -dotnet_style_qualification_for_property = false:suggestion -dotnet_style_qualification_for_method = false:suggestion -dotnet_style_qualification_for_event = false:suggestion - -# Language keywords vs BCL types preferences -dotnet_style_predefined_type_for_locals_parameters_members = true:suggestion -dotnet_style_predefined_type_for_member_access = true:suggestion - -# Parentheses preferences -dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:silent -dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:silent -dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:silent -dotnet_style_parentheses_in_other_operators = never_if_unnecessary:silent - -# Modifier preferences -csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:warning -dotnet_style_require_accessibility_modifiers = for_non_interface_members:warning -dotnet_style_readonly_field = true:warning - -# Expression-level preferences -dotnet_style_object_initializer = true:suggestion -dotnet_style_collection_initializer = true:suggestion -dotnet_style_explicit_tuple_names = true:suggestion -dotnet_style_null_propagation = true:suggestion -dotnet_style_coalesce_expression = true:suggestion -dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion -dotnet_style_prefer_inferred_tuple_names = true:suggestion -dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion -dotnet_style_prefer_auto_properties = true:suggestion -dotnet_style_prefer_conditional_expression_over_assignment = true:silent -dotnet_style_prefer_conditional_expression_over_return = true:silent -csharp_prefer_simple_default_expression = true:suggestion - -############################### -# Naming Conventions # -############################### -# Style Definitions -dotnet_naming_style.pascal_case_style.capitalization = pascal_case - -# Use PascalCase for constant fields -dotnet_naming_rule.constant_fields_should_be_pascal_case.severity = suggestion -dotnet_naming_rule.constant_fields_should_be_pascal_case.symbols = constant_fields -dotnet_naming_rule.constant_fields_should_be_pascal_case.style = pascal_case_style -dotnet_naming_symbols.constant_fields.applicable_kinds = field -dotnet_naming_symbols.constant_fields.applicable_accessibilities = * -dotnet_naming_symbols.constant_fields.required_modifiers = const -dotnet_naming_style.pascal_case_style.capitalization = pascal_case - -# static fields should have s_ prefix -dotnet_naming_rule.static_fields_should_have_prefix.severity = suggestion -dotnet_naming_rule.static_fields_should_have_prefix.symbols = static_fields -dotnet_naming_rule.static_fields_should_have_prefix.style = static_prefix_style -dotnet_naming_symbols.static_fields.applicable_kinds = field -dotnet_naming_symbols.static_fields.required_modifiers = static -dotnet_naming_symbols.static_fields.applicable_accessibilities = private, internal, private_protected -dotnet_naming_style.static_prefix_style.required_prefix = s_ -dotnet_naming_style.static_prefix_style.capitalization = camel_case - -# internal and private fields should be _camelCase -dotnet_naming_rule.camel_case_for_private_internal_fields.severity = suggestion -dotnet_naming_rule.camel_case_for_private_internal_fields.symbols = private_internal_fields -dotnet_naming_rule.camel_case_for_private_internal_fields.style = camel_case_underscore_style -dotnet_naming_symbols.private_internal_fields.applicable_kinds = field -dotnet_naming_symbols.private_internal_fields.applicable_accessibilities = private, internal -dotnet_naming_style.camel_case_underscore_style.required_prefix = _ -dotnet_naming_style.camel_case_underscore_style.capitalization = camel_case - -# Locals and parameters are camelCase -dotnet_naming_rule.locals_should_be_camel_case.severity = suggestion -dotnet_naming_rule.locals_should_be_camel_case.symbols = locals_and_parameters -dotnet_naming_rule.locals_should_be_camel_case.style = camel_case_style -dotnet_naming_symbols.locals_and_parameters.applicable_kinds = parameter, local -dotnet_naming_style.camel_case_style.capitalization = camel_case - -# Public property and methods should be PascalCase -dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.severity = suggestion -dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.symbols = public_methods_and_properties -dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.style = pascal_case -dotnet_naming_symbols.public_methods_and_properties.applicable_kinds = method, property -dotnet_naming_symbols.public_methods_and_properties.applicable_accessibilities = public -dotnet_naming_style.pascal_case.capitalization = pascal_case - -# Private and internal methods should be PascalCase -dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.severity = suggestion -dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.symbols = private_and_internal_methods -dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.style = pascal_case -dotnet_naming_symbols.private_and_internal_methods.applicable_kinds = method -dotnet_naming_symbols.private_and_internal_methods.applicable_accessibilities = private, internal -############################### -# C# Coding Conventions # -############################### -[*.cs] -# var preferences -csharp_style_var_for_built_in_types = true:silent -csharp_style_var_when_type_is_apparent = true:silent -csharp_style_var_elsewhere = true:silent -dotnet_style_predefined_type_for_locals_parameters_members = true:silent -dotnet_style_predefined_type_for_member_access = true:silent - -csharp_using_directive_placement = outside_namespace:warning -csharp_prefer_braces = true:suggestion -csharp_preserve_single_line_blocks = true:none -csharp_preserve_single_line_statements = false:none -csharp_prefer_static_local_function = true:suggestion -csharp_prefer_simple_using_statement = true:suggestion -csharp_style_prefer_switch_expression = true:suggestion - -# Expression-bodied members -csharp_style_expression_bodied_methods = true:suggestion -csharp_style_expression_bodied_constructors = true:suggestion -csharp_style_expression_bodied_operators = true:suggestion -csharp_style_expression_bodied_properties = true:suggestion -csharp_style_expression_bodied_indexers = true:suggestion -csharp_style_expression_bodied_accessors = true:suggestion -csharp_style_expression_bodied_lambdas = true:suggestion -csharp_style_expression_bodied_local_functions = true:suggestion - -# Pattern matching preferences -csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion -csharp_style_pattern_matching_over_as_with_null_check = true:suggestion -csharp_style_inlined_variable_declaration = true:suggestion - -# Null-checking preferences -csharp_style_throw_expression = true:suggestion -csharp_style_conditional_delegate_call = true:suggestion - -# Other features -csharp_style_prefer_index_operator = true:suggestion -csharp_style_prefer_range_operator = true:suggestion -csharp_style_pattern_local_over_anonymous_function = false:none - -############################### -# C# Formatting Rules # -############################### -indent_size = 4 -insert_final_newline = true -charset = utf-8 - -# New line preferences -csharp_new_line_before_open_brace = all -csharp_new_line_before_else = true -csharp_new_line_before_catch = true -csharp_new_line_before_finally = true -csharp_new_line_before_members_in_object_initializers = true -csharp_new_line_before_members_in_anonymous_types = true -csharp_new_line_between_query_expression_clauses = true - -# Indentation preferences -csharp_indent_block_contents = true -csharp_indent_braces = false -csharp_indent_case_contents = true -csharp_indent_case_contents_when_block = true -csharp_indent_switch_labels = true -csharp_indent_labels = one_less_than_current - -# Space preferences -csharp_space_after_cast = true -csharp_space_after_colon_in_inheritance_clause = true -csharp_space_after_comma = true -csharp_space_after_dot = false -csharp_space_after_keywords_in_control_flow_statements = true -csharp_space_after_semicolon_in_for_statement = true -csharp_space_between_method_call_parameter_list_parentheses = false -csharp_space_between_method_declaration_parameter_list_parentheses = false -csharp_space_between_parentheses = false -csharp_space_before_comma = false -csharp_space_before_dot = false -csharp_space_before_open_square_brackets = false -csharp_space_before_semicolon_in_for_statement = false -csharp_space_between_empty_square_brackets = false -csharp_space_around_declaration_statements = do_not_ignore -csharp_space_before_colon_in_inheritance_clause = true -csharp_space_after_colon_in_inheritance_clause = true -csharp_space_around_binary_operators = before_and_after -csharp_space_between_method_declaration_empty_parameter_list_parentheses = false -csharp_space_between_method_call_name_and_opening_parenthesis = false -csharp_space_between_method_call_empty_parameter_list_parentheses = false -csharp_space_between_method_declaration_name_and_open_parenthesis = false -csharp_space_between_square_brackets = false - -# Wrapping preferences -csharp_preserve_single_line_statements = true -csharp_preserve_single_line_blocks = true - -# License header -file_header_template = Copyright 2024 Google LLC\n\nLicensed under the Apache License, Version 2.0 (the "License").\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at \n\nhttps://www.apache.org/licenses/LICENSE-2.0 \n\nUnless required by applicable law or agreed to in writing, software \ndistributed under the License is distributed on an "AS IS" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and \nlimitations under the License. -csharp_style_namespace_declarations = file_scoped:suggestion -csharp_style_prefer_null_check_over_type_check = true:suggestion -csharp_style_prefer_local_over_anonymous_function = true:suggestion -csharp_style_prefer_tuple_swap = true:suggestion -csharp_style_implicit_object_creation_when_type_is_apparent = true:suggestion -csharp_style_unused_value_assignment_preference = discard_variable:suggestion -csharp_style_unused_value_expression_statement_preference = discard_variable:suggestion -csharp_style_prefer_pattern_matching = true:suggestion -csharp_style_prefer_not_pattern = true:suggestion -csharp_style_prefer_extended_property_pattern = true:suggestion -csharp_style_prefer_parameter_null_checking = true:suggestion -csharp_style_prefer_method_group_conversion = true:suggestion - -# Change the severity to `error` for the rules that must not be violated, to fail the build. - -# IDE0005: Remove unnecessary using directives. -dotnet_diagnostic.IDE0005.severity = suggestion - -# IDE0011: Add braces -dotnet_diagnostic.IDE0011.severity = suggestion - -# IDE0017: Object initialization can be simplified. -dotnet_diagnostic.IDE0017.severity = suggestion - -# IDE0019: Use pattern matching. -dotnet_diagnostic.IDE0019.severity = suggestion - -# IDE0021: Use expression body for constructor. -dotnet_diagnostic.IDE0021.severity = suggestion - -# IDE0028: Collection initialization can be simplified. -dotnet_diagnostic.IDE0028.severity = suggestion - -# IDE0035: Remove unreachable code. -dotnet_diagnostic.IDE0035.severity = suggestion - -# IDE0040: Accessibility modifiers required. -dotnet_diagnostic.IDE0040.severity = suggestion - -# IDE0052: Remove unread private member -dotnet_diagnostic.IDE0052.severity = suggestion - -# IDE0053: Use expression body for lambda expression. -dotnet_diagnostic.IDE0053.severity = suggestion - -# IDE0055: All C# and .NET formatting rules. -dotnet_diagnostic.IDE0055.severity = suggestion - -# IDE0060: Remove unused parameter -dotnet_diagnostic.IDE0060.severity = suggestion - -# IDE0063: Using statement can be simplified -dotnet_diagnostic.IDE0063.severity = suggestion - -# IDE0065: Using statement can be simplified -dotnet_diagnostic.IDE0065.severity = suggestion - -# IDE0071: Interpolation can be simplified. -dotnet_diagnostic.IDE0071.severity = suggestion - -# IDE0075: Simplify conditional expression. -dotnet_diagnostic.IDE0075.severity = suggestion From e4f5c85c1dd8d3ba5cd1074c4afb87161f9ea523 Mon Sep 17 00:00:00 2001 From: Mahendra Date: Wed, 16 Apr 2025 17:39:33 +0530 Subject: [PATCH 48/58] Delete storagetransfer/api/.editorconfig --- storagetransfer/api/.editorconfig | 281 ------------------------------ 1 file changed, 281 deletions(-) delete mode 100644 storagetransfer/api/.editorconfig diff --git a/storagetransfer/api/.editorconfig b/storagetransfer/api/.editorconfig deleted file mode 100644 index a0b75fc4dff..00000000000 --- a/storagetransfer/api/.editorconfig +++ /dev/null @@ -1,281 +0,0 @@ -# To learn more about .editorconfig see https://aka.ms/editorconfigdocs -# top-most EditorConfig file -root = true -############################### -# Core EditorConfig Options # -############################### -# All files -[*] -insert_final_newline = true -indent_style = space -indent_size = 4 -trim_trailing_whitespace = true - -# Generated code -[*{_AssemblyInfo.cs,.g.cs}] -generated_code = true - -# XML project files -[*.{csproj}] -indent_size = 2 - -# Code files - -############################### -# .NET Coding Conventions # -############################### -[*.{cs}] - -# Organize usings -dotnet_sort_system_directives_first = false - -# this. preferences -dotnet_style_qualification_for_field = false:suggestion -dotnet_style_qualification_for_property = false:suggestion -dotnet_style_qualification_for_method = false:suggestion -dotnet_style_qualification_for_event = false:suggestion - -# Language keywords vs BCL types preferences -dotnet_style_predefined_type_for_locals_parameters_members = true:suggestion -dotnet_style_predefined_type_for_member_access = true:suggestion - -# Parentheses preferences -dotnet_style_parentheses_in_arithmetic_binary_operators = always_for_clarity:silent -dotnet_style_parentheses_in_relational_binary_operators = always_for_clarity:silent -dotnet_style_parentheses_in_other_binary_operators = always_for_clarity:silent -dotnet_style_parentheses_in_other_operators = never_if_unnecessary:silent - -# Modifier preferences -csharp_preferred_modifier_order = public,private,protected,internal,static,extern,new,virtual,abstract,sealed,override,readonly,unsafe,volatile,async:warning -dotnet_style_require_accessibility_modifiers = for_non_interface_members:warning -dotnet_style_readonly_field = true:warning - -# Expression-level preferences -dotnet_style_object_initializer = true:suggestion -dotnet_style_collection_initializer = true:suggestion -dotnet_style_explicit_tuple_names = true:suggestion -dotnet_style_null_propagation = true:suggestion -dotnet_style_coalesce_expression = true:suggestion -dotnet_style_prefer_is_null_check_over_reference_equality_method = true:suggestion -dotnet_style_prefer_inferred_tuple_names = true:suggestion -dotnet_style_prefer_inferred_anonymous_type_member_names = true:suggestion -dotnet_style_prefer_auto_properties = true:suggestion -dotnet_style_prefer_conditional_expression_over_assignment = true:silent -dotnet_style_prefer_conditional_expression_over_return = true:silent -csharp_prefer_simple_default_expression = true:suggestion - -############################### -# Naming Conventions # -############################### -# Style Definitions -dotnet_naming_style.pascal_case_style.capitalization = pascal_case - -# Use PascalCase for constant fields -dotnet_naming_rule.constant_fields_should_be_pascal_case.severity = suggestion -dotnet_naming_rule.constant_fields_should_be_pascal_case.symbols = constant_fields -dotnet_naming_rule.constant_fields_should_be_pascal_case.style = pascal_case_style -dotnet_naming_symbols.constant_fields.applicable_kinds = field -dotnet_naming_symbols.constant_fields.applicable_accessibilities = * -dotnet_naming_symbols.constant_fields.required_modifiers = const -dotnet_naming_style.pascal_case_style.capitalization = pascal_case - -# static fields should have s_ prefix -dotnet_naming_rule.static_fields_should_have_prefix.severity = suggestion -dotnet_naming_rule.static_fields_should_have_prefix.symbols = static_fields -dotnet_naming_rule.static_fields_should_have_prefix.style = static_prefix_style -dotnet_naming_symbols.static_fields.applicable_kinds = field -dotnet_naming_symbols.static_fields.required_modifiers = static -dotnet_naming_symbols.static_fields.applicable_accessibilities = private, internal, private_protected -dotnet_naming_style.static_prefix_style.required_prefix = s_ -dotnet_naming_style.static_prefix_style.capitalization = camel_case - -# internal and private fields should be _camelCase -dotnet_naming_rule.camel_case_for_private_internal_fields.severity = suggestion -dotnet_naming_rule.camel_case_for_private_internal_fields.symbols = private_internal_fields -dotnet_naming_rule.camel_case_for_private_internal_fields.style = camel_case_underscore_style -dotnet_naming_symbols.private_internal_fields.applicable_kinds = field -dotnet_naming_symbols.private_internal_fields.applicable_accessibilities = private, internal -dotnet_naming_style.camel_case_underscore_style.required_prefix = _ -dotnet_naming_style.camel_case_underscore_style.capitalization = camel_case - -# Locals and parameters are camelCase -dotnet_naming_rule.locals_should_be_camel_case.severity = suggestion -dotnet_naming_rule.locals_should_be_camel_case.symbols = locals_and_parameters -dotnet_naming_rule.locals_should_be_camel_case.style = camel_case_style -dotnet_naming_symbols.locals_and_parameters.applicable_kinds = parameter, local -dotnet_naming_style.camel_case_style.capitalization = camel_case - -# Public property and methods should be PascalCase -dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.severity = suggestion -dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.symbols = public_methods_and_properties -dotnet_naming_rule.public_methods_and_properties_should_be_pascal_case.style = pascal_case -dotnet_naming_symbols.public_methods_and_properties.applicable_kinds = method, property -dotnet_naming_symbols.public_methods_and_properties.applicable_accessibilities = public -dotnet_naming_style.pascal_case.capitalization = pascal_case - -# Private and internal methods should be PascalCase -dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.severity = suggestion -dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.symbols = private_and_internal_methods -dotnet_naming_rule.private_and_internal_methods_should_be_pascal_case.style = pascal_case -dotnet_naming_symbols.private_and_internal_methods.applicable_kinds = method -dotnet_naming_symbols.private_and_internal_methods.applicable_accessibilities = private, internal -############################### -# C# Coding Conventions # -############################### -[*.cs] -# var preferences -csharp_style_var_for_built_in_types = true:silent -csharp_style_var_when_type_is_apparent = true:silent -csharp_style_var_elsewhere = true:silent -dotnet_style_predefined_type_for_locals_parameters_members = true:silent -dotnet_style_predefined_type_for_member_access = true:silent - -csharp_using_directive_placement = outside_namespace:warning -csharp_prefer_braces = true:suggestion -csharp_preserve_single_line_blocks = true:none -csharp_preserve_single_line_statements = false:none -csharp_prefer_static_local_function = true:suggestion -csharp_prefer_simple_using_statement = true:suggestion -csharp_style_prefer_switch_expression = true:suggestion - -# Expression-bodied members -csharp_style_expression_bodied_methods = true:suggestion -csharp_style_expression_bodied_constructors = true:suggestion -csharp_style_expression_bodied_operators = true:suggestion -csharp_style_expression_bodied_properties = true:suggestion -csharp_style_expression_bodied_indexers = true:suggestion -csharp_style_expression_bodied_accessors = true:suggestion -csharp_style_expression_bodied_lambdas = true:suggestion -csharp_style_expression_bodied_local_functions = true:suggestion - -# Pattern matching preferences -csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion -csharp_style_pattern_matching_over_as_with_null_check = true:suggestion -csharp_style_inlined_variable_declaration = true:suggestion - -# Null-checking preferences -csharp_style_throw_expression = true:suggestion -csharp_style_conditional_delegate_call = true:suggestion - -# Other features -csharp_style_prefer_index_operator = true:suggestion -csharp_style_prefer_range_operator = true:suggestion -csharp_style_pattern_local_over_anonymous_function = false:none - -############################### -# C# Formatting Rules # -############################### -indent_size = 4 -insert_final_newline = true -charset = utf-8 - -# New line preferences -csharp_new_line_before_open_brace = all -csharp_new_line_before_else = true -csharp_new_line_before_catch = true -csharp_new_line_before_finally = true -csharp_new_line_before_members_in_object_initializers = true -csharp_new_line_before_members_in_anonymous_types = true -csharp_new_line_between_query_expression_clauses = true - -# Indentation preferences -csharp_indent_block_contents = true -csharp_indent_braces = false -csharp_indent_case_contents = true -csharp_indent_case_contents_when_block = true -csharp_indent_switch_labels = true -csharp_indent_labels = one_less_than_current - -# Space preferences -csharp_space_after_cast = true -csharp_space_after_colon_in_inheritance_clause = true -csharp_space_after_comma = true -csharp_space_after_dot = false -csharp_space_after_keywords_in_control_flow_statements = true -csharp_space_after_semicolon_in_for_statement = true -csharp_space_between_method_call_parameter_list_parentheses = false -csharp_space_between_method_declaration_parameter_list_parentheses = false -csharp_space_between_parentheses = false -csharp_space_before_comma = false -csharp_space_before_dot = false -csharp_space_before_open_square_brackets = false -csharp_space_before_semicolon_in_for_statement = false -csharp_space_between_empty_square_brackets = false -csharp_space_around_declaration_statements = do_not_ignore -csharp_space_before_colon_in_inheritance_clause = true -csharp_space_after_colon_in_inheritance_clause = true -csharp_space_around_binary_operators = before_and_after -csharp_space_between_method_declaration_empty_parameter_list_parentheses = false -csharp_space_between_method_call_name_and_opening_parenthesis = false -csharp_space_between_method_call_empty_parameter_list_parentheses = false -csharp_space_between_method_declaration_name_and_open_parenthesis = false -csharp_space_between_square_brackets = false - -# Wrapping preferences -csharp_preserve_single_line_statements = true -csharp_preserve_single_line_blocks = true - -# License header -file_header_template = Copyright 2024 Google LLC\n\nLicensed under the Apache License, Version 2.0 (the "License").\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at \n\nhttps://www.apache.org/licenses/LICENSE-2.0 \n\nUnless required by applicable law or agreed to in writing, software \ndistributed under the License is distributed on an "AS IS" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and \nlimitations under the License. -csharp_style_namespace_declarations = file_scoped:suggestion -csharp_style_prefer_null_check_over_type_check = true:suggestion -csharp_style_prefer_local_over_anonymous_function = true:suggestion -csharp_style_prefer_tuple_swap = true:suggestion -csharp_style_implicit_object_creation_when_type_is_apparent = true:suggestion -csharp_style_unused_value_assignment_preference = discard_variable:suggestion -csharp_style_unused_value_expression_statement_preference = discard_variable:suggestion -csharp_style_prefer_pattern_matching = true:suggestion -csharp_style_prefer_not_pattern = true:suggestion -csharp_style_prefer_extended_property_pattern = true:suggestion -csharp_style_prefer_parameter_null_checking = true:suggestion -csharp_style_prefer_method_group_conversion = true:suggestion - -# Change the severity to `error` for the rules that must not be violated, to fail the build. - -# IDE0005: Remove unnecessary using directives. -dotnet_diagnostic.IDE0005.severity = suggestion - -# IDE0011: Add braces -dotnet_diagnostic.IDE0011.severity = suggestion - -# IDE0017: Object initialization can be simplified. -dotnet_diagnostic.IDE0017.severity = suggestion - -# IDE0019: Use pattern matching. -dotnet_diagnostic.IDE0019.severity = suggestion - -# IDE0021: Use expression body for constructor. -dotnet_diagnostic.IDE0021.severity = suggestion - -# IDE0028: Collection initialization can be simplified. -dotnet_diagnostic.IDE0028.severity = suggestion - -# IDE0035: Remove unreachable code. -dotnet_diagnostic.IDE0035.severity = suggestion - -# IDE0040: Accessibility modifiers required. -dotnet_diagnostic.IDE0040.severity = suggestion - -# IDE0052: Remove unread private member -dotnet_diagnostic.IDE0052.severity = suggestion - -# IDE0053: Use expression body for lambda expression. -dotnet_diagnostic.IDE0053.severity = suggestion - -# IDE0055: All C# and .NET formatting rules. -dotnet_diagnostic.IDE0055.severity = suggestion - -# IDE0060: Remove unused parameter -dotnet_diagnostic.IDE0060.severity = suggestion - -# IDE0063: Using statement can be simplified -dotnet_diagnostic.IDE0063.severity = suggestion - -# IDE0065: Using statement can be simplified -dotnet_diagnostic.IDE0065.severity = suggestion - -# IDE0071: Interpolation can be simplified. -dotnet_diagnostic.IDE0071.severity = suggestion - -# IDE0075: Simplify conditional expression. -dotnet_diagnostic.IDE0075.severity = suggestion From 54197f5f78c71c43e6dea6c9dd338e05e1db924c Mon Sep 17 00:00:00 2001 From: Mahendra Date: Thu, 17 Apr 2025 11:11:31 +0530 Subject: [PATCH 49/58] QuickStart sample and tests files renamed --- .../{QuickstartTest.cs => QuickStartTest.cs} | 0 .../{QuickstartSample.cs => QuickStartSample.cs} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename storagetransfer/api/StorageTransfer.Samples.Tests/{QuickstartTest.cs => QuickStartTest.cs} (100%) rename storagetransfer/api/StorageTransfer.Samples/{QuickstartSample.cs => QuickStartSample.cs} (100%) diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs b/storagetransfer/api/StorageTransfer.Samples.Tests/QuickStartTest.cs similarity index 100% rename from storagetransfer/api/StorageTransfer.Samples.Tests/QuickstartTest.cs rename to storagetransfer/api/StorageTransfer.Samples.Tests/QuickStartTest.cs diff --git a/storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs b/storagetransfer/api/StorageTransfer.Samples/QuickStartSample.cs similarity index 100% rename from storagetransfer/api/StorageTransfer.Samples/QuickstartSample.cs rename to storagetransfer/api/StorageTransfer.Samples/QuickStartSample.cs From 6f1147d8a418c8c0f015fe85870b08454095b5df Mon Sep 17 00:00:00 2001 From: Amanda Tarafa Mas Date: Tue, 20 May 2025 22:57:46 -0700 Subject: [PATCH 50/58] build: Update remaining sample sets to .NET 8 --- asset/api/Asset.Samples.Tests/Asset.Samples.Tests.csproj | 2 +- asset/api/Asset.sln | 4 ++-- auth/AuthSample/AuthSample.csproj | 2 +- auth/AuthTest/AuthTest.csproj | 2 +- bigquery/api/SimpleApp/SimpleApp.csproj | 2 +- bigquery/api/Snippets/Snippets.csproj | 2 +- bigquery/data-transfer/api/QuickStart/QuickStart.csproj | 2 +- .../data-transfer/api/QuickStartTest/QuickStartTest.csproj | 2 +- cloudtasks/api/TasksSample/TasksSample.csproj | 2 +- commandlineutil/Sample/CommandLineUtilSample.csproj | 2 +- commandlineutil/SampleTest/SampleTest.csproj | 2 +- .../DatastoreAdmin.Samples.Tests.csproj | 2 +- datastore/api/QuickStart/QuickStart.csproj | 2 +- datastore/api/QuickStartTest/QuickStartTest.csproj | 2 +- datastore/api/TaskList/TaskList.csproj | 2 +- datastore/api/TaskListTest/DatastoreTest.csproj | 2 +- dlp/api/Snippets.Tests/Snippets.Tests.csproj | 2 +- .../DocumentAI.Samples.Tests/DocumentAI.Samples.Tests.csproj | 2 +- documentai/api/DocumentAI.Samples/DocumentAI.Samples.csproj | 2 +- firestore/api/AddData/AddData.csproj | 2 +- firestore/api/DataModel/DataModel.csproj | 2 +- firestore/api/DeleteData/DeleteData.csproj | 2 +- firestore/api/FirestoreTest/FirestoreTest.csproj | 2 +- firestore/api/GetData/GetData.csproj | 2 +- firestore/api/ListenData/ListenData.csproj | 2 +- firestore/api/ManageIndexes/ManageIndexes.csproj | 2 +- firestore/api/OrderLimitData/OrderLimitData.csproj | 2 +- firestore/api/PaginateData/PaginateData.csproj | 2 +- firestore/api/QueryData/QueryData.csproj | 2 +- firestore/api/Quickstart/Quickstart.csproj | 2 +- firestore/api/SolutionCounter/SolutionCounter.csproj | 2 +- .../TransactionsAndBatchedWrites.csproj | 2 +- iam/api/AccessTest/AccessTest.csproj | 2 +- iam/api/CustomRoles/CustomRoles.csproj | 2 +- iam/api/CustomRolesTest/CustomRolesTest.csproj | 2 +- iam/api/QuickStart/QuickStart.csproj | 2 +- iam/api/QuickStartTest/QuickStartTest.csproj | 2 +- iam/api/ServiceAccountKeys/ServiceAccountKeys.csproj | 2 +- iam/api/ServiceAccountTests/ServiceAccountTests.csproj | 2 +- iam/api/ServiceAccounts/ServiceAccounts.csproj | 2 +- .../LiveStream.Samples.Tests/LiveStream.Samples.Tests.csproj | 2 +- .../api/Stitcher.Samples.Tests/Stitcher.Samples.Tests.csproj | 2 +- .../Transcoder.Samples.Tests/Transcoder.Samples.Tests.csproj | 2 +- monitoring/api/AlertSample/AlertSample.csproj | 2 +- monitoring/api/AlertSnippets/AlertSnippets.csproj | 2 +- monitoring/api/AlertTest/AlertTest.csproj | 2 +- monitoring/api/MonitoringSample/Monitoring.csproj | 2 +- monitoring/api/MonitoringTest/MonitoringTest.csproj | 2 +- monitoring/api/QuickStart/QuickStart.csproj | 2 +- monitoring/api/UptimeCheckSample/UptimeCheck.csproj | 2 +- monitoring/api/UptimeCheckTest/UptimeCheckTest.csproj | 2 +- pubsub/api/Pubsub.Samples.Tests/Pubsub.Samples.Tests.csproj | 2 +- .../Samples.Run.MarkdownPreview.Editor.Tests.csproj | 2 +- .../Samples.Run.MarkdownPreview.Editor.csproj | 2 +- .../Samples.Run.MarkdownPreview.Renderer.Tests.csproj | 2 +- .../Samples.Run.MarkdownPreview.Renderer.csproj | 2 +- .../Run.Samples.Pubsub.MinimalApi.Tests.csproj | 2 +- .../Run.Samples.Pubsub.MinimalApi.csproj | 2 +- .../api/Storage.Samples.Tests/Storage.Samples.Tests.csproj | 2 +- .../StorageInsights.Samples.Tests.csproj | 2 +- 60 files changed, 61 insertions(+), 61 deletions(-) diff --git a/asset/api/Asset.Samples.Tests/Asset.Samples.Tests.csproj b/asset/api/Asset.Samples.Tests/Asset.Samples.Tests.csproj index 7fc8e786a3c..59a45748ff4 100644 --- a/asset/api/Asset.Samples.Tests/Asset.Samples.Tests.csproj +++ b/asset/api/Asset.Samples.Tests/Asset.Samples.Tests.csproj @@ -1,7 +1,7 @@  - net6.0 + net8.0 false diff --git a/asset/api/Asset.sln b/asset/api/Asset.sln index 095021856c8..2225b86634f 100644 --- a/asset/api/Asset.sln +++ b/asset/api/Asset.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.31229.75 +# Visual Studio Version 17 +VisualStudioVersion = 17.13.35931.197 d17.13 MinimumVisualStudioVersion = 15.0.26124.0 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Asset.Samples", "Asset.Samples\Asset.Samples.csproj", "{9E6D0AED-B21C-4A20-90EA-F17875346C8D}" EndProject diff --git a/auth/AuthSample/AuthSample.csproj b/auth/AuthSample/AuthSample.csproj index 1429918703a..0b6fc092bc9 100644 --- a/auth/AuthSample/AuthSample.csproj +++ b/auth/AuthSample/AuthSample.csproj @@ -1,7 +1,7 @@ Exe - net6.0 + net8.0 diff --git a/auth/AuthTest/AuthTest.csproj b/auth/AuthTest/AuthTest.csproj index 9821e8f6b3c..2eb59d8d3e2 100644 --- a/auth/AuthTest/AuthTest.csproj +++ b/auth/AuthTest/AuthTest.csproj @@ -1,6 +1,6 @@ - net6.0 + net8.0 false diff --git a/bigquery/api/SimpleApp/SimpleApp.csproj b/bigquery/api/SimpleApp/SimpleApp.csproj index 6ecb4e1a1ca..6362f05e3d9 100644 --- a/bigquery/api/SimpleApp/SimpleApp.csproj +++ b/bigquery/api/SimpleApp/SimpleApp.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 false diff --git a/bigquery/api/Snippets/Snippets.csproj b/bigquery/api/Snippets/Snippets.csproj index 3a3da9f4387..bda8622e69b 100644 --- a/bigquery/api/Snippets/Snippets.csproj +++ b/bigquery/api/Snippets/Snippets.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 diff --git a/bigquery/data-transfer/api/QuickStart/QuickStart.csproj b/bigquery/data-transfer/api/QuickStart/QuickStart.csproj index b19b0793a3b..e6f48768273 100644 --- a/bigquery/data-transfer/api/QuickStart/QuickStart.csproj +++ b/bigquery/data-transfer/api/QuickStart/QuickStart.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 diff --git a/bigquery/data-transfer/api/QuickStartTest/QuickStartTest.csproj b/bigquery/data-transfer/api/QuickStartTest/QuickStartTest.csproj index c1e4d3d3815..4c6274b9794 100644 --- a/bigquery/data-transfer/api/QuickStartTest/QuickStartTest.csproj +++ b/bigquery/data-transfer/api/QuickStartTest/QuickStartTest.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 false diff --git a/cloudtasks/api/TasksSample/TasksSample.csproj b/cloudtasks/api/TasksSample/TasksSample.csproj index 893abcebf02..f65babd186f 100644 --- a/cloudtasks/api/TasksSample/TasksSample.csproj +++ b/cloudtasks/api/TasksSample/TasksSample.csproj @@ -1,6 +1,6 @@ - net6.0 + net8.0 diff --git a/commandlineutil/Sample/CommandLineUtilSample.csproj b/commandlineutil/Sample/CommandLineUtilSample.csproj index 31e669a88c9..c676f76ecfc 100644 --- a/commandlineutil/Sample/CommandLineUtilSample.csproj +++ b/commandlineutil/Sample/CommandLineUtilSample.csproj @@ -6,7 +6,7 @@ Exe - net6.0 + net8.0 diff --git a/commandlineutil/SampleTest/SampleTest.csproj b/commandlineutil/SampleTest/SampleTest.csproj index d20bb9fe7f8..43f1278591c 100644 --- a/commandlineutil/SampleTest/SampleTest.csproj +++ b/commandlineutil/SampleTest/SampleTest.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 false diff --git a/datastore/admin-api/DatastoreAdmin.Samples.Tests/DatastoreAdmin.Samples.Tests.csproj b/datastore/admin-api/DatastoreAdmin.Samples.Tests/DatastoreAdmin.Samples.Tests.csproj index 9fc045f5609..4554bcf0d57 100644 --- a/datastore/admin-api/DatastoreAdmin.Samples.Tests/DatastoreAdmin.Samples.Tests.csproj +++ b/datastore/admin-api/DatastoreAdmin.Samples.Tests/DatastoreAdmin.Samples.Tests.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 false diff --git a/datastore/api/QuickStart/QuickStart.csproj b/datastore/api/QuickStart/QuickStart.csproj index 06a8c793ef4..3bdff3e2b7b 100644 --- a/datastore/api/QuickStart/QuickStart.csproj +++ b/datastore/api/QuickStart/QuickStart.csproj @@ -1,7 +1,7 @@ Exe - net6.0 + net8.0 diff --git a/datastore/api/QuickStartTest/QuickStartTest.csproj b/datastore/api/QuickStartTest/QuickStartTest.csproj index eadae3239b6..407e48a382e 100644 --- a/datastore/api/QuickStartTest/QuickStartTest.csproj +++ b/datastore/api/QuickStartTest/QuickStartTest.csproj @@ -1,7 +1,7 @@  - net6.0 + net8.0 false diff --git a/datastore/api/TaskList/TaskList.csproj b/datastore/api/TaskList/TaskList.csproj index 61e6341cf28..62d32836bcf 100644 --- a/datastore/api/TaskList/TaskList.csproj +++ b/datastore/api/TaskList/TaskList.csproj @@ -1,7 +1,7 @@  Exe - net6.0 + net8.0 diff --git a/datastore/api/TaskListTest/DatastoreTest.csproj b/datastore/api/TaskListTest/DatastoreTest.csproj index 09414202f3d..f908efb12c5 100644 --- a/datastore/api/TaskListTest/DatastoreTest.csproj +++ b/datastore/api/TaskListTest/DatastoreTest.csproj @@ -1,7 +1,7 @@  Library - net6.0 + net8.0 diff --git a/dlp/api/Snippets.Tests/Snippets.Tests.csproj b/dlp/api/Snippets.Tests/Snippets.Tests.csproj index 63d61547253..3831a626067 100644 --- a/dlp/api/Snippets.Tests/Snippets.Tests.csproj +++ b/dlp/api/Snippets.Tests/Snippets.Tests.csproj @@ -1,6 +1,6 @@  - net6.0 + net8.0 false diff --git a/documentai/api/DocumentAI.Samples.Tests/DocumentAI.Samples.Tests.csproj b/documentai/api/DocumentAI.Samples.Tests/DocumentAI.Samples.Tests.csproj index 09fa5fe15ec..ffd9707da07 100644 --- a/documentai/api/DocumentAI.Samples.Tests/DocumentAI.Samples.Tests.csproj +++ b/documentai/api/DocumentAI.Samples.Tests/DocumentAI.Samples.Tests.csproj @@ -1,6 +1,6 @@ - net6.0 + net8.0 false diff --git a/documentai/api/DocumentAI.Samples/DocumentAI.Samples.csproj b/documentai/api/DocumentAI.Samples/DocumentAI.Samples.csproj index 90dba123732..28f7763c480 100644 --- a/documentai/api/DocumentAI.Samples/DocumentAI.Samples.csproj +++ b/documentai/api/DocumentAI.Samples/DocumentAI.Samples.csproj @@ -1,6 +1,6 @@ - net6.0 + net8.0 false diff --git a/firestore/api/AddData/AddData.csproj b/firestore/api/AddData/AddData.csproj index 97cd2ead27f..e24a8cc037c 100644 --- a/firestore/api/AddData/AddData.csproj +++ b/firestore/api/AddData/AddData.csproj @@ -1,7 +1,7 @@ Exe - net6.0 + net8.0 GoogleCloudSamples.AddData diff --git a/firestore/api/DataModel/DataModel.csproj b/firestore/api/DataModel/DataModel.csproj index 8f19930ee38..d4d70c82f7e 100644 --- a/firestore/api/DataModel/DataModel.csproj +++ b/firestore/api/DataModel/DataModel.csproj @@ -1,7 +1,7 @@ Exe - net6.0 + net8.0 GoogleCloudSamples.DataModel diff --git a/firestore/api/DeleteData/DeleteData.csproj b/firestore/api/DeleteData/DeleteData.csproj index b743197d2b8..6f34bfb3604 100644 --- a/firestore/api/DeleteData/DeleteData.csproj +++ b/firestore/api/DeleteData/DeleteData.csproj @@ -1,7 +1,7 @@ Exe - net6.0 + net8.0 GoogleCloudSamples.DeleteData diff --git a/firestore/api/FirestoreTest/FirestoreTest.csproj b/firestore/api/FirestoreTest/FirestoreTest.csproj index c4cda8c439e..ac6cfa15827 100644 --- a/firestore/api/FirestoreTest/FirestoreTest.csproj +++ b/firestore/api/FirestoreTest/FirestoreTest.csproj @@ -1,7 +1,7 @@ Library - net6.0 + net8.0 false diff --git a/firestore/api/GetData/GetData.csproj b/firestore/api/GetData/GetData.csproj index 9d14cea87e7..f51180ce14d 100644 --- a/firestore/api/GetData/GetData.csproj +++ b/firestore/api/GetData/GetData.csproj @@ -1,7 +1,7 @@ Exe - net6.0 + net8.0 GoogleCloudSamples.GetData latest diff --git a/firestore/api/ListenData/ListenData.csproj b/firestore/api/ListenData/ListenData.csproj index af6d4603c61..b9085f1e064 100644 --- a/firestore/api/ListenData/ListenData.csproj +++ b/firestore/api/ListenData/ListenData.csproj @@ -1,7 +1,7 @@ Exe - net6.0 + net8.0 GoogleCloudSamples.ListenData diff --git a/firestore/api/ManageIndexes/ManageIndexes.csproj b/firestore/api/ManageIndexes/ManageIndexes.csproj index 0614b7cf15a..b7dcd47a2eb 100644 --- a/firestore/api/ManageIndexes/ManageIndexes.csproj +++ b/firestore/api/ManageIndexes/ManageIndexes.csproj @@ -1,7 +1,7 @@ Exe - net6.0 + net8.0 GoogleCloudSamples.ManageIndexes diff --git a/firestore/api/OrderLimitData/OrderLimitData.csproj b/firestore/api/OrderLimitData/OrderLimitData.csproj index 89d2b98382c..14b1660109c 100644 --- a/firestore/api/OrderLimitData/OrderLimitData.csproj +++ b/firestore/api/OrderLimitData/OrderLimitData.csproj @@ -1,7 +1,7 @@ Exe - net6.0 + net8.0 GoogleCloudSamples.OrderLimitData diff --git a/firestore/api/PaginateData/PaginateData.csproj b/firestore/api/PaginateData/PaginateData.csproj index 8b3c270fe5b..57a8ea61104 100644 --- a/firestore/api/PaginateData/PaginateData.csproj +++ b/firestore/api/PaginateData/PaginateData.csproj @@ -1,7 +1,7 @@ Exe - net6.0 + net8.0 GoogleCloudSamples.PaginateData diff --git a/firestore/api/QueryData/QueryData.csproj b/firestore/api/QueryData/QueryData.csproj index 9269c3b6b1e..9a3618a5696 100644 --- a/firestore/api/QueryData/QueryData.csproj +++ b/firestore/api/QueryData/QueryData.csproj @@ -1,7 +1,7 @@ Exe - net6.0 + net8.0 GoogleCloudSamples.QueryData diff --git a/firestore/api/Quickstart/Quickstart.csproj b/firestore/api/Quickstart/Quickstart.csproj index 596215aaf14..51cae027f90 100644 --- a/firestore/api/Quickstart/Quickstart.csproj +++ b/firestore/api/Quickstart/Quickstart.csproj @@ -1,7 +1,7 @@ Exe - net6.0 + net8.0 GoogleCloudSamples.Quickstart diff --git a/firestore/api/SolutionCounter/SolutionCounter.csproj b/firestore/api/SolutionCounter/SolutionCounter.csproj index ac419949445..fb3d19d068a 100644 --- a/firestore/api/SolutionCounter/SolutionCounter.csproj +++ b/firestore/api/SolutionCounter/SolutionCounter.csproj @@ -1,7 +1,7 @@  Exe - net6.0 + net8.0 GoogleCloudSamples.DistributedCounter diff --git a/firestore/api/TransactionsAndBatchedWrites/TransactionsAndBatchedWrites.csproj b/firestore/api/TransactionsAndBatchedWrites/TransactionsAndBatchedWrites.csproj index 0f5722c8e92..8b42b971e87 100644 --- a/firestore/api/TransactionsAndBatchedWrites/TransactionsAndBatchedWrites.csproj +++ b/firestore/api/TransactionsAndBatchedWrites/TransactionsAndBatchedWrites.csproj @@ -1,7 +1,7 @@ Exe - net6.0 + net8.0 GoogleCloudSamples.TransactionsAndBatchedWrites diff --git a/iam/api/AccessTest/AccessTest.csproj b/iam/api/AccessTest/AccessTest.csproj index dcd300b3d1b..afca9f8810a 100644 --- a/iam/api/AccessTest/AccessTest.csproj +++ b/iam/api/AccessTest/AccessTest.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 false diff --git a/iam/api/CustomRoles/CustomRoles.csproj b/iam/api/CustomRoles/CustomRoles.csproj index 941b8749c0f..3121ab15fb7 100644 --- a/iam/api/CustomRoles/CustomRoles.csproj +++ b/iam/api/CustomRoles/CustomRoles.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 diff --git a/iam/api/CustomRolesTest/CustomRolesTest.csproj b/iam/api/CustomRolesTest/CustomRolesTest.csproj index 87e0faf12ac..c5c08b00c20 100644 --- a/iam/api/CustomRolesTest/CustomRolesTest.csproj +++ b/iam/api/CustomRolesTest/CustomRolesTest.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 false diff --git a/iam/api/QuickStart/QuickStart.csproj b/iam/api/QuickStart/QuickStart.csproj index f9bb4d218c6..74ea8602217 100644 --- a/iam/api/QuickStart/QuickStart.csproj +++ b/iam/api/QuickStart/QuickStart.csproj @@ -1,7 +1,7 @@ Exe - net6.0 + net8.0 diff --git a/iam/api/QuickStartTest/QuickStartTest.csproj b/iam/api/QuickStartTest/QuickStartTest.csproj index 4213223895d..20862271599 100644 --- a/iam/api/QuickStartTest/QuickStartTest.csproj +++ b/iam/api/QuickStartTest/QuickStartTest.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 false diff --git a/iam/api/ServiceAccountKeys/ServiceAccountKeys.csproj b/iam/api/ServiceAccountKeys/ServiceAccountKeys.csproj index 3f33039976a..5af75454772 100644 --- a/iam/api/ServiceAccountKeys/ServiceAccountKeys.csproj +++ b/iam/api/ServiceAccountKeys/ServiceAccountKeys.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 diff --git a/iam/api/ServiceAccountTests/ServiceAccountTests.csproj b/iam/api/ServiceAccountTests/ServiceAccountTests.csproj index 46ad5f91d2e..0969fe4cedd 100644 --- a/iam/api/ServiceAccountTests/ServiceAccountTests.csproj +++ b/iam/api/ServiceAccountTests/ServiceAccountTests.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 false diff --git a/iam/api/ServiceAccounts/ServiceAccounts.csproj b/iam/api/ServiceAccounts/ServiceAccounts.csproj index 367512fc982..d2e589674c0 100644 --- a/iam/api/ServiceAccounts/ServiceAccounts.csproj +++ b/iam/api/ServiceAccounts/ServiceAccounts.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 diff --git a/media/livestream/api/LiveStream.Samples.Tests/LiveStream.Samples.Tests.csproj b/media/livestream/api/LiveStream.Samples.Tests/LiveStream.Samples.Tests.csproj index e699a684378..a24ace8cc20 100644 --- a/media/livestream/api/LiveStream.Samples.Tests/LiveStream.Samples.Tests.csproj +++ b/media/livestream/api/LiveStream.Samples.Tests/LiveStream.Samples.Tests.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 false diff --git a/media/stitcher/api/Stitcher.Samples.Tests/Stitcher.Samples.Tests.csproj b/media/stitcher/api/Stitcher.Samples.Tests/Stitcher.Samples.Tests.csproj index f4a7678d690..bc7450d75a6 100644 --- a/media/stitcher/api/Stitcher.Samples.Tests/Stitcher.Samples.Tests.csproj +++ b/media/stitcher/api/Stitcher.Samples.Tests/Stitcher.Samples.Tests.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 false diff --git a/media/transcoder/api/Transcoder.Samples.Tests/Transcoder.Samples.Tests.csproj b/media/transcoder/api/Transcoder.Samples.Tests/Transcoder.Samples.Tests.csproj index 38357827e2f..ce060f91cae 100644 --- a/media/transcoder/api/Transcoder.Samples.Tests/Transcoder.Samples.Tests.csproj +++ b/media/transcoder/api/Transcoder.Samples.Tests/Transcoder.Samples.Tests.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 false diff --git a/monitoring/api/AlertSample/AlertSample.csproj b/monitoring/api/AlertSample/AlertSample.csproj index 22b005b0fac..2e67e8208e0 100644 --- a/monitoring/api/AlertSample/AlertSample.csproj +++ b/monitoring/api/AlertSample/AlertSample.csproj @@ -8,6 +8,6 @@ Exe - net6.0 + net8.0 \ No newline at end of file diff --git a/monitoring/api/AlertSnippets/AlertSnippets.csproj b/monitoring/api/AlertSnippets/AlertSnippets.csproj index 7189c62de0f..b79db2e6e3d 100644 --- a/monitoring/api/AlertSnippets/AlertSnippets.csproj +++ b/monitoring/api/AlertSnippets/AlertSnippets.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 false diff --git a/monitoring/api/AlertTest/AlertTest.csproj b/monitoring/api/AlertTest/AlertTest.csproj index 2ba2f66c2c1..c60535e8bff 100644 --- a/monitoring/api/AlertTest/AlertTest.csproj +++ b/monitoring/api/AlertTest/AlertTest.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 false diff --git a/monitoring/api/MonitoringSample/Monitoring.csproj b/monitoring/api/MonitoringSample/Monitoring.csproj index 245d16806a6..d2bfe5261f8 100644 --- a/monitoring/api/MonitoringSample/Monitoring.csproj +++ b/monitoring/api/MonitoringSample/Monitoring.csproj @@ -1,7 +1,7 @@  Exe - net6.0 + net8.0 GoogleCloudSamples.Monitoring diff --git a/monitoring/api/MonitoringTest/MonitoringTest.csproj b/monitoring/api/MonitoringTest/MonitoringTest.csproj index 48d0ad95b3c..59843ad24b0 100644 --- a/monitoring/api/MonitoringTest/MonitoringTest.csproj +++ b/monitoring/api/MonitoringTest/MonitoringTest.csproj @@ -1,7 +1,7 @@  Library - net6.0 + net8.0 diff --git a/monitoring/api/QuickStart/QuickStart.csproj b/monitoring/api/QuickStart/QuickStart.csproj index aecef2ae9c0..173bbc6efeb 100644 --- a/monitoring/api/QuickStart/QuickStart.csproj +++ b/monitoring/api/QuickStart/QuickStart.csproj @@ -1,7 +1,7 @@  Exe - net6.0 + net8.0 diff --git a/monitoring/api/UptimeCheckSample/UptimeCheck.csproj b/monitoring/api/UptimeCheckSample/UptimeCheck.csproj index 78d4160a3f0..cb85229cf87 100644 --- a/monitoring/api/UptimeCheckSample/UptimeCheck.csproj +++ b/monitoring/api/UptimeCheckSample/UptimeCheck.csproj @@ -2,7 +2,7 @@ Exe - net6.0 + net8.0 diff --git a/monitoring/api/UptimeCheckTest/UptimeCheckTest.csproj b/monitoring/api/UptimeCheckTest/UptimeCheckTest.csproj index 962d4abb110..a39d7aca8fd 100644 --- a/monitoring/api/UptimeCheckTest/UptimeCheckTest.csproj +++ b/monitoring/api/UptimeCheckTest/UptimeCheckTest.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 false diff --git a/pubsub/api/Pubsub.Samples.Tests/Pubsub.Samples.Tests.csproj b/pubsub/api/Pubsub.Samples.Tests/Pubsub.Samples.Tests.csproj index 7fb739e083f..f247d303e2b 100644 --- a/pubsub/api/Pubsub.Samples.Tests/Pubsub.Samples.Tests.csproj +++ b/pubsub/api/Pubsub.Samples.Tests/Pubsub.Samples.Tests.csproj @@ -1,7 +1,7 @@  - net6.0 + net8.0 false diff --git a/run/markdown-preview/Samples.Run.MarkdownPreview.Editor.Tests/Samples.Run.MarkdownPreview.Editor.Tests.csproj b/run/markdown-preview/Samples.Run.MarkdownPreview.Editor.Tests/Samples.Run.MarkdownPreview.Editor.Tests.csproj index fab7493f865..05dc19af3c2 100644 --- a/run/markdown-preview/Samples.Run.MarkdownPreview.Editor.Tests/Samples.Run.MarkdownPreview.Editor.Tests.csproj +++ b/run/markdown-preview/Samples.Run.MarkdownPreview.Editor.Tests/Samples.Run.MarkdownPreview.Editor.Tests.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 enable false diff --git a/run/markdown-preview/Samples.Run.MarkdownPreview.Editor/Samples.Run.MarkdownPreview.Editor.csproj b/run/markdown-preview/Samples.Run.MarkdownPreview.Editor/Samples.Run.MarkdownPreview.Editor.csproj index 0a109f23e10..ebc9525b85f 100644 --- a/run/markdown-preview/Samples.Run.MarkdownPreview.Editor/Samples.Run.MarkdownPreview.Editor.csproj +++ b/run/markdown-preview/Samples.Run.MarkdownPreview.Editor/Samples.Run.MarkdownPreview.Editor.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 enable enable editor diff --git a/run/markdown-preview/Samples.Run.MarkdownPreview.Renderer.Tests/Samples.Run.MarkdownPreview.Renderer.Tests.csproj b/run/markdown-preview/Samples.Run.MarkdownPreview.Renderer.Tests/Samples.Run.MarkdownPreview.Renderer.Tests.csproj index c8e7b368525..5b86cefc3d8 100644 --- a/run/markdown-preview/Samples.Run.MarkdownPreview.Renderer.Tests/Samples.Run.MarkdownPreview.Renderer.Tests.csproj +++ b/run/markdown-preview/Samples.Run.MarkdownPreview.Renderer.Tests/Samples.Run.MarkdownPreview.Renderer.Tests.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 enable false diff --git a/run/markdown-preview/Samples.Run.MarkdownPreview.Renderer/Samples.Run.MarkdownPreview.Renderer.csproj b/run/markdown-preview/Samples.Run.MarkdownPreview.Renderer/Samples.Run.MarkdownPreview.Renderer.csproj index 36380216e0b..352dbc6f1b6 100644 --- a/run/markdown-preview/Samples.Run.MarkdownPreview.Renderer/Samples.Run.MarkdownPreview.Renderer.csproj +++ b/run/markdown-preview/Samples.Run.MarkdownPreview.Renderer/Samples.Run.MarkdownPreview.Renderer.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 enable enable renderer diff --git a/run/pubsub/Run.Samples.Pubsub.MinimalApi.Tests/Run.Samples.Pubsub.MinimalApi.Tests.csproj b/run/pubsub/Run.Samples.Pubsub.MinimalApi.Tests/Run.Samples.Pubsub.MinimalApi.Tests.csproj index 704d499e575..33f58980afd 100644 --- a/run/pubsub/Run.Samples.Pubsub.MinimalApi.Tests/Run.Samples.Pubsub.MinimalApi.Tests.csproj +++ b/run/pubsub/Run.Samples.Pubsub.MinimalApi.Tests/Run.Samples.Pubsub.MinimalApi.Tests.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 enable false diff --git a/run/pubsub/Run.Samples.Pubsub.MinimalApi/Run.Samples.Pubsub.MinimalApi.csproj b/run/pubsub/Run.Samples.Pubsub.MinimalApi/Run.Samples.Pubsub.MinimalApi.csproj index b775b0957d0..13d81270429 100644 --- a/run/pubsub/Run.Samples.Pubsub.MinimalApi/Run.Samples.Pubsub.MinimalApi.csproj +++ b/run/pubsub/Run.Samples.Pubsub.MinimalApi/Run.Samples.Pubsub.MinimalApi.csproj @@ -1,7 +1,7 @@ - net6.0 + net8.0 enable enable diff --git a/storage/api/Storage.Samples.Tests/Storage.Samples.Tests.csproj b/storage/api/Storage.Samples.Tests/Storage.Samples.Tests.csproj index 1df5bdd5d39..b0df613426e 100644 --- a/storage/api/Storage.Samples.Tests/Storage.Samples.Tests.csproj +++ b/storage/api/Storage.Samples.Tests/Storage.Samples.Tests.csproj @@ -1,7 +1,7 @@  - net6.0 + net8.0 false diff --git a/storageinsights/api/StorageInsights.Samples.Tests/StorageInsights.Samples.Tests.csproj b/storageinsights/api/StorageInsights.Samples.Tests/StorageInsights.Samples.Tests.csproj index 66613d3a449..b0fbc606b60 100644 --- a/storageinsights/api/StorageInsights.Samples.Tests/StorageInsights.Samples.Tests.csproj +++ b/storageinsights/api/StorageInsights.Samples.Tests/StorageInsights.Samples.Tests.csproj @@ -2,7 +2,7 @@ - net6.0 + net8.0 false From 4a253df8e53f51c33cf4795a5252e49f2847b5c4 Mon Sep 17 00:00:00 2001 From: Amanda Tarafa Mas Date: Wed, 21 May 2025 00:06:00 -0700 Subject: [PATCH 51/58] tests(Asset): Fix tests that was relying on a specific IAM policy --- asset/api/Asset.Samples.Tests/SearchAllIamPoliciesTest.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/asset/api/Asset.Samples.Tests/SearchAllIamPoliciesTest.cs b/asset/api/Asset.Samples.Tests/SearchAllIamPoliciesTest.cs index 279fe72711e..1a9b283bbd8 100644 --- a/asset/api/Asset.Samples.Tests/SearchAllIamPoliciesTest.cs +++ b/asset/api/Asset.Samples.Tests/SearchAllIamPoliciesTest.cs @@ -34,7 +34,8 @@ public void TestSearchAllIamPolicies() { // Run the sample code. string scope = $"projects/{_fixture.ProjectId}"; - var result = _sample.SearchAllIamPolicies(scope, query: "policy:roles/owner"); + // On the test we use an empty query to make sure we find some policy. + var result = _sample.SearchAllIamPolicies(scope, query: ""); Assert.NotEmpty(result.Results); } From 77c19b6e621010b1c2130ec4fffd939cd4751a67 Mon Sep 17 00:00:00 2001 From: Amanda Tarafa Mas Date: Wed, 21 May 2025 00:18:19 -0700 Subject: [PATCH 52/58] fix(PubSub): Pass a DateTimeOffset instead of a string We were parsing the string to a DateTime before anyway. --- .../CreateTopicWithCloudStorageIngestionTest.cs | 4 ++-- .../CreateTopicWithCloudStorageIngestion.cs | 13 ++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/pubsub/api/Pubsub.Samples.Tests/CreateTopicWithCloudStorageIngestionTest.cs b/pubsub/api/Pubsub.Samples.Tests/CreateTopicWithCloudStorageIngestionTest.cs index 18ae6a5a640..5339f5bba85 100644 --- a/pubsub/api/Pubsub.Samples.Tests/CreateTopicWithCloudStorageIngestionTest.cs +++ b/pubsub/api/Pubsub.Samples.Tests/CreateTopicWithCloudStorageIngestionTest.cs @@ -37,7 +37,7 @@ public void CreateTopicWithCloudStorageIngestion() string inputFormat = "text"; string textDelimiter = "\n"; string matchGlob = "**.txt"; - string minimumObjectCreateTime = "1970-01-01T00:00:00Z"; + DateTimeOffset minimumObjectCreateTime = new DateTimeOffset(1970, 1, 1, 0, 0, 0, TimeSpan.Zero); Topic createdTopic = _createTopicWithCloudStorageIngestionSample.CreateTopicWithCloudStorageIngestion(_pubsubFixture.ProjectId, topicId, bucket, inputFormat, textDelimiter, matchGlob, minimumObjectCreateTime); // Confirm that the created topic and topic retrieved by ID are equal @@ -49,6 +49,6 @@ public void CreateTopicWithCloudStorageIngestion() Assert.NotNull(createdTopic.IngestionDataSourceSettings.CloudStorage.TextFormat); Assert.Equal(textDelimiter, createdTopic.IngestionDataSourceSettings.CloudStorage.TextFormat.Delimiter); Assert.Equal(matchGlob, createdTopic.IngestionDataSourceSettings.CloudStorage.MatchGlob); - Assert.Equal(Timestamp.FromDateTime(DateTime.Parse(minimumObjectCreateTime)), createdTopic.IngestionDataSourceSettings.CloudStorage.MinimumObjectCreateTime); + Assert.Equal(Timestamp.FromDateTimeOffset(minimumObjectCreateTime), createdTopic.IngestionDataSourceSettings.CloudStorage.MinimumObjectCreateTime); } } diff --git a/pubsub/api/Pubsub.Samples/CreateTopicWithCloudStorageIngestion.cs b/pubsub/api/Pubsub.Samples/CreateTopicWithCloudStorageIngestion.cs index 0a0c1f1d3e1..bf12c730f8a 100644 --- a/pubsub/api/Pubsub.Samples/CreateTopicWithCloudStorageIngestion.cs +++ b/pubsub/api/Pubsub.Samples/CreateTopicWithCloudStorageIngestion.cs @@ -21,10 +21,14 @@ public class CreateTopicWithCloudStorageIngestionSample { - public Topic CreateTopicWithCloudStorageIngestion(string projectId, string topicId, string bucket, string inputFormat, string textDelimiter, string matchGlob, string minimumObjectCreateTime) + public Topic CreateTopicWithCloudStorageIngestion(string projectId, string topicId, string bucket, string inputFormat, string textDelimiter, string matchGlob, DateTimeOffset minimumObjectCreateTime) { - IngestionDataSourceSettings.Types.CloudStorage cloudStorageSettings = new IngestionDataSourceSettings.Types.CloudStorage { Bucket = bucket }; + IngestionDataSourceSettings.Types.CloudStorage cloudStorageSettings = new IngestionDataSourceSettings.Types.CloudStorage + { + Bucket = bucket, + MinimumObjectCreateTime = Timestamp.FromDateTimeOffset(minimumObjectCreateTime), + }; switch (inputFormat) { @@ -49,11 +53,6 @@ public Topic CreateTopicWithCloudStorageIngestion(string projectId, string topic cloudStorageSettings.MatchGlob = matchGlob; } - if (!string.IsNullOrEmpty(minimumObjectCreateTime)) - { - cloudStorageSettings.MinimumObjectCreateTime = Timestamp.FromDateTime(DateTime.Parse(minimumObjectCreateTime)); - } - PublisherServiceApiClient publisher = PublisherServiceApiClient.Create(); Topic topic = new Topic() { From 7906de39be524fe852cd4829c7806971f547ccfc Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Wed, 21 May 2025 17:54:59 +0000 Subject: [PATCH 53/58] chore(deps): update dependency microsoft.net.test.sdk to 17.14.0 --- .../AIPlatform.Samples.Tests/AIPlatform.Samples.Tests.csproj | 2 +- .../flexible/Analytics/Analytics.Tests/Analytics.Tests.csproj | 2 +- .../CloudStorage/CloudStorage.Tests/CloudStorage.Tests.csproj | 2 +- .../flexible/Datastore/Datastore.Tests/Datastore.Tests.csproj | 2 +- .../HelloWorld/HelloWorld.Tests/HelloWorld.Tests.csproj | 2 +- .../flexible/Metadata/Metadata.Tests/Metadata.Tests.csproj | 2 +- appengine/flexible/Pubsub/Pubsub.Tests/Pubsub.Tests.csproj | 2 +- .../StaticContent.Tests/StaticContent.Tests.csproj | 2 +- .../flexible/WebSocket/WebSocket.Tests/WebSocket.Tests.csproj | 2 +- applications/leaderboard/LeaderboardTest/LeaderboardTest.csproj | 2 +- asset/api/Asset.Samples.Tests/Asset.Samples.Tests.csproj | 2 +- auth/AuthTest/AuthTest.csproj | 2 +- .../BigQueryStorage.Samples.Tests.csproj | 2 +- bigquery/api/SimpleApp/SimpleApp.csproj | 2 +- bigquery/api/Snippets/Snippets.csproj | 2 +- bigquery/data-transfer/api/QuickStartTest/QuickStartTest.csproj | 2 +- bigtable/api/BigtableTest/BigtableTest.csproj | 2 +- bigtable/api/SnippetsTest/SnippetsTest.csproj | 2 +- cdn/signUrl/SignUrlSample.Tests/SignUrlSample.Tests.csproj | 2 +- cloudtasks/api/TasksSample/TasksSample.csproj | 2 +- commandlineutil/SampleTest/SampleTest.csproj | 2 +- compute/api/Compute.Samples.Tests/Compute.Samples.Tests.csproj | 2 +- .../DatastoreAdmin.Samples.Tests.csproj | 2 +- datastore/api/QuickStartTest/QuickStartTest.csproj | 2 +- datastore/api/TaskListTest/DatastoreTest.csproj | 2 +- dlp/api/Snippets.Tests/Snippets.Tests.csproj | 2 +- .../DocumentAI.Samples.Tests/DocumentAI.Samples.Tests.csproj | 2 +- .../ErrorReportingSampleTests/ErrorReportingSampleTests.csproj | 2 +- firestore/api/FirestoreTest/FirestoreTest.csproj | 2 +- functions/concepts/Concepts.Tests/Concepts.Tests.csproj | 2 +- functions/firebase/Firebase.Tests/Firebase.Tests.csproj | 2 +- functions/helloworld/HelloWorld.Tests/HelloWorld.Tests.csproj | 2 +- functions/http/Http.Tests/Http.Tests.csproj | 2 +- functions/logging/Logging.Tests/Logging.Tests.csproj | 2 +- .../StreamBigQuery.Tests/StreamBigQuery.Tests.csproj | 2 +- functions/typed/Typed.Tests/Typed.Tests.csproj | 2 +- iam/api/AccessTest/AccessTest.csproj | 2 +- iam/api/CustomRolesTest/CustomRolesTest.csproj | 2 +- iam/api/QuickStartTest/QuickStartTest.csproj | 2 +- iam/api/ServiceAccountTests/ServiceAccountTests.csproj | 2 +- identity-aware-proxy/IAPClientTest/IAPClientTest.csproj | 2 +- kms/api/Kms.Samples.Tests/Kms.Samples.Tests.csproj | 2 +- logging/api/LoggingTest/LoggingTest.csproj | 2 +- .../LiveStream.Samples.Tests/LiveStream.Samples.Tests.csproj | 2 +- .../api/Stitcher.Samples.Tests/Stitcher.Samples.Tests.csproj | 2 +- .../Transcoder.Samples.Tests/Transcoder.Samples.Tests.csproj | 2 +- monitoring/api/AlertSnippets/AlertSnippets.csproj | 2 +- monitoring/api/AlertTest/AlertTest.csproj | 2 +- monitoring/api/MonitoringTest/MonitoringTest.csproj | 2 +- monitoring/api/UptimeCheckTest/UptimeCheckTest.csproj | 2 +- .../ParameterManager.RegionalSamples.Tests.csproj | 2 +- .../ParameterManager.Samples.Tests.csproj | 2 +- pubsub/api/Pubsub.Samples.Tests/Pubsub.Samples.Tests.csproj | 2 +- .../RetailEvents.Samples.Tests.csproj | 2 +- .../RetailProducts.Samples.Tests.csproj | 2 +- .../RetailSearch.Samples.Tests.csproj | 2 +- .../Samples.Run.MarkdownPreview.Editor.Tests.csproj | 2 +- .../Samples.Run.MarkdownPreview.Renderer.Tests.csproj | 2 +- .../Run.Samples.Pubsub.MinimalApi.Tests.csproj | 2 +- .../SecretManager.Samples.Tests.csproj | 2 +- .../SecurityCenter.Samples.Tests.csproj | 2 +- .../ServiceDirectory.Samples.Tests.csproj | 2 +- spanner/api/Spanner.Samples.Tests/Spanner.Samples.Tests.csproj | 2 +- storage/api/Storage.Samples.Tests/Storage.Samples.Tests.csproj | 2 +- .../StorageInsights.Samples.Tests.csproj | 2 +- .../StorageTransfer.Samples.Tests.csproj | 2 +- .../api/Workflow.Samples.Test/Workflow.Samples.Test.csproj | 2 +- 67 files changed, 67 insertions(+), 67 deletions(-) diff --git a/aiplatform/api/AIPlatform.Samples.Tests/AIPlatform.Samples.Tests.csproj b/aiplatform/api/AIPlatform.Samples.Tests/AIPlatform.Samples.Tests.csproj index 9e6ce533a02..b2633564cf3 100644 --- a/aiplatform/api/AIPlatform.Samples.Tests/AIPlatform.Samples.Tests.csproj +++ b/aiplatform/api/AIPlatform.Samples.Tests/AIPlatform.Samples.Tests.csproj @@ -6,7 +6,7 @@ - + diff --git a/appengine/flexible/Analytics/Analytics.Tests/Analytics.Tests.csproj b/appengine/flexible/Analytics/Analytics.Tests/Analytics.Tests.csproj index e7917358757..ce75b2c38f5 100644 --- a/appengine/flexible/Analytics/Analytics.Tests/Analytics.Tests.csproj +++ b/appengine/flexible/Analytics/Analytics.Tests/Analytics.Tests.csproj @@ -10,7 +10,7 @@ - + diff --git a/appengine/flexible/CloudStorage/CloudStorage.Tests/CloudStorage.Tests.csproj b/appengine/flexible/CloudStorage/CloudStorage.Tests/CloudStorage.Tests.csproj index 586887fa83d..4c4c476bffc 100644 --- a/appengine/flexible/CloudStorage/CloudStorage.Tests/CloudStorage.Tests.csproj +++ b/appengine/flexible/CloudStorage/CloudStorage.Tests/CloudStorage.Tests.csproj @@ -10,7 +10,7 @@ - + diff --git a/appengine/flexible/Datastore/Datastore.Tests/Datastore.Tests.csproj b/appengine/flexible/Datastore/Datastore.Tests/Datastore.Tests.csproj index ecb4783768b..220067f8ff3 100644 --- a/appengine/flexible/Datastore/Datastore.Tests/Datastore.Tests.csproj +++ b/appengine/flexible/Datastore/Datastore.Tests/Datastore.Tests.csproj @@ -10,7 +10,7 @@ - + diff --git a/appengine/flexible/HelloWorld/HelloWorld.Tests/HelloWorld.Tests.csproj b/appengine/flexible/HelloWorld/HelloWorld.Tests/HelloWorld.Tests.csproj index 524883dab39..641e5b3263a 100644 --- a/appengine/flexible/HelloWorld/HelloWorld.Tests/HelloWorld.Tests.csproj +++ b/appengine/flexible/HelloWorld/HelloWorld.Tests/HelloWorld.Tests.csproj @@ -10,7 +10,7 @@ - + diff --git a/appengine/flexible/Metadata/Metadata.Tests/Metadata.Tests.csproj b/appengine/flexible/Metadata/Metadata.Tests/Metadata.Tests.csproj index 7f018255c6b..728e3164cb7 100644 --- a/appengine/flexible/Metadata/Metadata.Tests/Metadata.Tests.csproj +++ b/appengine/flexible/Metadata/Metadata.Tests/Metadata.Tests.csproj @@ -10,7 +10,7 @@ - + diff --git a/appengine/flexible/Pubsub/Pubsub.Tests/Pubsub.Tests.csproj b/appengine/flexible/Pubsub/Pubsub.Tests/Pubsub.Tests.csproj index 1e9296a0589..759c19a3417 100644 --- a/appengine/flexible/Pubsub/Pubsub.Tests/Pubsub.Tests.csproj +++ b/appengine/flexible/Pubsub/Pubsub.Tests/Pubsub.Tests.csproj @@ -10,7 +10,7 @@ - + diff --git a/appengine/flexible/StaticContent/StaticContent.Tests/StaticContent.Tests.csproj b/appengine/flexible/StaticContent/StaticContent.Tests/StaticContent.Tests.csproj index d9a1079b316..2bd3366fe47 100644 --- a/appengine/flexible/StaticContent/StaticContent.Tests/StaticContent.Tests.csproj +++ b/appengine/flexible/StaticContent/StaticContent.Tests/StaticContent.Tests.csproj @@ -10,7 +10,7 @@ - + diff --git a/appengine/flexible/WebSocket/WebSocket.Tests/WebSocket.Tests.csproj b/appengine/flexible/WebSocket/WebSocket.Tests/WebSocket.Tests.csproj index 8490962425f..a57d8482ef5 100644 --- a/appengine/flexible/WebSocket/WebSocket.Tests/WebSocket.Tests.csproj +++ b/appengine/flexible/WebSocket/WebSocket.Tests/WebSocket.Tests.csproj @@ -10,7 +10,7 @@ - + diff --git a/applications/leaderboard/LeaderboardTest/LeaderboardTest.csproj b/applications/leaderboard/LeaderboardTest/LeaderboardTest.csproj index fd682d16419..ecab5eb0665 100644 --- a/applications/leaderboard/LeaderboardTest/LeaderboardTest.csproj +++ b/applications/leaderboard/LeaderboardTest/LeaderboardTest.csproj @@ -8,7 +8,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/asset/api/Asset.Samples.Tests/Asset.Samples.Tests.csproj b/asset/api/Asset.Samples.Tests/Asset.Samples.Tests.csproj index 59a45748ff4..236cc9d0bca 100644 --- a/asset/api/Asset.Samples.Tests/Asset.Samples.Tests.csproj +++ b/asset/api/Asset.Samples.Tests/Asset.Samples.Tests.csproj @@ -8,7 +8,7 @@ - + diff --git a/auth/AuthTest/AuthTest.csproj b/auth/AuthTest/AuthTest.csproj index 2eb59d8d3e2..d370c59652e 100644 --- a/auth/AuthTest/AuthTest.csproj +++ b/auth/AuthTest/AuthTest.csproj @@ -5,7 +5,7 @@ - + runtime; build; native; contentfiles; analyzers all diff --git a/bigquery-storage/api/BigQueryStorage.Samples.Tests/BigQueryStorage.Samples.Tests.csproj b/bigquery-storage/api/BigQueryStorage.Samples.Tests/BigQueryStorage.Samples.Tests.csproj index d9797b85a13..dc260170f77 100644 --- a/bigquery-storage/api/BigQueryStorage.Samples.Tests/BigQueryStorage.Samples.Tests.csproj +++ b/bigquery-storage/api/BigQueryStorage.Samples.Tests/BigQueryStorage.Samples.Tests.csproj @@ -6,7 +6,7 @@ - + diff --git a/bigquery/api/SimpleApp/SimpleApp.csproj b/bigquery/api/SimpleApp/SimpleApp.csproj index 6362f05e3d9..7bfa2055c83 100644 --- a/bigquery/api/SimpleApp/SimpleApp.csproj +++ b/bigquery/api/SimpleApp/SimpleApp.csproj @@ -9,7 +9,7 @@ - + runtime; build; native; contentfiles; analyzers all diff --git a/bigquery/api/Snippets/Snippets.csproj b/bigquery/api/Snippets/Snippets.csproj index bda8622e69b..a95c527927c 100644 --- a/bigquery/api/Snippets/Snippets.csproj +++ b/bigquery/api/Snippets/Snippets.csproj @@ -7,7 +7,7 @@ - + runtime; build; native; contentfiles; analyzers all diff --git a/bigquery/data-transfer/api/QuickStartTest/QuickStartTest.csproj b/bigquery/data-transfer/api/QuickStartTest/QuickStartTest.csproj index 4c6274b9794..6c964cbd0b4 100644 --- a/bigquery/data-transfer/api/QuickStartTest/QuickStartTest.csproj +++ b/bigquery/data-transfer/api/QuickStartTest/QuickStartTest.csproj @@ -8,7 +8,7 @@ - + runtime; build; native; contentfiles; analyzers diff --git a/bigtable/api/BigtableTest/BigtableTest.csproj b/bigtable/api/BigtableTest/BigtableTest.csproj index 99678b6efe6..17eb7787fa1 100644 --- a/bigtable/api/BigtableTest/BigtableTest.csproj +++ b/bigtable/api/BigtableTest/BigtableTest.csproj @@ -8,7 +8,7 @@ - + runtime; build; native; contentfiles; analyzers all diff --git a/bigtable/api/SnippetsTest/SnippetsTest.csproj b/bigtable/api/SnippetsTest/SnippetsTest.csproj index 603bf47829f..5f9eb8a5728 100644 --- a/bigtable/api/SnippetsTest/SnippetsTest.csproj +++ b/bigtable/api/SnippetsTest/SnippetsTest.csproj @@ -9,7 +9,7 @@ - + diff --git a/cdn/signUrl/SignUrlSample.Tests/SignUrlSample.Tests.csproj b/cdn/signUrl/SignUrlSample.Tests/SignUrlSample.Tests.csproj index 69a14940ab5..af1e51d36d6 100644 --- a/cdn/signUrl/SignUrlSample.Tests/SignUrlSample.Tests.csproj +++ b/cdn/signUrl/SignUrlSample.Tests/SignUrlSample.Tests.csproj @@ -8,7 +8,7 @@ - + runtime; build; native; contentfiles; analyzers diff --git a/cloudtasks/api/TasksSample/TasksSample.csproj b/cloudtasks/api/TasksSample/TasksSample.csproj index f65babd186f..67152b284f5 100644 --- a/cloudtasks/api/TasksSample/TasksSample.csproj +++ b/cloudtasks/api/TasksSample/TasksSample.csproj @@ -7,7 +7,7 @@ - + diff --git a/commandlineutil/SampleTest/SampleTest.csproj b/commandlineutil/SampleTest/SampleTest.csproj index 43f1278591c..c86706559d2 100644 --- a/commandlineutil/SampleTest/SampleTest.csproj +++ b/commandlineutil/SampleTest/SampleTest.csproj @@ -8,7 +8,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/compute/api/Compute.Samples.Tests/Compute.Samples.Tests.csproj b/compute/api/Compute.Samples.Tests/Compute.Samples.Tests.csproj index c1fa7789b91..fc61d9bc9c0 100644 --- a/compute/api/Compute.Samples.Tests/Compute.Samples.Tests.csproj +++ b/compute/api/Compute.Samples.Tests/Compute.Samples.Tests.csproj @@ -9,7 +9,7 @@ - + diff --git a/datastore/admin-api/DatastoreAdmin.Samples.Tests/DatastoreAdmin.Samples.Tests.csproj b/datastore/admin-api/DatastoreAdmin.Samples.Tests/DatastoreAdmin.Samples.Tests.csproj index 4554bcf0d57..2f2d1e661d9 100644 --- a/datastore/admin-api/DatastoreAdmin.Samples.Tests/DatastoreAdmin.Samples.Tests.csproj +++ b/datastore/admin-api/DatastoreAdmin.Samples.Tests/DatastoreAdmin.Samples.Tests.csproj @@ -8,7 +8,7 @@ - + diff --git a/datastore/api/QuickStartTest/QuickStartTest.csproj b/datastore/api/QuickStartTest/QuickStartTest.csproj index 407e48a382e..c5f037f5bbe 100644 --- a/datastore/api/QuickStartTest/QuickStartTest.csproj +++ b/datastore/api/QuickStartTest/QuickStartTest.csproj @@ -8,7 +8,7 @@ - + runtime; build; native; contentfiles; analyzers all diff --git a/datastore/api/TaskListTest/DatastoreTest.csproj b/datastore/api/TaskListTest/DatastoreTest.csproj index f908efb12c5..f122f7b083a 100644 --- a/datastore/api/TaskListTest/DatastoreTest.csproj +++ b/datastore/api/TaskListTest/DatastoreTest.csproj @@ -5,7 +5,7 @@ - + runtime; build; native; contentfiles; analyzers all diff --git a/dlp/api/Snippets.Tests/Snippets.Tests.csproj b/dlp/api/Snippets.Tests/Snippets.Tests.csproj index 3831a626067..3347f89d8f6 100644 --- a/dlp/api/Snippets.Tests/Snippets.Tests.csproj +++ b/dlp/api/Snippets.Tests/Snippets.Tests.csproj @@ -5,7 +5,7 @@ - + all diff --git a/documentai/api/DocumentAI.Samples.Tests/DocumentAI.Samples.Tests.csproj b/documentai/api/DocumentAI.Samples.Tests/DocumentAI.Samples.Tests.csproj index ffd9707da07..54a8eb5a926 100644 --- a/documentai/api/DocumentAI.Samples.Tests/DocumentAI.Samples.Tests.csproj +++ b/documentai/api/DocumentAI.Samples.Tests/DocumentAI.Samples.Tests.csproj @@ -6,7 +6,7 @@ - + diff --git a/error-reporting/api/ErrorReportingSampleTests/ErrorReportingSampleTests.csproj b/error-reporting/api/ErrorReportingSampleTests/ErrorReportingSampleTests.csproj index e917e2d381a..32f503c61e8 100644 --- a/error-reporting/api/ErrorReportingSampleTests/ErrorReportingSampleTests.csproj +++ b/error-reporting/api/ErrorReportingSampleTests/ErrorReportingSampleTests.csproj @@ -8,7 +8,7 @@ - + runtime; build; native; contentfiles; analyzers diff --git a/firestore/api/FirestoreTest/FirestoreTest.csproj b/firestore/api/FirestoreTest/FirestoreTest.csproj index ac6cfa15827..a3c11859f6c 100644 --- a/firestore/api/FirestoreTest/FirestoreTest.csproj +++ b/firestore/api/FirestoreTest/FirestoreTest.csproj @@ -6,7 +6,7 @@ - + runtime; build; native; contentfiles; analyzers all diff --git a/functions/concepts/Concepts.Tests/Concepts.Tests.csproj b/functions/concepts/Concepts.Tests/Concepts.Tests.csproj index 3310ab767da..4220336ce34 100644 --- a/functions/concepts/Concepts.Tests/Concepts.Tests.csproj +++ b/functions/concepts/Concepts.Tests/Concepts.Tests.csproj @@ -9,7 +9,7 @@ - + diff --git a/functions/firebase/Firebase.Tests/Firebase.Tests.csproj b/functions/firebase/Firebase.Tests/Firebase.Tests.csproj index 8a3a00dee04..fc8f00f9fff 100644 --- a/functions/firebase/Firebase.Tests/Firebase.Tests.csproj +++ b/functions/firebase/Firebase.Tests/Firebase.Tests.csproj @@ -9,7 +9,7 @@ - + diff --git a/functions/helloworld/HelloWorld.Tests/HelloWorld.Tests.csproj b/functions/helloworld/HelloWorld.Tests/HelloWorld.Tests.csproj index 1273a5ed38c..68e3bf3f02c 100644 --- a/functions/helloworld/HelloWorld.Tests/HelloWorld.Tests.csproj +++ b/functions/helloworld/HelloWorld.Tests/HelloWorld.Tests.csproj @@ -9,7 +9,7 @@ - + diff --git a/functions/http/Http.Tests/Http.Tests.csproj b/functions/http/Http.Tests/Http.Tests.csproj index ff06f627f42..706334569cb 100644 --- a/functions/http/Http.Tests/Http.Tests.csproj +++ b/functions/http/Http.Tests/Http.Tests.csproj @@ -8,7 +8,7 @@ - + diff --git a/functions/logging/Logging.Tests/Logging.Tests.csproj b/functions/logging/Logging.Tests/Logging.Tests.csproj index 6a34cc08056..7930baa0ec1 100644 --- a/functions/logging/Logging.Tests/Logging.Tests.csproj +++ b/functions/logging/Logging.Tests/Logging.Tests.csproj @@ -8,7 +8,7 @@ - + diff --git a/functions/responsestreaming/StreamBigQuery.Tests/StreamBigQuery.Tests.csproj b/functions/responsestreaming/StreamBigQuery.Tests/StreamBigQuery.Tests.csproj index e9014785286..37d7e2c0b8b 100644 --- a/functions/responsestreaming/StreamBigQuery.Tests/StreamBigQuery.Tests.csproj +++ b/functions/responsestreaming/StreamBigQuery.Tests/StreamBigQuery.Tests.csproj @@ -8,7 +8,7 @@ - + diff --git a/functions/typed/Typed.Tests/Typed.Tests.csproj b/functions/typed/Typed.Tests/Typed.Tests.csproj index 4742c6c8beb..fe1c1a9ddbc 100644 --- a/functions/typed/Typed.Tests/Typed.Tests.csproj +++ b/functions/typed/Typed.Tests/Typed.Tests.csproj @@ -8,7 +8,7 @@ - + diff --git a/iam/api/AccessTest/AccessTest.csproj b/iam/api/AccessTest/AccessTest.csproj index afca9f8810a..f0f982c6b6a 100644 --- a/iam/api/AccessTest/AccessTest.csproj +++ b/iam/api/AccessTest/AccessTest.csproj @@ -8,7 +8,7 @@ - + runtime; build; native; contentfiles; analyzers all diff --git a/iam/api/CustomRolesTest/CustomRolesTest.csproj b/iam/api/CustomRolesTest/CustomRolesTest.csproj index c5c08b00c20..67092cec778 100644 --- a/iam/api/CustomRolesTest/CustomRolesTest.csproj +++ b/iam/api/CustomRolesTest/CustomRolesTest.csproj @@ -7,7 +7,7 @@ - + runtime; build; native; contentfiles; analyzers all diff --git a/iam/api/QuickStartTest/QuickStartTest.csproj b/iam/api/QuickStartTest/QuickStartTest.csproj index 20862271599..c6d1373c3e3 100644 --- a/iam/api/QuickStartTest/QuickStartTest.csproj +++ b/iam/api/QuickStartTest/QuickStartTest.csproj @@ -7,7 +7,7 @@ - + runtime; build; native; contentfiles; analyzers diff --git a/iam/api/ServiceAccountTests/ServiceAccountTests.csproj b/iam/api/ServiceAccountTests/ServiceAccountTests.csproj index 0969fe4cedd..df694208bee 100644 --- a/iam/api/ServiceAccountTests/ServiceAccountTests.csproj +++ b/iam/api/ServiceAccountTests/ServiceAccountTests.csproj @@ -7,7 +7,7 @@ - + runtime; build; native; contentfiles; analyzers all diff --git a/identity-aware-proxy/IAPClientTest/IAPClientTest.csproj b/identity-aware-proxy/IAPClientTest/IAPClientTest.csproj index 2ca73213396..b6bc77624cd 100644 --- a/identity-aware-proxy/IAPClientTest/IAPClientTest.csproj +++ b/identity-aware-proxy/IAPClientTest/IAPClientTest.csproj @@ -9,7 +9,7 @@ - + runtime; build; native; contentfiles; analyzers diff --git a/kms/api/Kms.Samples.Tests/Kms.Samples.Tests.csproj b/kms/api/Kms.Samples.Tests/Kms.Samples.Tests.csproj index 86ea65067e3..a3b31cfa553 100644 --- a/kms/api/Kms.Samples.Tests/Kms.Samples.Tests.csproj +++ b/kms/api/Kms.Samples.Tests/Kms.Samples.Tests.csproj @@ -6,7 +6,7 @@ - + diff --git a/logging/api/LoggingTest/LoggingTest.csproj b/logging/api/LoggingTest/LoggingTest.csproj index bd8d663fa2c..63933b6cade 100644 --- a/logging/api/LoggingTest/LoggingTest.csproj +++ b/logging/api/LoggingTest/LoggingTest.csproj @@ -8,7 +8,7 @@ - + runtime; build; native; contentfiles; analyzers all diff --git a/media/livestream/api/LiveStream.Samples.Tests/LiveStream.Samples.Tests.csproj b/media/livestream/api/LiveStream.Samples.Tests/LiveStream.Samples.Tests.csproj index a24ace8cc20..b965b766644 100644 --- a/media/livestream/api/LiveStream.Samples.Tests/LiveStream.Samples.Tests.csproj +++ b/media/livestream/api/LiveStream.Samples.Tests/LiveStream.Samples.Tests.csproj @@ -7,7 +7,7 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/media/stitcher/api/Stitcher.Samples.Tests/Stitcher.Samples.Tests.csproj b/media/stitcher/api/Stitcher.Samples.Tests/Stitcher.Samples.Tests.csproj index bc7450d75a6..ab16640463e 100644 --- a/media/stitcher/api/Stitcher.Samples.Tests/Stitcher.Samples.Tests.csproj +++ b/media/stitcher/api/Stitcher.Samples.Tests/Stitcher.Samples.Tests.csproj @@ -7,7 +7,7 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/media/transcoder/api/Transcoder.Samples.Tests/Transcoder.Samples.Tests.csproj b/media/transcoder/api/Transcoder.Samples.Tests/Transcoder.Samples.Tests.csproj index ce060f91cae..10a4f17d3e6 100644 --- a/media/transcoder/api/Transcoder.Samples.Tests/Transcoder.Samples.Tests.csproj +++ b/media/transcoder/api/Transcoder.Samples.Tests/Transcoder.Samples.Tests.csproj @@ -7,7 +7,7 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/monitoring/api/AlertSnippets/AlertSnippets.csproj b/monitoring/api/AlertSnippets/AlertSnippets.csproj index b79db2e6e3d..0185e367c3d 100644 --- a/monitoring/api/AlertSnippets/AlertSnippets.csproj +++ b/monitoring/api/AlertSnippets/AlertSnippets.csproj @@ -8,7 +8,7 @@ - + runtime; build; native; contentfiles; analyzers diff --git a/monitoring/api/AlertTest/AlertTest.csproj b/monitoring/api/AlertTest/AlertTest.csproj index c60535e8bff..e963d48fad5 100644 --- a/monitoring/api/AlertTest/AlertTest.csproj +++ b/monitoring/api/AlertTest/AlertTest.csproj @@ -8,7 +8,7 @@ - + runtime; build; native; contentfiles; analyzers all diff --git a/monitoring/api/MonitoringTest/MonitoringTest.csproj b/monitoring/api/MonitoringTest/MonitoringTest.csproj index 59843ad24b0..4e1f21b49ff 100644 --- a/monitoring/api/MonitoringTest/MonitoringTest.csproj +++ b/monitoring/api/MonitoringTest/MonitoringTest.csproj @@ -5,7 +5,7 @@ - + runtime; build; native; contentfiles; analyzers all diff --git a/monitoring/api/UptimeCheckTest/UptimeCheckTest.csproj b/monitoring/api/UptimeCheckTest/UptimeCheckTest.csproj index a39d7aca8fd..ca09d2109d0 100644 --- a/monitoring/api/UptimeCheckTest/UptimeCheckTest.csproj +++ b/monitoring/api/UptimeCheckTest/UptimeCheckTest.csproj @@ -7,7 +7,7 @@ - + runtime; build; native; contentfiles; analyzers all diff --git a/parametermanager/api/ParameterManager.RegionalSamples.Tests/ParameterManager.RegionalSamples.Tests.csproj b/parametermanager/api/ParameterManager.RegionalSamples.Tests/ParameterManager.RegionalSamples.Tests.csproj index dd0daee8b2c..9c68f6e5523 100644 --- a/parametermanager/api/ParameterManager.RegionalSamples.Tests/ParameterManager.RegionalSamples.Tests.csproj +++ b/parametermanager/api/ParameterManager.RegionalSamples.Tests/ParameterManager.RegionalSamples.Tests.csproj @@ -12,7 +12,7 @@ - + diff --git a/parametermanager/api/ParameterManager.Samples.Tests/ParameterManager.Samples.Tests.csproj b/parametermanager/api/ParameterManager.Samples.Tests/ParameterManager.Samples.Tests.csproj index a4587a1e042..1dfd2d2efe0 100644 --- a/parametermanager/api/ParameterManager.Samples.Tests/ParameterManager.Samples.Tests.csproj +++ b/parametermanager/api/ParameterManager.Samples.Tests/ParameterManager.Samples.Tests.csproj @@ -12,7 +12,7 @@ - + diff --git a/pubsub/api/Pubsub.Samples.Tests/Pubsub.Samples.Tests.csproj b/pubsub/api/Pubsub.Samples.Tests/Pubsub.Samples.Tests.csproj index f247d303e2b..11bb9bd0aad 100644 --- a/pubsub/api/Pubsub.Samples.Tests/Pubsub.Samples.Tests.csproj +++ b/pubsub/api/Pubsub.Samples.Tests/Pubsub.Samples.Tests.csproj @@ -9,7 +9,7 @@ - + diff --git a/retail/interactive-tutorial/RetailEvents.Samples.Tests/RetailEvents.Samples.Tests.csproj b/retail/interactive-tutorial/RetailEvents.Samples.Tests/RetailEvents.Samples.Tests.csproj index bad5b5b66f2..67577846205 100644 --- a/retail/interactive-tutorial/RetailEvents.Samples.Tests/RetailEvents.Samples.Tests.csproj +++ b/retail/interactive-tutorial/RetailEvents.Samples.Tests/RetailEvents.Samples.Tests.csproj @@ -7,7 +7,7 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/retail/interactive-tutorial/RetailProducts.Samples.Tests/RetailProducts.Samples.Tests.csproj b/retail/interactive-tutorial/RetailProducts.Samples.Tests/RetailProducts.Samples.Tests.csproj index e441a64dd67..95480efa9c4 100644 --- a/retail/interactive-tutorial/RetailProducts.Samples.Tests/RetailProducts.Samples.Tests.csproj +++ b/retail/interactive-tutorial/RetailProducts.Samples.Tests/RetailProducts.Samples.Tests.csproj @@ -8,7 +8,7 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/retail/interactive-tutorial/RetailSearch.Samples.Tests/RetailSearch.Samples.Tests.csproj b/retail/interactive-tutorial/RetailSearch.Samples.Tests/RetailSearch.Samples.Tests.csproj index 9de2373fa47..ada710edaee 100644 --- a/retail/interactive-tutorial/RetailSearch.Samples.Tests/RetailSearch.Samples.Tests.csproj +++ b/retail/interactive-tutorial/RetailSearch.Samples.Tests/RetailSearch.Samples.Tests.csproj @@ -7,7 +7,7 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/run/markdown-preview/Samples.Run.MarkdownPreview.Editor.Tests/Samples.Run.MarkdownPreview.Editor.Tests.csproj b/run/markdown-preview/Samples.Run.MarkdownPreview.Editor.Tests/Samples.Run.MarkdownPreview.Editor.Tests.csproj index 05dc19af3c2..7f1ad00a41a 100644 --- a/run/markdown-preview/Samples.Run.MarkdownPreview.Editor.Tests/Samples.Run.MarkdownPreview.Editor.Tests.csproj +++ b/run/markdown-preview/Samples.Run.MarkdownPreview.Editor.Tests/Samples.Run.MarkdownPreview.Editor.Tests.csproj @@ -13,7 +13,7 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/run/markdown-preview/Samples.Run.MarkdownPreview.Renderer.Tests/Samples.Run.MarkdownPreview.Renderer.Tests.csproj b/run/markdown-preview/Samples.Run.MarkdownPreview.Renderer.Tests/Samples.Run.MarkdownPreview.Renderer.Tests.csproj index 5b86cefc3d8..25216596879 100644 --- a/run/markdown-preview/Samples.Run.MarkdownPreview.Renderer.Tests/Samples.Run.MarkdownPreview.Renderer.Tests.csproj +++ b/run/markdown-preview/Samples.Run.MarkdownPreview.Renderer.Tests/Samples.Run.MarkdownPreview.Renderer.Tests.csproj @@ -12,7 +12,7 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/run/pubsub/Run.Samples.Pubsub.MinimalApi.Tests/Run.Samples.Pubsub.MinimalApi.Tests.csproj b/run/pubsub/Run.Samples.Pubsub.MinimalApi.Tests/Run.Samples.Pubsub.MinimalApi.Tests.csproj index 33f58980afd..b171d46cf79 100644 --- a/run/pubsub/Run.Samples.Pubsub.MinimalApi.Tests/Run.Samples.Pubsub.MinimalApi.Tests.csproj +++ b/run/pubsub/Run.Samples.Pubsub.MinimalApi.Tests/Run.Samples.Pubsub.MinimalApi.Tests.csproj @@ -10,7 +10,7 @@ - + runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/secretmanager/api/SecretManager.Samples.Tests/SecretManager.Samples.Tests.csproj b/secretmanager/api/SecretManager.Samples.Tests/SecretManager.Samples.Tests.csproj index 1330ff7dadc..6076917272b 100644 --- a/secretmanager/api/SecretManager.Samples.Tests/SecretManager.Samples.Tests.csproj +++ b/secretmanager/api/SecretManager.Samples.Tests/SecretManager.Samples.Tests.csproj @@ -6,7 +6,7 @@ - + diff --git a/securitycenter/api/SecurityCenter.Samples.Tests/SecurityCenter.Samples.Tests.csproj b/securitycenter/api/SecurityCenter.Samples.Tests/SecurityCenter.Samples.Tests.csproj index ed6518ea43c..c880ecb881e 100644 --- a/securitycenter/api/SecurityCenter.Samples.Tests/SecurityCenter.Samples.Tests.csproj +++ b/securitycenter/api/SecurityCenter.Samples.Tests/SecurityCenter.Samples.Tests.csproj @@ -4,7 +4,7 @@ false - + diff --git a/servicedirectory/api/ServiceDirectory.Samples.Tests/ServiceDirectory.Samples.Tests.csproj b/servicedirectory/api/ServiceDirectory.Samples.Tests/ServiceDirectory.Samples.Tests.csproj index 5cb701c45af..97f7a518e3e 100644 --- a/servicedirectory/api/ServiceDirectory.Samples.Tests/ServiceDirectory.Samples.Tests.csproj +++ b/servicedirectory/api/ServiceDirectory.Samples.Tests/ServiceDirectory.Samples.Tests.csproj @@ -6,7 +6,7 @@ - + diff --git a/spanner/api/Spanner.Samples.Tests/Spanner.Samples.Tests.csproj b/spanner/api/Spanner.Samples.Tests/Spanner.Samples.Tests.csproj index 491ba6806d2..a926ad06b4b 100644 --- a/spanner/api/Spanner.Samples.Tests/Spanner.Samples.Tests.csproj +++ b/spanner/api/Spanner.Samples.Tests/Spanner.Samples.Tests.csproj @@ -14,7 +14,7 @@ - + diff --git a/storage/api/Storage.Samples.Tests/Storage.Samples.Tests.csproj b/storage/api/Storage.Samples.Tests/Storage.Samples.Tests.csproj index b0df613426e..d668856b4f3 100644 --- a/storage/api/Storage.Samples.Tests/Storage.Samples.Tests.csproj +++ b/storage/api/Storage.Samples.Tests/Storage.Samples.Tests.csproj @@ -8,7 +8,7 @@ - + diff --git a/storageinsights/api/StorageInsights.Samples.Tests/StorageInsights.Samples.Tests.csproj b/storageinsights/api/StorageInsights.Samples.Tests/StorageInsights.Samples.Tests.csproj index b0fbc606b60..4eca63ac62e 100644 --- a/storageinsights/api/StorageInsights.Samples.Tests/StorageInsights.Samples.Tests.csproj +++ b/storageinsights/api/StorageInsights.Samples.Tests/StorageInsights.Samples.Tests.csproj @@ -14,7 +14,7 @@ - + diff --git a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageTransfer.Samples.Tests.csproj b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageTransfer.Samples.Tests.csproj index 88979dd6906..96018b432d3 100644 --- a/storagetransfer/api/StorageTransfer.Samples.Tests/StorageTransfer.Samples.Tests.csproj +++ b/storagetransfer/api/StorageTransfer.Samples.Tests/StorageTransfer.Samples.Tests.csproj @@ -13,7 +13,7 @@ - + diff --git a/workflows/api/Workflow.Samples.Test/Workflow.Samples.Test.csproj b/workflows/api/Workflow.Samples.Test/Workflow.Samples.Test.csproj index 3091a235bae..e185308a423 100644 --- a/workflows/api/Workflow.Samples.Test/Workflow.Samples.Test.csproj +++ b/workflows/api/Workflow.Samples.Test/Workflow.Samples.Test.csproj @@ -10,7 +10,7 @@ - + From fd2616dbe6dce70024fc58021466d8e9927ac4c5 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Wed, 21 May 2025 23:56:59 +0000 Subject: [PATCH 54/58] chore(deps): update dependency xunit.runner.visualstudio to v3 --- .../ParameterManager.RegionalSamples.Tests.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parametermanager/api/ParameterManager.RegionalSamples.Tests/ParameterManager.RegionalSamples.Tests.csproj b/parametermanager/api/ParameterManager.RegionalSamples.Tests/ParameterManager.RegionalSamples.Tests.csproj index 9c68f6e5523..570e5e0caf2 100644 --- a/parametermanager/api/ParameterManager.RegionalSamples.Tests/ParameterManager.RegionalSamples.Tests.csproj +++ b/parametermanager/api/ParameterManager.RegionalSamples.Tests/ParameterManager.RegionalSamples.Tests.csproj @@ -14,7 +14,7 @@ - + From abd1b9ab05822ce1d8efd6dbc6f03648517b6952 Mon Sep 17 00:00:00 2001 From: Harsh Nasit <131268456+harshnasitcrest@users.noreply.github.com> Date: Fri, 23 May 2025 23:08:13 +0530 Subject: [PATCH 55/58] feat(modelarmor): Added sample for Model Armor Quickstart (#3062) --- .github/blunderbuss.yml | 10 +- CODEOWNERS | 2 + .../ModelArmor.Samples.Tests.csproj | 16 +++ .../ModelArmorFixture.cs | 67 +++++++++ .../QuickStartTest.cs | 46 +++++++ .../ModelArmor.Samples.csproj | 9 ++ .../api/ModelArmor.Samples/QuickStart.cs | 128 ++++++++++++++++++ modelarmor/api/ModelArmor.sln | 48 +++++++ modelarmor/api/runTest.ps1 | 16 +++ 9 files changed, 341 insertions(+), 1 deletion(-) create mode 100644 modelarmor/api/ModelArmor.Samples.Tests/ModelArmor.Samples.Tests.csproj create mode 100644 modelarmor/api/ModelArmor.Samples.Tests/ModelArmorFixture.cs create mode 100644 modelarmor/api/ModelArmor.Samples.Tests/QuickStartTest.cs create mode 100644 modelarmor/api/ModelArmor.Samples/ModelArmor.Samples.csproj create mode 100644 modelarmor/api/ModelArmor.Samples/QuickStart.cs create mode 100644 modelarmor/api/ModelArmor.sln create mode 100755 modelarmor/api/runTest.ps1 diff --git a/.github/blunderbuss.yml b/.github/blunderbuss.yml index 08be36a46ee..84164bfc5a6 100644 --- a/.github/blunderbuss.yml +++ b/.github/blunderbuss.yml @@ -49,6 +49,10 @@ assign_issues_by: - 'api: parametermanager' to: - GoogleCloudPlatform/cloud-parameters-team +- labels: + - "api: modelarmor" + to: + - GoogleCloudPlatform/cloud-modelarmor-team assign_issues: - GoogleCloudPlatform/dotnet-samples-reviewers @@ -110,4 +114,8 @@ assign_prs_by: - labels: - 'api: parametermanager' to: - - GoogleCloudPlatform/cloud-parameters-team \ No newline at end of file + - GoogleCloudPlatform/cloud-parameters-team +- labels: + - "api: modelarmor" + to: + - GoogleCloudPlatform/cloud-modelarmor-team diff --git a/CODEOWNERS b/CODEOWNERS index d1ebe9e6c4c..7dc6468465e 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -36,6 +36,8 @@ applications/googlehome-meets-dotnetcontainers/ @meteatamel @GoogleCloudPlatfor run/ @meteatamel @GoogleCloudPlatform/dotnet-samples-reviewers aiplatform/ @meteatamel @GoogleCloudPlatform/dotnet-samples-reviewers +modelarmor/ @GoogleCloudPlatform/dotnet-samples-reviewers @GoogleCloudPlatform/cloud-modelarmor-team + retail/ @chenlei1216 @GoogleCloudPlatform/dotnet-samples-reviewers logging/ @GoogleCloudPlatform/dee-platform-ops @GoogleCloudPlatform/dotnet-samples-reviewers diff --git a/modelarmor/api/ModelArmor.Samples.Tests/ModelArmor.Samples.Tests.csproj b/modelarmor/api/ModelArmor.Samples.Tests/ModelArmor.Samples.Tests.csproj new file mode 100644 index 00000000000..017af3b9e30 --- /dev/null +++ b/modelarmor/api/ModelArmor.Samples.Tests/ModelArmor.Samples.Tests.csproj @@ -0,0 +1,16 @@ + + + net8.0 + false + + + + + + + + + + + + diff --git a/modelarmor/api/ModelArmor.Samples.Tests/ModelArmorFixture.cs b/modelarmor/api/ModelArmor.Samples.Tests/ModelArmorFixture.cs new file mode 100644 index 00000000000..96c36d35274 --- /dev/null +++ b/modelarmor/api/ModelArmor.Samples.Tests/ModelArmorFixture.cs @@ -0,0 +1,67 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using Google.Cloud.ModelArmor.V1; +using Xunit; + +[CollectionDefinition(nameof(ModelArmorFixture))] +public class ModelArmorFixture : IDisposable, ICollectionFixture +{ + public ModelArmorClient Client { get; } + public string ProjectId { get; } + public string LocationId { get; } + public TemplateName TemplateForQuickstartName { get; } + + public ModelArmorFixture() + { + // Get the Google Cloud Project ID. + ProjectId = Environment.GetEnvironmentVariable("GOOGLE_PROJECT_ID"); + if (string.IsNullOrEmpty(ProjectId)) + { + throw new Exception("Missing GOOGLE_PROJECT_ID environment variable"); + } + + // Get location ID from environment variable or use default. + LocationId = Environment.GetEnvironmentVariable("GOOGLE_CLOUD_LOCATION") ?? "us-central1"; + + // Create client. + Client = ModelArmorClient.Create(); + + // Create a template name for quickstart test. + string templateId = $"test-csharp-{Guid.NewGuid().ToString("N").Substring(0, 8)}"; + TemplateForQuickstartName = TemplateName.FromProjectLocationTemplate( + ProjectId, + LocationId, + templateId + ); + } + + public void Dispose() + { + // Clean up resources after tests. + try + { + Client.DeleteTemplate( + new DeleteTemplateRequest { Name = TemplateForQuickstartName.ToString() } + ); + } + catch (Exception) + { + // Ignore errors during cleanup. + } + } +} diff --git a/modelarmor/api/ModelArmor.Samples.Tests/QuickStartTest.cs b/modelarmor/api/ModelArmor.Samples.Tests/QuickStartTest.cs new file mode 100644 index 00000000000..b9a6bffb5e9 --- /dev/null +++ b/modelarmor/api/ModelArmor.Samples.Tests/QuickStartTest.cs @@ -0,0 +1,46 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Google.Cloud.ModelArmor.V1; +using Xunit; +using Xunit.Abstractions; + +[Collection(nameof(ModelArmorFixture))] +public class QuickstartTests +{ + private readonly ModelArmorFixture _fixture; + private readonly QuickstartSample _sample; + + public QuickstartTests(ModelArmorFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _sample = new QuickstartSample(); + } + + [Fact] + public void Runs() + { + // Get template name from fixture + TemplateName templateName = _fixture.TemplateForQuickstartName; + + // Run the quickstart sample + _sample.Quickstart( + projectId: templateName.ProjectId, + locationId: templateName.LocationId, + templateId: templateName.TemplateId + ); + } +} diff --git a/modelarmor/api/ModelArmor.Samples/ModelArmor.Samples.csproj b/modelarmor/api/ModelArmor.Samples/ModelArmor.Samples.csproj new file mode 100644 index 00000000000..2327504e74e --- /dev/null +++ b/modelarmor/api/ModelArmor.Samples/ModelArmor.Samples.csproj @@ -0,0 +1,9 @@ + + + net8.0 + false + + + + + diff --git a/modelarmor/api/ModelArmor.Samples/QuickStart.cs b/modelarmor/api/ModelArmor.Samples/QuickStart.cs new file mode 100644 index 00000000000..44699901f29 --- /dev/null +++ b/modelarmor/api/ModelArmor.Samples/QuickStart.cs @@ -0,0 +1,128 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +//[START modelarmor_quickstart] +using System; +using System.Text; +using Google.Api.Gax.ResourceNames; +using Google.Cloud.ModelArmor.V1; + +public class QuickstartSample +{ + public void Quickstart( + string projectId = "my-project", + string locationId = "us-central1", + string templateId = "my-template" + ) + { + ModelArmorClientBuilder clientBuilder = new ModelArmorClientBuilder + { + Endpoint = $"modelarmor.{locationId}.rep.googleapis.com", + }; + + // Create the client. + Google.Cloud.ModelArmor.V1.ModelArmorClient client = clientBuilder.Build(); + + LocationName parent = LocationName.FromProjectLocation(projectId, locationId); + + // Build the Model Armor template with preferred filters + RaiFilterSettings raiFilterSettings = new RaiFilterSettings(); + raiFilterSettings.RaiFilters.Add( + new RaiFilterSettings.Types.RaiFilter + { + FilterType = RaiFilterType.Dangerous, + ConfidenceLevel = DetectionConfidenceLevel.High, + } + ); + raiFilterSettings.RaiFilters.Add( + new RaiFilterSettings.Types.RaiFilter + { + FilterType = RaiFilterType.HateSpeech, + ConfidenceLevel = DetectionConfidenceLevel.MediumAndAbove, + } + ); + raiFilterSettings.RaiFilters.Add( + new RaiFilterSettings.Types.RaiFilter + { + FilterType = RaiFilterType.SexuallyExplicit, + ConfidenceLevel = DetectionConfidenceLevel.MediumAndAbove, + } + ); + raiFilterSettings.RaiFilters.Add( + new RaiFilterSettings.Types.RaiFilter + { + FilterType = RaiFilterType.Harassment, + ConfidenceLevel = DetectionConfidenceLevel.MediumAndAbove, + } + ); + + // Create the filter config with RAI settings + FilterConfig modelArmorFilter = new FilterConfig { RaiSettings = raiFilterSettings }; + + // Create the template + Template template = new Template { FilterConfig = modelArmorFilter }; + + // Create the request + CreateTemplateRequest request = new CreateTemplateRequest + { + ParentAsLocationName = parent, + TemplateId = templateId, + Template = template, + }; + + // Send the request + Template createdTemplate = client.CreateTemplate(request); + + Console.WriteLine($"Created template: {createdTemplate.Name}"); + + // Sanitize a user prompt using the created template + string userPrompt = "Unsafe user prompt"; + + TemplateName templateName = TemplateName.FromProjectLocationTemplate( + projectId, + locationId, + templateId + ); + + SanitizeUserPromptRequest userPromptSanitizeRequest = new SanitizeUserPromptRequest + { + TemplateName = templateName, + UserPromptData = new DataItem { Text = userPrompt }, + }; + + SanitizeUserPromptResponse userPromptSanitizeResponse = client.SanitizeUserPrompt( + userPromptSanitizeRequest + ); + + Console.WriteLine($"Result for User Prompt Sanitization: {userPromptSanitizeResponse}"); + + // Sanitize a model response using the created template + string modelResponse = "Unsanitized model output"; + + SanitizeModelResponseRequest modelSanitizeRequest = new SanitizeModelResponseRequest + { + TemplateName = templateName, + ModelResponseData = new DataItem { Text = modelResponse }, + }; + + SanitizeModelResponseResponse modelSanitizeResponse = client.SanitizeModelResponse( + modelSanitizeRequest + ); + + Console.WriteLine($"Result for Model Response Sanitization: {modelSanitizeResponse}"); + } +} +// [END modelarmor_quickstart] diff --git a/modelarmor/api/ModelArmor.sln b/modelarmor/api/ModelArmor.sln new file mode 100644 index 00000000000..b541616a768 --- /dev/null +++ b/modelarmor/api/ModelArmor.sln @@ -0,0 +1,48 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ModelArmor.Samples", "ModelArmor.Samples\ModelArmor.Samples.csproj", "{BA205D98-AC20-4BE1-92D0-4C2081378EBC}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ModelArmor.Samples.Tests", "ModelArmor.Samples.Tests\ModelArmor.Samples.Tests.csproj", "{EAD16385-9F71-47CA-A8E9-91B20C5A9BBE}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {BA205D98-AC20-4BE1-92D0-4C2081378EBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BA205D98-AC20-4BE1-92D0-4C2081378EBC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BA205D98-AC20-4BE1-92D0-4C2081378EBC}.Debug|x64.ActiveCfg = Debug|Any CPU + {BA205D98-AC20-4BE1-92D0-4C2081378EBC}.Debug|x64.Build.0 = Debug|Any CPU + {BA205D98-AC20-4BE1-92D0-4C2081378EBC}.Debug|x86.ActiveCfg = Debug|Any CPU + {BA205D98-AC20-4BE1-92D0-4C2081378EBC}.Debug|x86.Build.0 = Debug|Any CPU + {BA205D98-AC20-4BE1-92D0-4C2081378EBC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BA205D98-AC20-4BE1-92D0-4C2081378EBC}.Release|Any CPU.Build.0 = Release|Any CPU + {BA205D98-AC20-4BE1-92D0-4C2081378EBC}.Release|x64.ActiveCfg = Release|Any CPU + {BA205D98-AC20-4BE1-92D0-4C2081378EBC}.Release|x64.Build.0 = Release|Any CPU + {BA205D98-AC20-4BE1-92D0-4C2081378EBC}.Release|x86.ActiveCfg = Release|Any CPU + {BA205D98-AC20-4BE1-92D0-4C2081378EBC}.Release|x86.Build.0 = Release|Any CPU + {EAD16385-9F71-47CA-A8E9-91B20C5A9BBE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EAD16385-9F71-47CA-A8E9-91B20C5A9BBE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EAD16385-9F71-47CA-A8E9-91B20C5A9BBE}.Debug|x64.ActiveCfg = Debug|Any CPU + {EAD16385-9F71-47CA-A8E9-91B20C5A9BBE}.Debug|x64.Build.0 = Debug|Any CPU + {EAD16385-9F71-47CA-A8E9-91B20C5A9BBE}.Debug|x86.ActiveCfg = Debug|Any CPU + {EAD16385-9F71-47CA-A8E9-91B20C5A9BBE}.Debug|x86.Build.0 = Debug|Any CPU + {EAD16385-9F71-47CA-A8E9-91B20C5A9BBE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EAD16385-9F71-47CA-A8E9-91B20C5A9BBE}.Release|Any CPU.Build.0 = Release|Any CPU + {EAD16385-9F71-47CA-A8E9-91B20C5A9BBE}.Release|x64.ActiveCfg = Release|Any CPU + {EAD16385-9F71-47CA-A8E9-91B20C5A9BBE}.Release|x64.Build.0 = Release|Any CPU + {EAD16385-9F71-47CA-A8E9-91B20C5A9BBE}.Release|x86.ActiveCfg = Release|Any CPU + {EAD16385-9F71-47CA-A8E9-91B20C5A9BBE}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/modelarmor/api/runTest.ps1 b/modelarmor/api/runTest.ps1 new file mode 100755 index 00000000000..05d154e7749 --- /dev/null +++ b/modelarmor/api/runTest.ps1 @@ -0,0 +1,16 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dotnet restore --force +dotnet test --no-restore --test-adapter-path:. --logger:junit 2>&1 | %{ "$_" } From 4ce4adb344c2519447d88776d225e5601d614bb8 Mon Sep 17 00:00:00 2001 From: Durgesh Ninave Date: Fri, 23 May 2025 23:16:18 +0530 Subject: [PATCH 56/58] feat(parametermanager): Added samples for get, list and render global parameter (#2990) --- .../GetParameterTests.cs | 41 ++++++++++ .../GetParameterVersionTests.cs | 44 ++++++++++ .../ListParameterVersionsTests.cs | 43 ++++++++++ .../ListParametersTests.cs | 39 +++++++++ .../ParameterManagerFixture.cs | 81 +++++++++++++++++++ .../RenderParameterVersionTests.cs | 51 ++++++++++++ .../ParameterManager.Samples/GetParameter.cs | 49 +++++++++++ .../GetParameterVersion.cs | 57 +++++++++++++ .../ListParameterVersions.cs | 53 ++++++++++++ .../ListParameters.cs | 51 ++++++++++++ .../RenderParameterVersion.cs | 55 +++++++++++++ 11 files changed, 564 insertions(+) create mode 100644 parametermanager/api/ParameterManager.Samples.Tests/GetParameterTests.cs create mode 100644 parametermanager/api/ParameterManager.Samples.Tests/GetParameterVersionTests.cs create mode 100644 parametermanager/api/ParameterManager.Samples.Tests/ListParameterVersionsTests.cs create mode 100644 parametermanager/api/ParameterManager.Samples.Tests/ListParametersTests.cs create mode 100644 parametermanager/api/ParameterManager.Samples.Tests/RenderParameterVersionTests.cs create mode 100644 parametermanager/api/ParameterManager.Samples/GetParameter.cs create mode 100644 parametermanager/api/ParameterManager.Samples/GetParameterVersion.cs create mode 100644 parametermanager/api/ParameterManager.Samples/ListParameterVersions.cs create mode 100644 parametermanager/api/ParameterManager.Samples/ListParameters.cs create mode 100644 parametermanager/api/ParameterManager.Samples/RenderParameterVersion.cs diff --git a/parametermanager/api/ParameterManager.Samples.Tests/GetParameterTests.cs b/parametermanager/api/ParameterManager.Samples.Tests/GetParameterTests.cs new file mode 100644 index 00000000000..570e3974a72 --- /dev/null +++ b/parametermanager/api/ParameterManager.Samples.Tests/GetParameterTests.cs @@ -0,0 +1,41 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Google.Cloud.ParameterManager.V1; + +[Collection(nameof(ParameterManagerFixture))] +public class GetParameterTests +{ + private readonly ParameterManagerFixture _fixture; + private readonly GetParameterSample _sample; + + public GetParameterTests(ParameterManagerFixture fixture) + { + _fixture = fixture; + _sample = new GetParameterSample(); + } + + [Fact] + public void GetParameter() + { + string parameterId = _fixture.RandomId(); + Parameter parameter = _fixture.CreateParameter(parameterId, ParameterFormat.Json); + Parameter result = _sample.GetParameter(projectId: _fixture.ProjectId, parameterId: parameterId); + + Assert.NotNull(result); + Assert.Equal(parameterId, result.ParameterName.ParameterId); + } +} diff --git a/parametermanager/api/ParameterManager.Samples.Tests/GetParameterVersionTests.cs b/parametermanager/api/ParameterManager.Samples.Tests/GetParameterVersionTests.cs new file mode 100644 index 00000000000..ba7134049b1 --- /dev/null +++ b/parametermanager/api/ParameterManager.Samples.Tests/GetParameterVersionTests.cs @@ -0,0 +1,44 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Google.Cloud.ParameterManager.V1; + +[Collection(nameof(ParameterManagerFixture))] +public class GetParameterVersionTests +{ + private readonly ParameterManagerFixture _fixture; + private readonly GetParameterVersionSample _sample; + + public GetParameterVersionTests(ParameterManagerFixture fixture) + { + _fixture = fixture; + _sample = new GetParameterVersionSample(); + } + + [Fact] + public void GetParameterVersion() + { + string parameterId = _fixture.RandomId(); + string versionId = _fixture.RandomId(); + Parameter parameter = _fixture.CreateParameter(parameterId, ParameterFormat.Unformatted); + string payload = "test123"; + ParameterVersion parameterVersion = _fixture.CreateParameterVersion(parameterId, versionId, payload); + ParameterVersion result = _sample.GetParameterVersion(projectId: _fixture.ProjectId, parameterId: parameterId, versionId: versionId); + + Assert.NotNull(result); + Assert.Equal(versionId, result.ParameterVersionName.ParameterVersionId); + } +} diff --git a/parametermanager/api/ParameterManager.Samples.Tests/ListParameterVersionsTests.cs b/parametermanager/api/ParameterManager.Samples.Tests/ListParameterVersionsTests.cs new file mode 100644 index 00000000000..d1522604a99 --- /dev/null +++ b/parametermanager/api/ParameterManager.Samples.Tests/ListParameterVersionsTests.cs @@ -0,0 +1,43 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Google.Cloud.ParameterManager.V1; + +[Collection(nameof(ParameterManagerFixture))] +public class ListParameterVersionsTests +{ + private readonly ParameterManagerFixture _fixture; + private readonly ListParameterVersionsSample _sample; + + public ListParameterVersionsTests(ParameterManagerFixture fixture) + { + _fixture = fixture; + _sample = new ListParameterVersionsSample(); + } + + [Fact] + public void ListParameterVersions() + { + string parameterId = _fixture.RandomId(); + Parameter parameter = _fixture.CreateParameter(parameterId, ParameterFormat.Unformatted); + string payload = "test123"; + ParameterVersion parameterVersion = _fixture.CreateParameterVersion(parameterId, _fixture.RandomId(), payload); + ParameterVersion parameterVersion1 = _fixture.CreateParameterVersion(parameterId, _fixture.RandomId(), payload); + + IEnumerable result = _sample.ListParameterVersions(projectId: _fixture.ProjectId, parameterId: parameterId); + Assert.NotNull(result); + } +} diff --git a/parametermanager/api/ParameterManager.Samples.Tests/ListParametersTests.cs b/parametermanager/api/ParameterManager.Samples.Tests/ListParametersTests.cs new file mode 100644 index 00000000000..fb3b6633520 --- /dev/null +++ b/parametermanager/api/ParameterManager.Samples.Tests/ListParametersTests.cs @@ -0,0 +1,39 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Google.Cloud.ParameterManager.V1; + +[Collection(nameof(ParameterManagerFixture))] +public class ListParametersTests +{ + private readonly ParameterManagerFixture _fixture; + private readonly ListParametersSample _sample; + + public ListParametersTests(ParameterManagerFixture fixture) + { + _fixture = fixture; + _sample = new ListParametersSample(); + } + + [Fact] + public void ListParameters() + { + Parameter parameter = _fixture.CreateParameter(_fixture.RandomId(), ParameterFormat.Json); + Parameter parameter1 = _fixture.CreateParameter(_fixture.RandomId(), ParameterFormat.Json); + IEnumerable result = _sample.ListParameters(projectId: _fixture.ProjectId); + Assert.NotNull(result); + } +} diff --git a/parametermanager/api/ParameterManager.Samples.Tests/ParameterManagerFixture.cs b/parametermanager/api/ParameterManager.Samples.Tests/ParameterManagerFixture.cs index 8b01f16472b..12791423862 100644 --- a/parametermanager/api/ParameterManager.Samples.Tests/ParameterManagerFixture.cs +++ b/parametermanager/api/ParameterManager.Samples.Tests/ParameterManagerFixture.cs @@ -26,7 +26,9 @@ public class ParameterManagerFixture : IDisposable, ICollectionFixture ParametersToDelete { get; } = new List(); + internal List SecretsToDelete { get; } = new List(); internal List ParameterVersionsToDelete { get; } = new List(); public ParameterManagerFixture() @@ -38,6 +40,7 @@ public ParameterManagerFixture() } Client = ParameterManagerClient.Create(); + SecretClient = SecretManagerServiceClient.Create(); } public void Dispose() @@ -50,6 +53,10 @@ public void Dispose() { DeleteParameter(parameter); } + foreach (var secret in SecretsToDelete) + { + DeleteSecret(secret); + } } public string RandomId() @@ -71,6 +78,22 @@ public Parameter CreateParameter(string parameterId, ParameterFormat format) return Parameter; } + public ParameterVersion CreateParameterVersion(string parameterId, string versionId, string payload) + { + ParameterName parameterName = new ParameterName(ProjectId, LocationId, parameterId); + ParameterVersion parameterVersion = new ParameterVersion + { + Payload = new ParameterVersionPayload + { + Data = ByteString.CopyFrom(payload, Encoding.UTF8) + } + }; + + ParameterVersion ParameterVersion = Client.CreateParameterVersion(parameterName, parameterVersion, versionId); + ParameterVersionsToDelete.Add(ParameterVersion.ParameterVersionName); + return ParameterVersion; + } + private void DeleteParameter(ParameterName name) { try @@ -94,4 +117,62 @@ private void DeleteParameterVersion(ParameterVersionName name) // Ignore error - Parameter version was already deleted } } + + public Secret CreateSecret(string secretId) + { + ProjectName projectName = new ProjectName(ProjectId); + + Secret secret = new Secret + { + Replication = new Replication + { + Automatic = new Replication.Types.Automatic(), + }, + }; + Secret Secret = SecretClient.CreateSecret(projectName, secretId, secret); + SecretsToDelete.Add(Secret.SecretName); + return Secret; + } + + public Policy GrantIAMAccess(SecretName secretName, string member) + { + // Get current policy. + Policy policy = SecretClient.GetIamPolicy(new GetIamPolicyRequest + { + ResourceAsResourceName = secretName, + }); + + // Add the user to the list of bindings. + policy.AddRoleMember("roles/secretmanager.secretAccessor", member); + + // Save the updated policy. + policy = SecretClient.SetIamPolicy(new SetIamPolicyRequest + { + ResourceAsResourceName = secretName, + Policy = policy, + }); + return policy; + } + + public SecretVersion AddSecretVersion(Secret secret) + { + SecretPayload payload = new SecretPayload + { + Data = ByteString.CopyFrom("my super secret data", Encoding.UTF8), + }; + + return SecretClient.AddSecretVersion(secret.SecretName, payload); + } + + private void DeleteSecret(SecretName name) + { + try + { + SecretClient.DeleteSecret(name); + } + catch (Grpc.Core.RpcException e) when (e.StatusCode == Grpc.Core.StatusCode.NotFound) + { + // Ignore error - secret was already deleted + } + } } diff --git a/parametermanager/api/ParameterManager.Samples.Tests/RenderParameterVersionTests.cs b/parametermanager/api/ParameterManager.Samples.Tests/RenderParameterVersionTests.cs new file mode 100644 index 00000000000..553a452e06c --- /dev/null +++ b/parametermanager/api/ParameterManager.Samples.Tests/RenderParameterVersionTests.cs @@ -0,0 +1,51 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Google.Cloud.Iam.V1; +using Google.Cloud.ParameterManager.V1; +using Google.Cloud.SecretManager.V1; + +[Collection(nameof(ParameterManagerFixture))] +public class RenderParameterVersionTests +{ + private readonly ParameterManagerFixture _fixture; + private readonly RenderParameterVersionSample _sample; + + public RenderParameterVersionTests(ParameterManagerFixture fixture) + { + _fixture = fixture; + _sample = new RenderParameterVersionSample(); + } + + [Fact] + public void RenderParameterVersion() + { + string parameterId = _fixture.RandomId(); + string versionId = _fixture.RandomId(); + Parameter parameter = _fixture.CreateParameter(parameterId, ParameterFormat.Unformatted); + + Secret secret = _fixture.CreateSecret(_fixture.RandomId()); + _fixture.AddSecretVersion(secret); + string payload = $"{{\"username\": \"test-user\", \"password\": \"__REF__(//secretmanager.googleapis.com/{secret.SecretName}/versions/latest)\"}}"; + _fixture.GrantIAMAccess(secret.SecretName, parameter.PolicyMember.IamPolicyUidPrincipal); + + ParameterVersion parameterVersion = _fixture.CreateParameterVersion(parameterId, versionId, payload); + + string result = _sample.RenderParameterVersion(projectId: _fixture.ProjectId, parameterId: parameterId, versionId); + + Assert.NotNull(result); + } +} diff --git a/parametermanager/api/ParameterManager.Samples/GetParameter.cs b/parametermanager/api/ParameterManager.Samples/GetParameter.cs new file mode 100644 index 00000000000..1f3390d3d39 --- /dev/null +++ b/parametermanager/api/ParameterManager.Samples/GetParameter.cs @@ -0,0 +1,49 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// [START parametermanager_get_param] + +using Google.Cloud.ParameterManager.V1; + +public class GetParameterSample +{ + /// + /// This function retrieves a parameter using the Parameter Manager SDK for GCP. + /// + /// The ID of the project where the parameter is located. + /// The ID of the parameter to be retrieved. + /// The retrieved Parameter object. + public Parameter GetParameter( + string projectId, + string parameterId) + { + // Create the client. + ParameterManagerClient client = ParameterManagerClient.Create(); + + // Build the resource name for the parameter. + ParameterName parameterName = new ParameterName(projectId, "global", parameterId); + + // Call the API to get the parameter. + Parameter parameter = client.GetParameter(parameterName); + + // Print the retrieved parameter name. + Console.WriteLine($"Found the parameter {parameter.Name} with format {parameter.Format}"); + + // Return the retrieved parameter. + return parameter; + } +} +// [END parametermanager_get_param] diff --git a/parametermanager/api/ParameterManager.Samples/GetParameterVersion.cs b/parametermanager/api/ParameterManager.Samples/GetParameterVersion.cs new file mode 100644 index 00000000000..4c227ebb492 --- /dev/null +++ b/parametermanager/api/ParameterManager.Samples/GetParameterVersion.cs @@ -0,0 +1,57 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// [START parametermanager_get_param_version] + +using Google.Cloud.ParameterManager.V1; +using System.Text; + +public class GetParameterVersionSample +{ + /// + /// This function retrieves a parameter version using the Parameter Manager SDK for GCP. + /// + /// The ID of the project where the parameter is located. + /// The ID of the parameter for which the version is to be retrieved. + /// The ID of the version to be retrieved. + /// The retrieved ParameterVersion object. + public ParameterVersion GetParameterVersion( + string projectId, + string parameterId, + string versionId) + { + // Create the client. + ParameterManagerClient client = ParameterManagerClient.Create(); + + // Build the resource name for the parameter version. + ParameterVersionName parameterVersionName = new ParameterVersionName(projectId, "global", parameterId, versionId); + + // Call the API to get the parameter version. + ParameterVersion parameterVersion = client.GetParameterVersion(parameterVersionName); + + // Print the retrieved parameter version name. + Console.WriteLine($"Found regional parameter version {parameterVersion.Name} with state {(parameterVersion.Disabled ? "disabled" : "enabled")}"); + + if (!parameterVersion.Disabled) + { + Console.WriteLine($"Payload: {Encoding.UTF8.GetString(parameterVersion.Payload.Data.ToByteArray())}"); + } + + // Return the retrieved parameter version. + return parameterVersion; + } +} +// [END parametermanager_get_param_version] diff --git a/parametermanager/api/ParameterManager.Samples/ListParameterVersions.cs b/parametermanager/api/ParameterManager.Samples/ListParameterVersions.cs new file mode 100644 index 00000000000..a87009a2fec --- /dev/null +++ b/parametermanager/api/ParameterManager.Samples/ListParameterVersions.cs @@ -0,0 +1,53 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// [START parametermanager_list_param_versions] + +using Google.Api.Gax; +using Google.Cloud.ParameterManager.V1; + +public class ListParameterVersionsSample +{ + /// + /// This function lists parameter version using the Parameter Manager SDK for GCP. + /// + /// The ID of the project where the parameter is located. + /// The ID of the parameter for which the version is to be listed. + /// A list of ParameterVersion objects. + public IEnumerable ListParameterVersions( + string projectId, + string parameterId) + { + // Create the client. + ParameterManagerClient client = ParameterManagerClient.Create(); + + // Build the parent resource name for the parameter. + ParameterName parent = new ParameterName(projectId, "global", parameterId); + + // Call the API to list the parameter versions. + PagedEnumerable response = client.ListParameterVersions(parent); + + // Print each parameter version name. + foreach (ParameterVersion parameterVersion in response) + { + Console.WriteLine($"Found parameter version: {parameterVersion.Name}"); + } + + // Return the list of parameter versions. + return response; + } +} +// [END parametermanager_list_param_versions] diff --git a/parametermanager/api/ParameterManager.Samples/ListParameters.cs b/parametermanager/api/ParameterManager.Samples/ListParameters.cs new file mode 100644 index 00000000000..3dc4925705a --- /dev/null +++ b/parametermanager/api/ParameterManager.Samples/ListParameters.cs @@ -0,0 +1,51 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// [START parametermanager_list_params] + +using Google.Api.Gax; +using Google.Api.Gax.ResourceNames; +using Google.Cloud.ParameterManager.V1; + +public class ListParametersSample +{ + /// + /// This function lists parameter using the Parameter Manager SDK for GCP. + /// + /// The ID of the project where the parameter is located. + /// A list of Parameter objects. + public IEnumerable ListParameters(string projectId) + { + // Create the client. + ParameterManagerClient client = ParameterManagerClient.Create(); + + // Build the parent resource name. + LocationName parent = new LocationName(projectId, "global"); + + // Call the API to list the parameters. + PagedEnumerable response = client.ListParameters(parent); + + // Print each parameter name. + foreach (Parameter parameter in response) + { + Console.WriteLine($"Found parameter {parameter.Name} with format {parameter.Format}"); + } + + // Return the list of parameters. + return response; + } +} +// [END parametermanager_list_params] diff --git a/parametermanager/api/ParameterManager.Samples/RenderParameterVersion.cs b/parametermanager/api/ParameterManager.Samples/RenderParameterVersion.cs new file mode 100644 index 00000000000..cc632aad283 --- /dev/null +++ b/parametermanager/api/ParameterManager.Samples/RenderParameterVersion.cs @@ -0,0 +1,55 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// [START parametermanager_render_param_version] + +using Google.Cloud.ParameterManager.V1; +using System.Text; + +public class RenderParameterVersionSample +{ + /// + /// This function renders a parameter version using the Parameter Manager SDK for GCP. + /// + /// The ID of the project where the parameter is located. + /// The ID of the parameter for which the version is to be rendered. + /// The ID of the version to be rendered. + /// The rendered parameter version data as a string. + public string RenderParameterVersion( + string projectId, + string parameterId, + string versionId) + { + // Create the client. + ParameterManagerClient client = ParameterManagerClient.Create(); + + // Build the resource name for the parameter version. + ParameterVersionName parameterVersionName = new ParameterVersionName(projectId, "global", parameterId, versionId); + + // Call the API to render the parameter version. + RenderParameterVersionResponse renderParameterVersionResponse = client.RenderParameterVersion(parameterVersionName); + + // Decode the base64 encoded data. + string decodedData = Encoding.UTF8.GetString(renderParameterVersionResponse.RenderedPayload.ToByteArray()); + + // Print the rendered parameter version data. + Console.WriteLine($"Rendered parameter version payload: {decodedData}"); + + // Return the rendered parameter version data. + return decodedData; + } +} +// [END parametermanager_render_param_version] From c21bc0be251176c0c6c86bd5b7fd65809dc0ee18 Mon Sep 17 00:00:00 2001 From: Durgesh Ninave Date: Fri, 23 May 2025 23:27:28 +0530 Subject: [PATCH 57/58] feat(parametermanager): Added samples for global and regional parameter manager quickstart (#2986) --- .../RegionalQuickstartTests.cs | 50 +++++++++++ .../RegionalQuickstart.cs | 87 +++++++++++++++++++ .../QuickstartTests.cs | 48 ++++++++++ .../CreateParameterVersion.cs | 2 +- .../CreateParameterVersionWithSecret.cs | 2 +- .../CreateStructuredParameterVersion.cs | 2 +- .../ParameterManager.Samples/Quickstart.cs | 79 +++++++++++++++++ parametermanager/api/ParameterManager.sln | 52 +++++------ parametermanager/api/README.md | 32 +++++++ 9 files changed, 325 insertions(+), 29 deletions(-) create mode 100644 parametermanager/api/ParameterManager.RegionalSamples.Tests/RegionalQuickstartTests.cs create mode 100644 parametermanager/api/ParameterManager.RegionalSamples/RegionalQuickstart.cs create mode 100644 parametermanager/api/ParameterManager.Samples.Tests/QuickstartTests.cs create mode 100644 parametermanager/api/ParameterManager.Samples/Quickstart.cs create mode 100644 parametermanager/api/README.md diff --git a/parametermanager/api/ParameterManager.RegionalSamples.Tests/RegionalQuickstartTests.cs b/parametermanager/api/ParameterManager.RegionalSamples.Tests/RegionalQuickstartTests.cs new file mode 100644 index 00000000000..8f10944d75f --- /dev/null +++ b/parametermanager/api/ParameterManager.RegionalSamples.Tests/RegionalQuickstartTests.cs @@ -0,0 +1,50 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Google.Cloud.ParameterManager.V1; + +[Collection(nameof(ParameterManagerRegionalFixture))] +public class RegionalQuickstartTests +{ + private readonly ParameterManagerRegionalFixture _fixture; + private readonly RegionalQuickstartSample _sample; + + public RegionalQuickstartTests(ParameterManagerRegionalFixture fixture) + { + _fixture = fixture; + _sample = new RegionalQuickstartSample(); + } + + [Fact] + public void RegionalQuickstart() + { + string parameterId = _fixture.RandomId(); + ParameterName parameterName = new ParameterName(_fixture.ProjectId, ParameterManagerRegionalFixture.LocationId, parameterId); + ParameterVersionName parameterVersionName = new ParameterVersionName(_fixture.ProjectId, ParameterManagerRegionalFixture.LocationId, parameterId, _fixture.RandomId()); + + _sample.RegionalQuickstart( + projectId: parameterVersionName.ProjectId, parameterId: parameterVersionName.ParameterId, locationId: ParameterManagerRegionalFixture.LocationId, versionId: parameterVersionName.ParameterVersionId); + _fixture.ParametersToDelete.Add(parameterName); + _fixture.ParameterVersionsToDelete.Add(parameterVersionName); + + // Create the client with the regional endpoint + ParameterManagerClient client = _fixture.Client; + ParameterVersion result = client.GetParameterVersion(parameterVersionName); + + Assert.NotNull(result); + Assert.Equal(parameterVersionName.ParameterVersionId, result.ParameterVersionName.ParameterVersionId); + } +} diff --git a/parametermanager/api/ParameterManager.RegionalSamples/RegionalQuickstart.cs b/parametermanager/api/ParameterManager.RegionalSamples/RegionalQuickstart.cs new file mode 100644 index 00000000000..1aa1315891e --- /dev/null +++ b/parametermanager/api/ParameterManager.RegionalSamples/RegionalQuickstart.cs @@ -0,0 +1,87 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// [START parametermanager_regional_quickstart] + +using Google.Api.Gax.ResourceNames; +using Google.Cloud.ParameterManager.V1; +using Google.Protobuf; +using System.Text; + +public class RegionalQuickstartSample +{ + /// + /// This function demonstrates how to use the Parameter Manager API with the Parameter Manager SDK for Google Cloud. + /// It covers structured parameter creation and creation of parameter version with JSON format payload. + /// Finally, it fetches the decoded payload and prints them. + /// + /// The ID of the project where the parameter is located. + /// The ID of the region where the parameter is located. + /// The ID of the parameter for which the version is to be created. + /// The ID of the version to be created. + public void RegionalQuickstart( + string projectId, + string locationId, + string parameterId, + string versionId) + { + // Define the regional endpoint + string regionalEndpoint = $"parametermanager.{locationId}.rep.googleapis.com"; + + // Create the client with the regional endpoint + ParameterManagerClient client = new ParameterManagerClientBuilder + { + Endpoint = regionalEndpoint + }.Build(); + + // Build the parent resource name for the regional locationId + LocationName parent = new LocationName(projectId, locationId); + + // Create a parameter + Parameter parameter = new Parameter + { + Format = ParameterFormat.Json + }; + + // Call the API to create the parameter. + Parameter createdParameter = client.CreateParameter(parent, parameter, parameterId); + Console.WriteLine($"Created regional parameter {createdParameter.Name} with format {createdParameter.Format}"); + + // Build the parent resource name using ParameterName. + ParameterName parameterName = new ParameterName(projectId, locationId, parameterId); + + // Create a parameter version with a JSON payload + string payload = "{\"username\": \"test-user\", \"host\": \"localhost\"}"; + ByteString data = ByteString.CopyFrom(payload, Encoding.UTF8); + ParameterVersion parameterVersion = new ParameterVersion + { + Payload = new ParameterVersionPayload + { + Data = data + } + }; + + // Call the API to create the parameter version. + ParameterVersion createdParameterVersion = client.CreateParameterVersion(parameterName, parameterVersion, versionId); + Console.WriteLine($"Created regional parameter version: {createdParameterVersion.Name}"); + + // Fetch the parameter version data. + ParameterVersion getParameterVersion = client.GetParameterVersion(createdParameterVersion.Name); + string decodedData = Encoding.UTF8.GetString(getParameterVersion.Payload.Data.ToByteArray()); + Console.WriteLine($"Payload: {decodedData}"); + } +} +// [END parametermanager_regional_quickstart] diff --git a/parametermanager/api/ParameterManager.Samples.Tests/QuickstartTests.cs b/parametermanager/api/ParameterManager.Samples.Tests/QuickstartTests.cs new file mode 100644 index 00000000000..4f2b6238f87 --- /dev/null +++ b/parametermanager/api/ParameterManager.Samples.Tests/QuickstartTests.cs @@ -0,0 +1,48 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using Google.Cloud.ParameterManager.V1; + +[Collection(nameof(ParameterManagerFixture))] +public class QuickstartTests +{ + private readonly ParameterManagerFixture _fixture; + private readonly QuickstartSample _sample; + + public QuickstartTests(ParameterManagerFixture fixture) + { + _fixture = fixture; + _sample = new QuickstartSample(); + } + + [Fact] + public void Quickstart() + { + string parameterId = _fixture.RandomId(); + ParameterName parameterName = new ParameterName(_fixture.ProjectId, ParameterManagerFixture.LocationId, parameterId); + ParameterVersionName parameterVersionName = new ParameterVersionName(_fixture.ProjectId, ParameterManagerFixture.LocationId, parameterId, _fixture.RandomId()); + _sample.Quickstart( + projectId: parameterVersionName.ProjectId, parameterId: parameterVersionName.ParameterId, versionId: parameterVersionName.ParameterVersionId); + _fixture.ParametersToDelete.Add(parameterName); + _fixture.ParameterVersionsToDelete.Add(parameterVersionName); + + ParameterManagerClient client = _fixture.Client; + ParameterVersion result = client.GetParameterVersion(parameterVersionName); + + Assert.NotNull(result); + Assert.Equal(parameterVersionName.ParameterVersionId, result.ParameterVersionName.ParameterVersionId); + } +} diff --git a/parametermanager/api/ParameterManager.Samples/CreateParameterVersion.cs b/parametermanager/api/ParameterManager.Samples/CreateParameterVersion.cs index 1d56fd60b79..2a6d7a15575 100644 --- a/parametermanager/api/ParameterManager.Samples/CreateParameterVersion.cs +++ b/parametermanager/api/ParameterManager.Samples/CreateParameterVersion.cs @@ -54,7 +54,7 @@ public ParameterVersion CreateParameterVersion( }; // Call the API to create the parameter version. - ParameterVersion createdParameterVersion = client.CreateParameterVersion(parent.ToString(), parameterVersion, versionId); + ParameterVersion createdParameterVersion = client.CreateParameterVersion(parent, parameterVersion, versionId); // Print the created parameter version name. Console.WriteLine($"Created parameter version: {createdParameterVersion.Name}"); diff --git a/parametermanager/api/ParameterManager.Samples/CreateParameterVersionWithSecret.cs b/parametermanager/api/ParameterManager.Samples/CreateParameterVersionWithSecret.cs index 729a71d1969..6c7557dbf15 100644 --- a/parametermanager/api/ParameterManager.Samples/CreateParameterVersionWithSecret.cs +++ b/parametermanager/api/ParameterManager.Samples/CreateParameterVersionWithSecret.cs @@ -56,7 +56,7 @@ public ParameterVersion CreateParameterVersionWithSecret( }; // Call the API to create the parameter version. - ParameterVersion createdParameterVersion = client.CreateParameterVersion(parent.ToString(), parameterVersion, versionId); + ParameterVersion createdParameterVersion = client.CreateParameterVersion(parent, parameterVersion, versionId); // Print the created parameter version name. Console.WriteLine($"Created parameter version: {createdParameterVersion.Name}"); diff --git a/parametermanager/api/ParameterManager.Samples/CreateStructuredParameterVersion.cs b/parametermanager/api/ParameterManager.Samples/CreateStructuredParameterVersion.cs index 76f914ca837..b5fda0b2e34 100644 --- a/parametermanager/api/ParameterManager.Samples/CreateStructuredParameterVersion.cs +++ b/parametermanager/api/ParameterManager.Samples/CreateStructuredParameterVersion.cs @@ -54,7 +54,7 @@ public ParameterVersion CreateStructuredParameterVersion( }; // Call the API to create the parameter version. - ParameterVersion createdParameterVersion = client.CreateParameterVersion(parent.ToString(), parameterVersion, versionId); + ParameterVersion createdParameterVersion = client.CreateParameterVersion(parent, parameterVersion, versionId); // Print the created parameter version name. Console.WriteLine($"Created parameter version: {createdParameterVersion.Name}"); diff --git a/parametermanager/api/ParameterManager.Samples/Quickstart.cs b/parametermanager/api/ParameterManager.Samples/Quickstart.cs new file mode 100644 index 00000000000..d99a5e52cee --- /dev/null +++ b/parametermanager/api/ParameterManager.Samples/Quickstart.cs @@ -0,0 +1,79 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// [START parametermanager_quickstart] + +using Google.Api.Gax.ResourceNames; +using Google.Cloud.ParameterManager.V1; +using Google.Protobuf; +using System.Text; + +public class QuickstartSample +{ + /// + /// This function demonstrates how to use the Parameter Manager API with the Parameter Manager SDK for Google Cloud. + /// It covers structured parameter creation and creation of parameter version with JSON format payload. + /// Finally, it fetches the decoded payload and prints them. + /// + /// The ID of the project where the parameter is located. + /// The ID of the parameter for which the version is to be created. + /// The ID of the version to be created. + public void Quickstart( + string projectId, + string parameterId, + string versionId) + { + // Create the client. + ParameterManagerClient client = ParameterManagerClient.Create(); + + // Build the parent resource name. + LocationName parent = new LocationName(projectId, "global"); + + // Create a structured parameter. + Parameter parameter = new Parameter + { + Format = ParameterFormat.Json + }; + + // Call the API to create the parameter. + Parameter createdParameter = client.CreateParameter(parent, parameter, parameterId); + Console.WriteLine($"Created parameter {createdParameter.Name} with format {createdParameter.Format}"); + + // Build the parent resource name using ParameterName. + ParameterName parameterName = new ParameterName(projectId, "global", parameterId); + + // Define the JSON payload + string payload = "{\"username\": \"test-user\", \"host\": \"localhost\"}"; + ByteString data = ByteString.CopyFrom(payload, Encoding.UTF8); + ParameterVersion parameterVersion = new ParameterVersion + { + Payload = new ParameterVersionPayload + { + Data = data + } + }; + + // Call the API to create the parameter version. + ParameterVersion createdParameterVersion = client.CreateParameterVersion(parameterName, parameterVersion, versionId); + Console.WriteLine($"Created parameter version: {createdParameterVersion.Name}"); + + // Fetch the parameter version data. + ParameterVersion getParameterVersion = client.GetParameterVersion(createdParameterVersion.Name); + string decodedData = Encoding.UTF8.GetString(getParameterVersion.Payload.Data.ToByteArray()); + Console.WriteLine($"Payload: {decodedData}"); + } +} +// [END parametermanager_quickstart] diff --git a/parametermanager/api/ParameterManager.sln b/parametermanager/api/ParameterManager.sln index b0a12d2134f..6a90829d1c2 100644 --- a/parametermanager/api/ParameterManager.sln +++ b/parametermanager/api/ParameterManager.sln @@ -7,9 +7,9 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ParameterManager.Samples", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ParameterManager.Samples.Tests", "ParameterManager.Samples.Tests\ParameterManager.Samples.Tests.csproj", "{2ED39537-73EA-4186-A92D-82F59005FE14}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ParameterManager.RegionalSamples.Tests", "ParameterManager.RegionalSamples.Tests\ParameterManager.RegionalSamples.Tests.csproj", "{2688B889-A2A8-4A31-B903-65A4DAC566D5}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ParameterManager.RegionalSamples", "ParameterManager.RegionalSamples\ParameterManager.RegionalSamples.csproj", "{5EF4297D-DE95-4CF1-8271-B3232CA00705}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ParameterManager.RegionalSamples", "ParameterManager.RegionalSamples\ParameterManager.RegionalSamples.csproj", "{9230440A-5059-4C58-B853-F443A92516C1}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ParameterManager.RegionalSamples.Tests", "ParameterManager.RegionalSamples.Tests\ParameterManager.RegionalSamples.Tests.csproj", "{121A82D9-C771-4A9F-8FE7-DFED986B909B}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -45,30 +45,30 @@ Global {2ED39537-73EA-4186-A92D-82F59005FE14}.Release|x64.Build.0 = Release|Any CPU {2ED39537-73EA-4186-A92D-82F59005FE14}.Release|x86.ActiveCfg = Release|Any CPU {2ED39537-73EA-4186-A92D-82F59005FE14}.Release|x86.Build.0 = Release|Any CPU - {2688B889-A2A8-4A31-B903-65A4DAC566D5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {2688B889-A2A8-4A31-B903-65A4DAC566D5}.Debug|Any CPU.Build.0 = Debug|Any CPU - {2688B889-A2A8-4A31-B903-65A4DAC566D5}.Debug|x64.ActiveCfg = Debug|Any CPU - {2688B889-A2A8-4A31-B903-65A4DAC566D5}.Debug|x64.Build.0 = Debug|Any CPU - {2688B889-A2A8-4A31-B903-65A4DAC566D5}.Debug|x86.ActiveCfg = Debug|Any CPU - {2688B889-A2A8-4A31-B903-65A4DAC566D5}.Debug|x86.Build.0 = Debug|Any CPU - {2688B889-A2A8-4A31-B903-65A4DAC566D5}.Release|Any CPU.ActiveCfg = Release|Any CPU - {2688B889-A2A8-4A31-B903-65A4DAC566D5}.Release|Any CPU.Build.0 = Release|Any CPU - {2688B889-A2A8-4A31-B903-65A4DAC566D5}.Release|x64.ActiveCfg = Release|Any CPU - {2688B889-A2A8-4A31-B903-65A4DAC566D5}.Release|x64.Build.0 = Release|Any CPU - {2688B889-A2A8-4A31-B903-65A4DAC566D5}.Release|x86.ActiveCfg = Release|Any CPU - {2688B889-A2A8-4A31-B903-65A4DAC566D5}.Release|x86.Build.0 = Release|Any CPU - {9230440A-5059-4C58-B853-F443A92516C1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {9230440A-5059-4C58-B853-F443A92516C1}.Debug|Any CPU.Build.0 = Debug|Any CPU - {9230440A-5059-4C58-B853-F443A92516C1}.Debug|x64.ActiveCfg = Debug|Any CPU - {9230440A-5059-4C58-B853-F443A92516C1}.Debug|x64.Build.0 = Debug|Any CPU - {9230440A-5059-4C58-B853-F443A92516C1}.Debug|x86.ActiveCfg = Debug|Any CPU - {9230440A-5059-4C58-B853-F443A92516C1}.Debug|x86.Build.0 = Debug|Any CPU - {9230440A-5059-4C58-B853-F443A92516C1}.Release|Any CPU.ActiveCfg = Release|Any CPU - {9230440A-5059-4C58-B853-F443A92516C1}.Release|Any CPU.Build.0 = Release|Any CPU - {9230440A-5059-4C58-B853-F443A92516C1}.Release|x64.ActiveCfg = Release|Any CPU - {9230440A-5059-4C58-B853-F443A92516C1}.Release|x64.Build.0 = Release|Any CPU - {9230440A-5059-4C58-B853-F443A92516C1}.Release|x86.ActiveCfg = Release|Any CPU - {9230440A-5059-4C58-B853-F443A92516C1}.Release|x86.Build.0 = Release|Any CPU + {5EF4297D-DE95-4CF1-8271-B3232CA00705}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5EF4297D-DE95-4CF1-8271-B3232CA00705}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5EF4297D-DE95-4CF1-8271-B3232CA00705}.Debug|x64.ActiveCfg = Debug|Any CPU + {5EF4297D-DE95-4CF1-8271-B3232CA00705}.Debug|x64.Build.0 = Debug|Any CPU + {5EF4297D-DE95-4CF1-8271-B3232CA00705}.Debug|x86.ActiveCfg = Debug|Any CPU + {5EF4297D-DE95-4CF1-8271-B3232CA00705}.Debug|x86.Build.0 = Debug|Any CPU + {5EF4297D-DE95-4CF1-8271-B3232CA00705}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5EF4297D-DE95-4CF1-8271-B3232CA00705}.Release|Any CPU.Build.0 = Release|Any CPU + {5EF4297D-DE95-4CF1-8271-B3232CA00705}.Release|x64.ActiveCfg = Release|Any CPU + {5EF4297D-DE95-4CF1-8271-B3232CA00705}.Release|x64.Build.0 = Release|Any CPU + {5EF4297D-DE95-4CF1-8271-B3232CA00705}.Release|x86.ActiveCfg = Release|Any CPU + {5EF4297D-DE95-4CF1-8271-B3232CA00705}.Release|x86.Build.0 = Release|Any CPU + {121A82D9-C771-4A9F-8FE7-DFED986B909B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {121A82D9-C771-4A9F-8FE7-DFED986B909B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {121A82D9-C771-4A9F-8FE7-DFED986B909B}.Debug|x64.ActiveCfg = Debug|Any CPU + {121A82D9-C771-4A9F-8FE7-DFED986B909B}.Debug|x64.Build.0 = Debug|Any CPU + {121A82D9-C771-4A9F-8FE7-DFED986B909B}.Debug|x86.ActiveCfg = Debug|Any CPU + {121A82D9-C771-4A9F-8FE7-DFED986B909B}.Debug|x86.Build.0 = Debug|Any CPU + {121A82D9-C771-4A9F-8FE7-DFED986B909B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {121A82D9-C771-4A9F-8FE7-DFED986B909B}.Release|Any CPU.Build.0 = Release|Any CPU + {121A82D9-C771-4A9F-8FE7-DFED986B909B}.Release|x64.ActiveCfg = Release|Any CPU + {121A82D9-C771-4A9F-8FE7-DFED986B909B}.Release|x64.Build.0 = Release|Any CPU + {121A82D9-C771-4A9F-8FE7-DFED986B909B}.Release|x86.ActiveCfg = Release|Any CPU + {121A82D9-C771-4A9F-8FE7-DFED986B909B}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/parametermanager/api/README.md b/parametermanager/api/README.md new file mode 100644 index 00000000000..ccf54a1be45 --- /dev/null +++ b/parametermanager/api/README.md @@ -0,0 +1,32 @@ +# .NET Parameter Manager Regional Samples + +These samples demonstrate how to interact with the [Parameter Manager][parametermanager] using C# and +the .NET client libraries to call the Parameter Manager API. + +The samples require [.NET Core 8.0][net-core] or later. + +## Setup + +1. Set up a [.NET development environment](https://cloud.google.com/dotnet/docs/setup). + +1. [Enable the Parameter Manager API][enable-api] in your Google Cloud project. + + +## Contributing changes + +* See [CONTRIBUTING.md](../../CONTRIBUTING.md) + + +## Licensing + +* See [LICENSE](../../LICENSE) + + +## Testing + +* See [TESTING.md](../../TESTING.md) + + +[parametermanager]: https://cloud.google.com/secret-manager/parameter-manager/docs/overview +[enable-api]: https://console.cloud.google.com/apis/enableflow?apiid=parametermanager.googleapis.com +[net-core]: https://www.microsoft.com/net/core From ba7276a3cce741e1926154e9d566c6f116a2a44f Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Sat, 24 May 2025 01:08:27 +0000 Subject: [PATCH 58/58] chore(deps): update dependency microsoft.net.test.sdk to 17.14.0 --- .../ModelArmor.Samples.Tests/ModelArmor.Samples.Tests.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modelarmor/api/ModelArmor.Samples.Tests/ModelArmor.Samples.Tests.csproj b/modelarmor/api/ModelArmor.Samples.Tests/ModelArmor.Samples.Tests.csproj index 017af3b9e30..dab25c9205d 100644 --- a/modelarmor/api/ModelArmor.Samples.Tests/ModelArmor.Samples.Tests.csproj +++ b/modelarmor/api/ModelArmor.Samples.Tests/ModelArmor.Samples.Tests.csproj @@ -6,7 +6,7 @@ - +