add failure policy to upload directory (#4181)

Author: philasmar

generator/.DevConfigs/c49077d9-90b3-437f-b316-6d8d8833ae77.json

@@ -0,0 +1,12 @@
+{
+  "services": [
+    {
+      "serviceName": "S3",
+      "type": "minor",
+      "changeLogMessages": [
+        "Add FailurePolicy property to TransferUtilityUploadDirectoryRequest to allow configuration of failure handling behavior during directory uploads. The default behavior is set to abort on failure. Users can now choose to either abort the entire operation or continue uploading remaining files when a failure occurs.",
+        "Add ObjectUploadFailedEvent event to TransferUtilityUploadDirectoryRequest to notify users when an individual file upload fails during a directory upload operation. This event provides details about the failed upload, including the original request, the specific file request and the exception encountered."
+      ]
+    }
+  ]
+}

generator/ServiceClientGeneratorLib/Generators/SourceFiles/AssemblyInfo.cs

@@ -1,7 +1,7 @@
 // ------------------------------------------------------------------------------
 // <auto-generated>
 //     This code was generated by a tool.
-//     Runtime Version: 17.0.0.0
+//     Runtime Version: 18.0.0.0
 //  
 //     Changes to this file may cause incorrect behavior and will be lost if
 //     the code is regenerated.
@@ -15,8 +15,8 @@ namespace ServiceClientGenerator.Generators.SourceFiles
     /// Class to produce the template output
     /// </summary>
 
-    #line 1 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
-    [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.TextTemplating", "17.0.0.0")]
+    #line 1 "D:\CodeBase\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
+    [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.TextTemplating", "18.0.0.0")]
     public partial class AssemblyInfo : BaseGenerator
     {
 #line hidden
@@ -36,43 +36,43 @@ public override string TransformText()
 // associated with an assembly.
 [assembly: AssemblyTitle(""");
 
-            #line 12 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
+            #line 12 "D:\CodeBase\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
             this.Write(this.ToStringHelper.ToStringWithCulture(this.Config.AssemblyTitle));
 
             #line default
             #line hidden
             this.Write("\")]\r\n#if BCL\r\n[assembly: AssemblyDescription(\"");
 
-            #line 14 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
+            #line 14 "D:\CodeBase\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
             this.Write(this.ToStringHelper.ToStringWithCulture(this.Config.AssemblyDescription(versionIdentifier: "4.7.2")));
 
             #line default
             #line hidden
             this.Write("\")]\r\n#elif NETSTANDARD20\r\n[assembly: AssemblyDescription(\"");
 
-            #line 16 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
+            #line 16 "D:\CodeBase\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
             this.Write(this.ToStringHelper.ToStringWithCulture(this.Config.AssemblyDescription(versionIdentifier: "NetStandard 2.0")));
 
             #line default
             #line hidden
             this.Write("\")]\r\n#elif NETCOREAPP3_1\r\n[assembly: AssemblyDescription(\"");
 
-            #line 18 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
+            #line 18 "D:\CodeBase\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
             this.Write(this.ToStringHelper.ToStringWithCulture(this.Config.AssemblyDescription(versionIdentifier: ".NET Core 3.1")));
 
             #line default
             #line hidden
             this.Write("\")]\r\n#elif NET8_0\r\n[assembly: AssemblyDescription(\"");
 
-            #line 20 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
+            #line 20 "D:\CodeBase\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
             this.Write(this.ToStringHelper.ToStringWithCulture(this.Config.AssemblyDescription(versionIdentifier: ".NET 8.0")));
 
             #line default
             #line hidden
             this.Write("\")]\r\n#else\r\n#error Unknown platform constant - unable to set correct AssemblyDesc" +
                     "ription\r\n#endif\r\n\r\n");
 
-            #line 25 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
+            #line 25 "D:\CodeBase\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
  if (this.Config.AssemblyTitle=="AWSSDK.DynamoDBv2") { 
 
             #line default
@@ -81,23 +81,43 @@ public override string TransformText()
 [assembly: InternalsVisibleTo(""AWSSDK.UnitTests.NetFramework, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4"")]
 ");
 
-            #line 28 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
+            #line 28 "D:\CodeBase\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
  } 
 
             #line default
             #line hidden
 
-            #line 29 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
+            #line 29 "D:\CodeBase\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
  if (this.Config.AssemblyTitle=="AWSSDK.S3") { 
 
             #line default
             #line hidden
-            this.Write(@"[assembly: InternalsVisibleTo(""AWSSDK.UnitTests.S3.NetFramework, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4"")]
-[assembly: InternalsVisibleTo(""AWSSDK.UnitTests.NetFramework, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4"")]
-[assembly: InternalsVisibleTo(""DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7"")]
-");
-            
-            #line 33 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
+            this.Write("[assembly: InternalsVisibleTo(\"AWSSDK.UnitTests.S3.NetFramework, PublicKey=002400" +
+                    "0004800000940000000602000000240000525341310004000001000100db5f59f098d27276c78338" +
+                    "75a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa7" +
+                    "6adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a4799" +
+                    "67c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4\")]\r\n[" +
+                    "assembly: InternalsVisibleTo(\"AWSSDK.UnitTests.NetFramework, PublicKey=002400000" +
+                    "4800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a" +
+                    "6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76ad" +
+                    "c811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c" +
+                    "3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4\")]\r\n\r\n//" +
+                    " We should remove this in the future when TransferUtility Upload/Download direct" +
+                    "ory methods return responses.\r\n// We should update the Integration Tests in Tran" +
+                    "sferUtilityTests.cs to not use the internal methods and instead use the new publ" +
+                    "ic ones that return responses.\r\n[assembly: InternalsVisibleTo(\"AWSSDK.Integratio" +
+                    "nTests.S3.NetFramework, PublicKey=0024000004800000940000000602000000240000525341" +
+                    "310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745" +
+                    "db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba89" +
+                    "1f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28" +
+                    "f3a0752a81203f03ccb7f684db373518b4\")]\r\n[assembly: InternalsVisibleTo(\"DynamicPro" +
+                    "xyGenAssembly2, PublicKey=002400000480000094000000060200000024000052534131000400" +
+                    "0001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734a" +
+                    "a39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64" +
+                    "bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be" +
+                    "11e6a7d3113e92484cf7045cc7\")]\r\n");
+            
+            #line 37 "D:\CodeBase\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
  } 
 
             #line default
@@ -126,14 +146,14 @@ public override string TransformText()
 // [assembly: AssemblyVersion(""1.0.*"")]
 [assembly: AssemblyVersion(""");
 
-            #line 56 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
+            #line 60 "D:\CodeBase\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
             this.Write(this.ToStringHelper.ToStringWithCulture(this.Config.ServiceVersion));
 
             #line default
             #line hidden
             this.Write("\")]\r\n[assembly: AssemblyFileVersion(\"");
 
-            #line 57 "C:\dev\repos\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
+            #line 61 "D:\CodeBase\aws-sdk-net\generator\ServiceClientGeneratorLib\Generators\SourceFiles\AssemblyInfo.tt"
             this.Write(this.ToStringHelper.ToStringWithCulture(this.Config.ServiceFileVersion));
 
             #line default

generator/ServiceClientGeneratorLib/Generators/SourceFiles/AssemblyInfo.tt

@@ -29,6 +29,10 @@ using System.Runtime.CompilerServices;
 <# if (this.Config.AssemblyTitle=="AWSSDK.S3") { #>
 [assembly: InternalsVisibleTo("AWSSDK.UnitTests.S3.NetFramework, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4")]
 [assembly: InternalsVisibleTo("AWSSDK.UnitTests.NetFramework, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4")]
+
+// We should remove this in the future when TransferUtility Upload/Download directory methods return responses.
+// We should update the Integration Tests in TransferUtilityTests.cs to not use the internal methods and instead use the new public ones that return responses.
+[assembly: InternalsVisibleTo("AWSSDK.IntegrationTests.S3.NetFramework, PublicKey=0024000004800000940000000602000000240000525341310004000001000100db5f59f098d27276c7833875a6263a3cc74ab17ba9a9df0b52aedbe7252745db7274d5271fd79c1f08f668ecfa8eaab5626fa76adc811d3c8fc55859b0d09d3bc0a84eecd0ba891f2b8a2fc55141cdcc37c2053d53491e650a479967c3622762977900eddbf1252ed08a2413f00a28f3a0752a81203f03ccb7f684db373518b4")]
 [assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")]
 <# } #>
 [assembly: AssemblyConfiguration("")]

sdk/src/Services/S3/Custom/Transfer/Model/DirectoryResult.cs renamed to sdk/src/Services/S3/Custom/Transfer/DirectoryResult.cs

@@ -20,7 +20,7 @@
  *
  */
 
-namespace Amazon.S3.Transfer.Model
+namespace Amazon.S3.Transfer
 {
     /// <summary>
     /// Overall outcome of a directory operation.

sdk/src/Services/S3/Custom/Transfer/Model/FailurePolicy.cs renamed to sdk/src/Services/S3/Custom/Transfer/FailurePolicy.cs

@@ -20,7 +20,7 @@
  *
  */
 
-namespace Amazon.S3.Transfer.Model
+namespace Amazon.S3.Transfer
 {
     /// <summary>
     /// Specifies the policy to apply when a failure occurs during a directory transfer operation.

sdk/src/Services/S3/Custom/Transfer/Internal/DownloadDirectoryCommand.cs

@@ -31,7 +31,6 @@
 using Amazon.S3.Util;
 using Amazon.Util.Internal;
 using Amazon.Runtime;
-using Amazon.S3.Transfer.Model;
 
 namespace Amazon.S3.Transfer.Internal
 {

sdk/src/Services/S3/Custom/Transfer/Internal/UploadDirectoryCommand.cs

@@ -20,6 +20,7 @@
  *
  */
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
 using System.Text;
@@ -34,12 +35,15 @@ namespace Amazon.S3.Transfer.Internal
     /// </summary>
     internal partial class UploadDirectoryCommand : BaseCommand<TransferUtilityUploadDirectoryResponse>
     {
+        private IFailurePolicy _failurePolicy;
+        private ConcurrentBag<Exception> _errors = new ConcurrentBag<Exception>();
         TransferUtilityUploadDirectoryRequest _request;
         TransferUtility _utility;
         TransferUtilityConfig _config;
 
         int _totalNumberOfFiles;
         int _numberOfFilesUploaded;
+        int _numberOfFilesSuccessfullyUploaded;
         long _totalBytes;
         long _transferredBytes;        
 
@@ -48,6 +52,10 @@ internal UploadDirectoryCommand(TransferUtility utility, TransferUtilityConfig c
             this._utility = utility;
             this._request = request;
             this._config = config;
+            _failurePolicy =
+                request.FailurePolicy == FailurePolicy.AbortOnFailure
+                    ? new AbortOnFailurePolicy()
+                    : new ContinueOnFailurePolicy(_errors);
         }
 
         internal TransferUtilityUploadRequest ConstructRequest(string basePath, string filepath, string prefix)

sdk/src/Services/S3/Custom/Transfer/Internal/_async/BaseCommand.async.cs

@@ -13,12 +13,6 @@
  * permissions and limitations under the License.
  */
 
-using Amazon.S3.Model;
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -30,24 +24,5 @@ internal abstract partial class BaseCommand<TResponse> where TResponse : class
         /// Executes the command and returns a typed response
         /// </summary>
         public abstract Task<TResponse> ExecuteAsync(CancellationToken cancellationToken);
-
-        protected static async Task ExecuteCommandAsync<T>(BaseCommand<T> command, CancellationTokenSource internalCts) where T : class
-        {
-            try
-            {
-                await command.ExecuteAsync(internalCts.Token)
-                    .ConfigureAwait(continueOnCapturedContext: false);
-            }
-            catch (Exception exception)
-            {
-                if (!(exception is OperationCanceledException))
-                {
-                    // Cancel scheduling any more tasks.
-                    // Cancel other upload requests.
-                    internalCts.Cancel();
-                }
-                throw;
-            }
-        }
     }
 }

sdk/src/Services/S3/Custom/Transfer/Internal/_bcl+netstandard/DownloadDirectoryCommand.cs

@@ -15,7 +15,6 @@
 
 using Amazon.S3.Model;
 using Amazon.S3.Util;
-using Amazon.S3.Transfer.Model;
 using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;