Adding No overwrite for copy operation (#9592)

rakshil14-2 · web-flow · commit 2493f8b41e73 · 2025-07-25T17:31:50.000-04:00
diff --git a/awscli/customizations/s3/results.py b/awscli/customizations/s3/results.py
@@ -23,7 +23,6 @@
 from awscli.customizations.s3.subscribers import OnDoneFilteredSubscriber
 from awscli.customizations.s3.utils import (
     WarningResult,
-    create_warning,
     human_readable_size,
 )
 from awscli.customizations.utils import uni_print
diff --git a/awscli/customizations/s3/s3handler.py b/awscli/customizations/s3/s3handler.py
@@ -13,6 +13,7 @@
 import logging
 import os
 
+from botocore.exceptions import ClientError
 from s3transfer.manager import TransferManager
 
 from awscli.compat import get_binary_stdin
@@ -481,7 +482,46 @@ def _submit_transfer_request(self, fileinfo, extra_args, subscribers):
         )
 
     def _get_warning_handlers(self):
-        return [self._warn_glacier]
+        return [
+            self._warn_glacier,
+            self._warn_if_zero_byte_file_exists_with_no_overwrite,
+        ]
+
+    def _warn_if_zero_byte_file_exists_with_no_overwrite(self, fileinfo):
+        """
+        Warning handler to skip zero-byte files when no_overwrite is set and file exists.
+
+        This method handles the transfer of zero-byte objects when the no-overwrite parameter is specified.
+        To prevent overwrite, it uses head_object to verify if the object exists at the destination:
+        If the object is present at destination: skip the file (return True)
+        If the object is not present at destination: allow transfer (return False)
+
+        :type fileinfo: FileInfo
+        :param fileinfo: The FileInfo object containing transfer details
+
+        :rtype: bool
+        :return: True if file should be skipped, False if transfer should proceed
+        """
+        if not self._cli_params.get('no_overwrite') or (
+            getattr(fileinfo, 'size') and fileinfo.size > 0
+        ):
+            return False
+
+        bucket, key = find_bucket_key(fileinfo.dest)
+        client = fileinfo.source_client
+        try:
+            client.head_object(Bucket=bucket, Key=key)
+            LOGGER.debug(
+                "Warning: Skipping file %s as it already exists on %s",
+                fileinfo.src,
+                fileinfo.dest,
+            )
+            return True
+        except ClientError as e:
+            if e.response['Error']['Code'] == '404':
+                return False
+            else:
+                raise
 
     def _format_src_dest(self, fileinfo):
         src = self._format_s3_path(fileinfo.src)
diff --git a/awscli/customizations/s3/utils.py b/awscli/customizations/s3/utils.py
@@ -521,6 +521,7 @@ def map_copy_object_params(cls, request_params, cli_params):
         )
         cls._set_request_payer_param(request_params, cli_params)
         cls._set_checksum_algorithm_param(request_params, cli_params)
+        cls._set_no_overwrite_param(request_params, cli_params)
 
     @classmethod
     def map_head_object_params(cls, request_params, cli_params):
diff --git a/awscli/s3transfer/copies.py b/awscli/s3transfer/copies.py
@@ -13,6 +13,7 @@
 import copy
 import math
 
+from botocore.exceptions import ClientError
 from s3transfer.tasks import (
     CompleteMultipartUploadTask,
     CreateMultipartUploadTask,
@@ -67,6 +68,7 @@ class CopySubmissionTask(SubmissionTask):
         'CopySourceSSECustomerKeyMD5',
         'MetadataDirective',
         'TaggingDirective',
+        'IfNoneMatch',
     ]
 
     COMPLETE_MULTIPART_ARGS = [
@@ -75,6 +77,11 @@ class CopySubmissionTask(SubmissionTask):
         'SSECustomerKeyMD5',
         'RequestPayer',
         'ExpectedBucketOwner',
+        'IfNoneMatch',
+    ]
+
+    COPY_OBJECT_ARGS_BLOCKLIST = [
+        'IfNoneMatch',
     ]
 
     def _submit(
@@ -98,14 +105,14 @@ def _submit(
         :param transfer_future: The transfer future associated with the
             transfer request that tasks are being submitted for
         """
+        call_args = transfer_future.meta.call_args
         # Determine the size if it was not provided
         if transfer_future.meta.size is None:
             # If a size was not provided figure out the size for the
             # user. Note that we will only use the client provided to
             # the TransferManager. If the object is outside of the region
             # of the client, they may have to provide the file size themselves
             # with a completely new client.
-            call_args = transfer_future.meta.call_args
             head_object_request = (
                 self._get_head_object_request_from_copy_source(
                     call_args.copy_source
@@ -127,10 +134,24 @@ def _submit(
             transfer_future.meta.provide_transfer_size(
                 response['ContentLength']
             )
-
-        # If it is greater than threshold do a multipart copy, otherwise
-        # do a regular copy object.
-        if transfer_future.meta.size < config.multipart_threshold:
+        # Check for ifNoneMatch is enabled and file has content
+        # Special handling for 0-byte files: Since multipart copy works with object size
+        # and divides the object into smaller chunks, there's an edge case when the object
+        # size is zero. This would result in 0 parts being calculated, and the
+        # CompleteMultipartUpload operation throws a MalformedXML error when transferring
+        # 0 parts because the XML does not validate against the published schema.
+        # Therefore, 0-byte files are always handled via single copy request regardless
+        # of the multipart threshold setting.
+        should_overwrite = (
+            call_args.extra_args.get("IfNoneMatch")
+            and transfer_future.meta.size != 0
+        )
+        # If it is less than threshold and ifNoneMatch is not in parameters
+        # do a regular copy else do multipart copy.
+        if (
+            transfer_future.meta.size < config.multipart_threshold
+            and not should_overwrite
+        ):
             self._submit_copy_request(
                 client, config, osutil, request_executor, transfer_future
             )
@@ -147,19 +168,25 @@ def _submit_copy_request(
         # Get the needed progress callbacks for the task
         progress_callbacks = get_callbacks(transfer_future, 'progress')
 
-        # Submit the request of a single copy.
+        # Submit the request of a single copy and make sure it
+        # does not include any blocked arguments.
+        copy_object_extra_args = {
+            param: val
+            for param, val in call_args.extra_args.items()
+            if param not in self.COPY_OBJECT_ARGS_BLOCKLIST
+        }
         self._transfer_coordinator.submit(
             request_executor,
             CopyObjectTask(
                 transfer_coordinator=self._transfer_coordinator,
                 main_kwargs={
-                    'client': client,
-                    'copy_source': call_args.copy_source,
-                    'bucket': call_args.bucket,
-                    'key': call_args.key,
-                    'extra_args': call_args.extra_args,
-                    'callbacks': progress_callbacks,
-                    'size': transfer_future.meta.size,
+                    "client": client,
+                    "copy_source": call_args.copy_source,
+                    "bucket": call_args.bucket,
+                    "key": call_args.key,
+                    "extra_args": copy_object_extra_args,
+                    "callbacks": progress_callbacks,
+                    "size": transfer_future.meta.size,
                 },
                 is_final=True,
             ),
diff --git a/awscli/s3transfer/manager.py b/awscli/s3transfer/manager.py
@@ -188,14 +188,14 @@ class TransferManager:
         'SSEKMSEncryptionContext',
         'Tagging',
         'WebsiteRedirectLocation',
+        'IfNoneMatch',
     ]
 
     ALLOWED_UPLOAD_ARGS = (
         _ALLOWED_SHARED_ARGS
         + [
             'ChecksumType',
             'MpuObjectSize',
-            'IfNoneMatch',
         ]
         + FULL_OBJECT_CHECKSUM_ARGS
     )
diff --git a/tests/functional/s3/__init__.py b/tests/functional/s3/__init__.py
@@ -257,6 +257,16 @@ def mp_copy_responses(self):
             self.complete_mpu_response(),
         ]
 
+    def precondition_failed_error_response(self, condition='If-None-Match'):
+        return {
+            'Error': {
+                'Code': 'PreconditionFailed',
+                'Message': 'At least one of the pre-conditions you '
+                'specified did not hold',
+                'Condition': condition,
+            }
+        }
+
 
 class BaseS3CLIRunnerTest(unittest.TestCase):
     def setUp(self):
diff --git a/tests/functional/s3/test_cp_command.py b/tests/functional/s3/test_cp_command.py
@@ -335,7 +335,6 @@ def test_no_overwrite_flag_multipart_upload_when_object_not_exists_on_target(
         # Create a large file that will trigger multipart upload
         full_path = self.files.create_file('foo.txt', 'a' * 10 * (1024**2))
         cmdline = f'{self.prefix} {full_path} s3://bucket --no-overwrite'
-
         # Set up responses for multipart upload
         self.parsed_responses = [
             {'UploadId': 'foo'},  # CreateMultipartUpload response
@@ -397,6 +396,160 @@ def test_no_overwrite_flag_multipart_upload_when_object_exists_on_target(
         # Verify the IfNoneMatch condition was set in the CompleteMultipartUpload request
         self.assertEqual(self.operations_called[3][1]['IfNoneMatch'], '*')
 
+    def test_no_overwrite_flag_on_copy_when_small_object_does_not_exist_on_target(
+        self,
+    ):
+        cmdline = f'{self.prefix} s3://bucket1/key.txt s3://bucket/key1.txt --no-overwrite'
+        # Set up responses for multipart copy (since no-overwrite always uses multipart)
+        self.parsed_responses = [
+            self.head_object_response(),  # HeadObject to get source metadata
+            self.create_mpu_response('foo'),  # CreateMultipartUpload response
+            self.upload_part_copy_response(),  # UploadPartCopy response
+            {},  # CompleteMultipartUpload response
+        ]
+        self.run_cmd(cmdline, expected_rc=0)
+        # Verify all multipart operations were called
+        self.assertEqual(len(self.operations_called), 4)
+        self.assertEqual(self.operations_called[0][0].name, 'HeadObject')
+        self.assertEqual(
+            self.operations_called[1][0].name, 'CreateMultipartUpload'
+        )
+        self.assertEqual(self.operations_called[2][0].name, 'UploadPartCopy')
+        self.assertEqual(
+            self.operations_called[3][0].name, 'CompleteMultipartUpload'
+        )
+        # Verify the IfNoneMatch condition was set in the CompleteMultipartUpload request
+        self.assertEqual(self.operations_called[3][1]['IfNoneMatch'], '*')
+
+    def test_no_overwrite_flag_on_copy_when_small_object_exists_on_target(
+        self,
+    ):
+        cmdline = f'{self.prefix} s3://bucket1/key.txt s3://bucket/key.txt --no-overwrite'
+        # Set up responses for multipart copy (since no-overwrite always uses multipart)
+        self.parsed_responses = [
+            self.head_object_response(),  # HeadObject to get source metadata
+            self.create_mpu_response('foo'),  # CreateMultipartUpload response
+            self.upload_part_copy_response(),  # UploadPartCopy response
+            self.precondition_failed_error_response(),  # CompleteMultipartUpload
+            {},  # AbortMultipartUpload response
+        ]
+        self.run_cmd(cmdline, expected_rc=0)
+        # Verify all multipart operations were called
+        self.assertEqual(len(self.operations_called), 5)
+        self.assertEqual(self.operations_called[0][0].name, 'HeadObject')
+        self.assertEqual(
+            self.operations_called[1][0].name, 'CreateMultipartUpload'
+        )
+        self.assertEqual(self.operations_called[2][0].name, 'UploadPartCopy')
+        self.assertEqual(
+            self.operations_called[3][0].name, 'CompleteMultipartUpload'
+        )
+        self.assertEqual(
+            self.operations_called[4][0].name, 'AbortMultipartUpload'
+        )
+        # Verify the IfNoneMatch condition was set in the CompleteMultipartUpload request
+        self.assertEqual(self.operations_called[3][1]['IfNoneMatch'], '*')
+
+    def test_no_overwrite_flag_on_copy_when_zero_size_object_exists_at_destination(
+        self,
+    ):
+        cmdline = f'{self.prefix} s3://bucket1/file.txt s3://bucket2/file.txt --no-overwrite'
+        self.parsed_responses = [
+            self.head_object_response(
+                ContentLength=0
+            ),  # Source object (zero size)
+            self.head_object_response(),  # Checking the object at destination
+        ]
+        self.run_cmd(cmdline, expected_rc=0)
+        self.assertEqual(len(self.operations_called), 2)
+        self.assertEqual(self.operations_called[0][0].name, 'HeadObject')
+        self.assertEqual(self.operations_called[1][0].name, 'HeadObject')
+
+    def test_no_overwrite_flag_on_copy_when_zero_size_object_not_exists_at_destination(
+        self,
+    ):
+        cmdline = f'{self.prefix} s3://bucket1/file.txt s3://bucket2/file1.txt --no-overwrite'
+        self.parsed_responses = [
+            self.head_object_response(
+                ContentLength=0
+            ),  # Source object (zero size)
+            {
+                'Error': {'Code': '404', 'Message': 'Not Found'}
+            },  # At destination object does not exists
+            self.copy_object_response(),  # Copy Request when object does not exists
+        ]
+        self.run_cmd(cmdline, expected_rc=0)
+        self.assertEqual(len(self.operations_called), 3)
+        self.assertEqual(self.operations_called[0][0].name, 'HeadObject')
+        self.assertEqual(self.operations_called[1][0].name, 'HeadObject')
+        self.assertEqual(self.operations_called[2][0].name, 'CopyObject')
+
+    def test_no_overwrite_flag_on_copy_when_large_object_exists_on_target(
+        self,
+    ):
+        cmdline = f'{self.prefix} s3://bucket1/key.txt s3://bucket/key.txt --no-overwrite'
+        # Set up responses for multipart copy with large object
+        self.parsed_responses = [
+            self.head_object_response(
+                ContentLength=10 * (1024**2)
+            ),  # HeadObject with large content
+            self.get_object_tagging_response({}),  # GetObjectTagging response
+            self.create_mpu_response('foo'),  # CreateMultipartUpload response
+            self.upload_part_copy_response(),  # UploadPartCopy response part 1
+            self.upload_part_copy_response(),  # UploadPartCopy response part 2
+            self.precondition_failed_error_response(),  # CompleteMultipartUpload fails with PreconditionFailed
+            {},  # AbortMultipartUpload response
+        ]
+        self.run_cmd(cmdline, expected_rc=0)
+        # Verify all multipart operations were called
+        self.assertEqual(len(self.operations_called), 7)
+        self.assertEqual(self.operations_called[0][0].name, 'HeadObject')
+        self.assertEqual(self.operations_called[1][0].name, 'GetObjectTagging')
+        self.assertEqual(
+            self.operations_called[2][0].name, 'CreateMultipartUpload'
+        )
+        self.assertEqual(self.operations_called[3][0].name, 'UploadPartCopy')
+        self.assertEqual(self.operations_called[4][0].name, 'UploadPartCopy')
+        self.assertEqual(
+            self.operations_called[5][0].name, 'CompleteMultipartUpload'
+        )
+        self.assertEqual(
+            self.operations_called[6][0].name, 'AbortMultipartUpload'
+        )
+        # Verify the IfNoneMatch condition was set in the CompleteMultipartUpload request
+        self.assertEqual(self.operations_called[5][1]['IfNoneMatch'], '*')
+
+    def test_no_overwrite_flag_on_copy_when_large_object_does_not_exist_on_target(
+        self,
+    ):
+        cmdline = f'{self.prefix} s3://bucket1/key.txt s3://bucket/key1.txt --no-overwrite'
+        # Set up responses for multipart copy with large object
+        self.parsed_responses = [
+            self.head_object_response(
+                ContentLength=10 * (1024**2)
+            ),  # HeadObject with large content
+            self.get_object_tagging_response({}),  # GetObjectTagging response
+            self.create_mpu_response('foo'),  # CreateMultipartUpload response
+            self.upload_part_copy_response(),  # UploadPartCopy response part 1
+            self.upload_part_copy_response(),  # UploadPartCopy response part 2
+            {},  # CompleteMultipartUpload response
+        ]
+        self.run_cmd(cmdline, expected_rc=0)
+        # Verify all multipart operations were called
+        self.assertEqual(len(self.operations_called), 6)
+        self.assertEqual(self.operations_called[0][0].name, 'HeadObject')
+        self.assertEqual(self.operations_called[1][0].name, 'GetObjectTagging')
+        self.assertEqual(
+            self.operations_called[2][0].name, 'CreateMultipartUpload'
+        )
+        self.assertEqual(self.operations_called[3][0].name, 'UploadPartCopy')
+        self.assertEqual(self.operations_called[4][0].name, 'UploadPartCopy')
+        self.assertEqual(
+            self.operations_called[5][0].name, 'CompleteMultipartUpload'
+        )
+        # Verify the IfNoneMatch condition was set in the CompleteMultipartUpload request
+        self.assertEqual(self.operations_called[5][1]['IfNoneMatch'], '*')
+
     def test_dryrun_download(self):
         self.parsed_responses = [self.head_object_response()]
         target = self.files.full_path('file.txt')
diff --git a/tests/functional/s3/test_mv_command.py b/tests/functional/s3/test_mv_command.py
diff --git a/tests/functional/s3transfer/test_copy.py b/tests/functional/s3transfer/test_copy.py
diff --git a/tests/unit/customizations/s3/test_utils.py b/tests/unit/customizations/s3/test_utils.py

Original file line number	Diff line number	Diff line change
`@@ -23,7 +23,6 @@`
`23`	`23`	`from awscli.customizations.s3.subscribers import OnDoneFilteredSubscriber`
`24`	`24`	`from awscli.customizations.s3.utils import (`
`25`	`25`	`WarningResult,`
`26`		`- create_warning,`
`27`	`26`	`human_readable_size,`
`28`	`27`	`)`
`29`	`28`	`from awscli.customizations.utils import uni_print`
Original file line number	Diff line number	Diff line change
`@@ -521,6 +521,7 @@ def map_copy_object_params(cls, request_params, cli_params):`
`521`	`521`	`)`
`522`	`522`	`cls._set_request_payer_param(request_params, cli_params)`
`523`	`523`	`cls._set_checksum_algorithm_param(request_params, cli_params)`
	`524`	`+ cls._set_no_overwrite_param(request_params, cli_params)`
`524`	`525`
`525`	`526`	`@classmethod`
`526`	`527`	`def map_head_object_params(cls, request_params, cli_params):`
Original file line number	Diff line number	Diff line change
`@@ -188,14 +188,14 @@ class TransferManager:`
`188`	`188`	`'SSEKMSEncryptionContext',`
`189`	`189`	`'Tagging',`
`190`	`190`	`'WebsiteRedirectLocation',`
	`191`	`+ 'IfNoneMatch',`
`191`	`192`	`]`
`192`	`193`
`193`	`194`	`ALLOWED_UPLOAD_ARGS = (`
`194`	`195`	`_ALLOWED_SHARED_ARGS`
`195`	`196`	`+ [`
`196`	`197`	`'ChecksumType',`
`197`	`198`	`'MpuObjectSize',`
`198`		`- 'IfNoneMatch',`
`199`	`199`	`]`
`200`	`200`	`+ FULL_OBJECT_CHECKSUM_ARGS`
`201`	`201`	`)`