From 1db4ae6bd0a3bbb74d0fd841c1425896441a01eb Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Thu, 22 Jan 2026 14:44:20 -0500 Subject: [PATCH 1/4] Add support for rename of S3 storage backend fixes: #7228 (cherry picked from commit 0d632d495492d885532b54f5ec7e425ee729f374) --- CHANGES/7228.bugfix | 1 + pulpcore/app/serializers/domain.py | 4 +- pulpcore/app/util.py | 5 +- pulpcore/constants.py | 1 + pulpcore/content/handler.py | 5 +- .../api/test_artifact_distribution.py | 1 + .../tests/functional/api/test_crud_domains.py | 6 ++- .../tests/unit/serializers/test_domain.py | 51 +++++++++++++++++-- 8 files changed, 66 insertions(+), 8 deletions(-) create mode 100644 CHANGES/7228.bugfix diff --git a/CHANGES/7228.bugfix b/CHANGES/7228.bugfix new file mode 100644 index 00000000000..56454c2b5fc --- /dev/null +++ b/CHANGES/7228.bugfix @@ -0,0 +1 @@ +Fixed not supporting the new rename of the S3 storage backend. diff --git a/pulpcore/app/serializers/domain.py b/pulpcore/app/serializers/domain.py index 4348081066a..e15d65b7fbb 100644 --- a/pulpcore/app/serializers/domain.py +++ b/pulpcore/app/serializers/domain.py @@ -16,7 +16,8 @@ BACKEND_CHOICES = ( ("pulpcore.app.models.storage.FileSystem", "Use local filesystem as storage"), # ("pulpcore.app.models.storage.PulpSFTPStorage", "Use SFTP server as storage"), - ("storages.backends.s3boto3.S3Boto3Storage", "Use Amazon S3 as storage"), + ("storages.backends.s3boto3.S3Boto3Storage", "Use Amazon S3 as storage [deprecated]"), + ("storages.backends.s3.S3Storage", "Use Amazon S3 as storage"), ("storages.backends.azure_storage.AzureStorage", "Use Azure Blob as storage"), # ("storages.backends.gcloud.GoogleCloudStorage", "Use Google Cloud as storage"), ) @@ -115,6 +116,7 @@ class SFTPSettingsSerializer(BaseSettingsClass): class AmazonS3SettingsSerializer(BaseSettingsClass): """A Serializer for Amazon S3 storage settings.""" + STORAGE_CLASS = "storages.backends.s3.S3Storage" SETTING_MAPPING = { "aws_s3_access_key_id": "access_key", "aws_access_key_id": "access_key", diff --git a/pulpcore/app/util.py b/pulpcore/app/util.py index 96f214509c9..5e44f4947d7 100644 --- a/pulpcore/app/util.py +++ b/pulpcore/app/util.py @@ -392,7 +392,10 @@ def get_artifact_url(artifact, headers=None, http_method=None): or not artifact_domain.redirect_to_object_storage ): return _artifact_serving_distribution().artifact_url(artifact) - elif artifact_domain.storage_class == "storages.backends.s3boto3.S3Boto3Storage": + elif artifact_domain.storage_class in ( + "storages.backends.s3boto3.S3Boto3Storage", + "storages.backends.s3.S3Storage", + ): parameters = {"ResponseContentDisposition": content_disposition} if headers and headers.get("Content-Type"): parameters["ResponseContentType"] = headers.get("Content-Type") diff --git a/pulpcore/constants.py b/pulpcore/constants.py index 30759e6696e..ab26722b29d 100644 --- a/pulpcore/constants.py +++ b/pulpcore/constants.py @@ -110,6 +110,7 @@ # Storage-type mapped to storage-response-map STORAGE_RESPONSE_MAP = { "storages.backends.s3boto3.S3Boto3Storage": S3_RESPONSE_HEADER_MAP, + "storages.backends.s3.S3Storage": S3_RESPONSE_HEADER_MAP, "storages.backends.azure_storage.AzureStorage": AZURE_RESPONSE_HEADER_MAP, "storages.backends.gcloud.GoogleCloudStorage": GCS_RESPONSE_HEADER_MAP, } diff --git a/pulpcore/content/handler.py b/pulpcore/content/handler.py index e138a0ba2c8..d3de5e6023b 100644 --- a/pulpcore/content/handler.py +++ b/pulpcore/content/handler.py @@ -988,7 +988,10 @@ def _set_params_from_headers(hdrs, storage_domain): return FileResponse(path, headers=headers) elif not domain.redirect_to_object_storage: return ArtifactResponse(content_artifact.artifact, headers=headers) - elif domain.storage_class == "storages.backends.s3boto3.S3Boto3Storage": + elif domain.storage_class in ( + "storages.backends.s3boto3.S3Boto3Storage", + "storages.backends.s3.S3Storage", + ): headers["Content-Disposition"] = content_disposition parameters = _set_params_from_headers(headers, domain.storage_class) url = URL( diff --git a/pulpcore/tests/functional/api/test_artifact_distribution.py b/pulpcore/tests/functional/api/test_artifact_distribution.py index d6735347c22..c743c8a7188 100644 --- a/pulpcore/tests/functional/api/test_artifact_distribution.py +++ b/pulpcore/tests/functional/api/test_artifact_distribution.py @@ -7,6 +7,7 @@ OBJECT_STORAGES = ( "storages.backends.s3boto3.S3Boto3Storage", + "storages.backends.s3.S3Storage", "storages.backends.azure_storage.AzureStorage", "storages.backends.gcloud.GoogleCloudStorage", ) diff --git a/pulpcore/tests/functional/api/test_crud_domains.py b/pulpcore/tests/functional/api/test_crud_domains.py index 9d9caa84c38..aed86d04051 100644 --- a/pulpcore/tests/functional/api/test_crud_domains.py +++ b/pulpcore/tests/functional/api/test_crud_domains.py @@ -187,6 +187,7 @@ def test_special_domain_creation(domains_api_client, gen_object_with_cleanup): "pulpcore.app.models.storage.FileSystem", # "pulpcore.app.models.storage.PulpSFTPStorage", "storages.backends.s3boto3.S3Boto3Storage", + "storages.backends.s3.S3Storage", "storages.backends.azure_storage.AzureStorage", # "storages.backends.gcloud.GoogleCloudStorage", } @@ -201,7 +202,7 @@ def test_special_domain_creation(domains_api_client, gen_object_with_cleanup): "key_filename": "/etc/pulp/certs/storage_id_ed25519", }, }, - "storages.backends.s3boto3.S3Boto3Storage": { + "storages.backends.s3.S3Storage": { "AWS_ACCESS_KEY_ID": "random", "AWS_SECRET_ACCESS_KEY": "random", "AWS_STORAGE_BUCKET_NAME": "pulp3", @@ -224,6 +225,9 @@ def test_special_domain_creation(domains_api_client, gen_object_with_cleanup): "GS_CUSTOM_ENDPOINT": "http://custom-endpoint", }, } + storage_settings["storages.backends.s3boto3.S3Boto3Storage"] = storage_settings[ + "storages.backends.s3.S3Storage" + ] installed_backends = [] domain_names = set() diff --git a/pulpcore/tests/unit/serializers/test_domain.py b/pulpcore/tests/unit/serializers/test_domain.py index 68c852db5f5..651ee960c62 100644 --- a/pulpcore/tests/unit/serializers/test_domain.py +++ b/pulpcore/tests/unit/serializers/test_domain.py @@ -30,6 +30,7 @@ def _no_validate_storage_backend(monkeypatch): params=[ "pulpcore.app.models.storage.FileSystem", "storages.backends.s3boto3.S3Boto3Storage", + "storages.backends.s3.S3Storage", "storages.backends.azure_storage.AzureStorage", ] ) @@ -41,7 +42,10 @@ def storage_class(request): def serializer_class(storage_class): if storage_class == "pulpcore.app.models.storage.FileSystem": return FileSystemSettingsSerializer - elif storage_class == "storages.backends.s3boto3.S3Boto3Storage": + elif storage_class in ( + "storages.backends.s3boto3.S3Boto3Storage", + "storages.backends.s3.S3Storage", + ): return AmazonS3SettingsSerializer elif storage_class == "storages.backends.azure_storage.AzureStorage": return AzureSettingsSerializer @@ -51,14 +55,28 @@ def serializer_class(storage_class): def required_settings(storage_class): if storage_class == "pulpcore.app.models.storage.FileSystem": return {"location": "/var/lib/pulp/media/"} - elif storage_class == "storages.backends.s3boto3.S3Boto3Storage": - return {"access_key": "testing", "secret_key": "secret", "bucket_name": "test"} + elif storage_class in ( + "storages.backends.s3boto3.S3Boto3Storage", + "storages.backends.s3.S3Storage", + ): + return {"access_key": "testing", "bucket_name": "test"} elif storage_class == "storages.backends.azure_storage.AzureStorage": return {"account_name": "test", "account_key": "secret", "azure_container": "test"} @pytest.fixture -def all_settings(serializer_class, required_settings): +def extra_required_settings(storage_class): + """For fields required in the serializer's validate, but not on the field itself.""" + if storage_class in ( + "storages.backends.s3boto3.S3Boto3Storage", + "storages.backends.s3.S3Storage", + ): + return {"secret_key": "secret"} + return {} + + +@pytest.fixture +def all_settings(serializer_class, required_settings, extra_required_settings): serializer = serializer_class() fields = serializer.get_fields() default_settings = { @@ -113,6 +131,31 @@ def test_using_setting_names(storage_class, serializer_class, all_settings): assert storage_settings == all_settings +<<<<<<< HEAD +======= +@pytest.mark.django_db +def test_cloudfront_s3_storage_settings(storage_class, required_settings): + if storage_class not in ( + "storages.backends.s3boto3.S3Boto3Storage", + "storages.backends.s3.S3Storage", + ): + pytest.skip("This test only make sense when using S3 as storage backend.") + + domain = SimpleNamespace(storage_class=storage_class, **MIN_DOMAIN_SETTINGS) + data = { + "storage_settings": { + "secret_key": "secret_key", + "custom_domain": "custom_domain.cloudfront.net", + "cloudfront_key_id": "key_id", + "cloudfront_key": "cloudfront_key", + **required_settings, + } + } + serializer = DomainSerializer(domain, data=data, partial=True) + + assert serializer.is_valid(raise_exception=True) + + class DomainSettingsBaseMixin: storage_class = None serializer_class = None From ee02412fb13b9651ef5eaae48660b9b290641290 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Mon, 26 Jan 2026 10:53:16 -0500 Subject: [PATCH 2/4] Fixed test_crud_domains random backend testing (cherry picked from commit fd3114522ce5db5942b87151d9266cdea4fb2e2f) --- pulpcore/tests/functional/api/test_crud_domains.py | 4 +++- pulpcore/tests/unit/serializers/test_domain.py | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pulpcore/tests/functional/api/test_crud_domains.py b/pulpcore/tests/functional/api/test_crud_domains.py index aed86d04051..58907a4c66e 100644 --- a/pulpcore/tests/functional/api/test_crud_domains.py +++ b/pulpcore/tests/functional/api/test_crud_domains.py @@ -245,7 +245,8 @@ def test_special_domain_creation(domains_api_client, gen_object_with_cleanup): assert e.status == 400 assert "Backend is not installed on Pulp." in e.body else: - installed_backends.append(backend) + if backend != "storages.backends.s3boto3.S3Boto3Storage": + installed_backends.append(backend) domain_names.add(domain.name) # Try creating domains with correct settings for backend in installed_backends: @@ -258,6 +259,7 @@ def test_special_domain_creation(domains_api_client, gen_object_with_cleanup): domain_names.add(domain.name) # Try creating domains with incorrect settings + storage_types.remove("storages.backends.s3boto3.S3Boto3Storage") for backend in installed_backends: random_backend = random.choice(tuple(storage_types - {backend})) body = { diff --git a/pulpcore/tests/unit/serializers/test_domain.py b/pulpcore/tests/unit/serializers/test_domain.py index 651ee960c62..5fd12a02347 100644 --- a/pulpcore/tests/unit/serializers/test_domain.py +++ b/pulpcore/tests/unit/serializers/test_domain.py @@ -131,8 +131,6 @@ def test_using_setting_names(storage_class, serializer_class, all_settings): assert storage_settings == all_settings -<<<<<<< HEAD -======= @pytest.mark.django_db def test_cloudfront_s3_storage_settings(storage_class, required_settings): if storage_class not in ( From a2141896a66dc719683b530ed32e2922757a4440 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Mon, 26 Jan 2026 14:14:51 -0500 Subject: [PATCH 3/4] Add missing storage class mapping for storages.backends.s3.S3Storage --- pulpcore/app/serializers/domain.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pulpcore/app/serializers/domain.py b/pulpcore/app/serializers/domain.py index e15d65b7fbb..0219a9f9f16 100644 --- a/pulpcore/app/serializers/domain.py +++ b/pulpcore/app/serializers/domain.py @@ -263,6 +263,7 @@ class StorageSettingsSerializer(serializers.Serializer): STORAGE_MAPPING = { "pulpcore.app.models.storage.FileSystem": FileSystemSettingsSerializer, "pulpcore.app.models.storage.PulpSFTPStorage": SFTPSettingsSerializer, + "storages.backends.s3.S3Storage": AmazonS3SettingsSerializer, "storages.backends.s3boto3.S3Boto3Storage": AmazonS3SettingsSerializer, "storages.backends.azure_storage.AzureStorage": AzureSettingsSerializer, "storages.backends.gcloud.GoogleCloudStorage": GoogleSettingsSerializer, From 0efb835e57cf0970032192c0c537c9ed1027bb58 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Mon, 26 Jan 2026 14:31:43 -0500 Subject: [PATCH 4/4] Keep original fixture behavior, just add new backend name to if statements --- pulpcore/tests/unit/serializers/test_domain.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/pulpcore/tests/unit/serializers/test_domain.py b/pulpcore/tests/unit/serializers/test_domain.py index 5fd12a02347..0a213c1b8ab 100644 --- a/pulpcore/tests/unit/serializers/test_domain.py +++ b/pulpcore/tests/unit/serializers/test_domain.py @@ -59,24 +59,13 @@ def required_settings(storage_class): "storages.backends.s3boto3.S3Boto3Storage", "storages.backends.s3.S3Storage", ): - return {"access_key": "testing", "bucket_name": "test"} + return {"access_key": "testing", "secret_key": "secret", "bucket_name": "test"} elif storage_class == "storages.backends.azure_storage.AzureStorage": return {"account_name": "test", "account_key": "secret", "azure_container": "test"} @pytest.fixture -def extra_required_settings(storage_class): - """For fields required in the serializer's validate, but not on the field itself.""" - if storage_class in ( - "storages.backends.s3boto3.S3Boto3Storage", - "storages.backends.s3.S3Storage", - ): - return {"secret_key": "secret"} - return {} - - -@pytest.fixture -def all_settings(serializer_class, required_settings, extra_required_settings): +def all_settings(serializer_class, required_settings): serializer = serializer_class() fields = serializer.get_fields() default_settings = {