diff --git a/CHANGES/1250.feature b/CHANGES/1250.feature new file mode 100644 index 000000000..85737a745 --- /dev/null +++ b/CHANGES/1250.feature @@ -0,0 +1 @@ +Enabled the checkpoint feature in pulp_deb. \ No newline at end of file diff --git a/pulp_deb/app/serializers/publication_serializers.py b/pulp_deb/app/serializers/publication_serializers.py index 74e9059cb..a7d8c2cf5 100644 --- a/pulp_deb/app/serializers/publication_serializers.py +++ b/pulp_deb/app/serializers/publication_serializers.py @@ -1,4 +1,5 @@ from rest_framework.serializers import BooleanField, ValidationError +from rest_framework import serializers from pulpcore.plugin.models import Publication from pulpcore.plugin.serializers import ( RelatedField, @@ -36,6 +37,7 @@ class AptPublicationSerializer(PublicationSerializer): ) structured = BooleanField(help_text="Activate structured publishing mode.", default=True) publish_upstream_release_fields = BooleanField(help_text="", required=False) + checkpoint = serializers.BooleanField(required=False) signing_service = RelatedField( help_text="Sign Release files with this signing key", many=False, @@ -57,6 +59,7 @@ class Meta: fields = PublicationSerializer.Meta.fields + ( "simple", "structured", + "checkpoint", "signing_service", "publish_upstream_release_fields", ) @@ -75,7 +78,8 @@ class AptDistributionSerializer(DistributionSerializer): queryset=Publication.objects.exclude(complete=False), allow_null=True, ) + checkpoint = serializers.BooleanField(required=False) class Meta: - fields = DistributionSerializer.Meta.fields + ("publication",) + fields = DistributionSerializer.Meta.fields + ("publication", "checkpoint") model = AptDistribution diff --git a/pulp_deb/app/tasks/publishing.py b/pulp_deb/app/tasks/publishing.py index f4b612669..489526ae3 100644 --- a/pulp_deb/app/tasks/publishing.py +++ b/pulp_deb/app/tasks/publishing.py @@ -82,6 +82,7 @@ def publish( repository_version_pk, simple, structured, + checkpoint=False, signing_service_pk=None, publish_upstream_release_fields=None, ): @@ -92,6 +93,7 @@ def publish( repository_version_pk (str): Create a publication from this repository version. simple (bool): Create a simple publication with all packages contained in default/all. structured (bool): Create a structured publication with releases and components. + checkpoint (bool): Whether to create a checkpoint publication. signing_service_pk (str): Use this SigningService to sign the Release files. """ @@ -115,7 +117,9 @@ def publish( ) ) with tempfile.TemporaryDirectory(".") as temp_dir: - with AptPublication.create(repo_version, pass_through=False) as publication: + with AptPublication.create( + repo_version, pass_through=False, checkpoint=checkpoint + ) as publication: publication.simple = simple publication.structured = structured publication.signing_service = signing_service diff --git a/pulp_deb/app/viewsets/publication.py b/pulp_deb/app/viewsets/publication.py index f0d5dc567..3c0624132 100644 --- a/pulp_deb/app/viewsets/publication.py +++ b/pulp_deb/app/viewsets/publication.py @@ -211,21 +211,25 @@ def create(self, request): repository_version = serializer.validated_data.get("repository_version") simple = serializer.validated_data.get("simple") structured = serializer.validated_data.get("structured") + checkpoint = serializer.validated_data.get("checkpoint") signing_service = serializer.validated_data.get("signing_service") publish_upstream_release_fields = serializer.validated_data.get( "publish_upstream_release_fields" ) + kwargs = { + "repository_version_pk": repository_version.pk, + "simple": simple, + "structured": structured, + "signing_service_pk": getattr(signing_service, "pk", None), + "publish_upstream_release_fields": publish_upstream_release_fields, + } + if checkpoint: + kwargs["checkpoint"] = True result = dispatch( func=tasks.publish, shared_resources=[repository_version.repository], - kwargs={ - "repository_version_pk": repository_version.pk, - "simple": simple, - "structured": structured, - "signing_service_pk": getattr(signing_service, "pk", None), - "publish_upstream_release_fields": publish_upstream_release_fields, - }, + kwargs=kwargs, ) return OperationPostponedResponse(result, request) diff --git a/pulp_deb/tests/conftest.py b/pulp_deb/tests/conftest.py index 2429606eb..9b3460ea5 100644 --- a/pulp_deb/tests/conftest.py +++ b/pulp_deb/tests/conftest.py @@ -59,7 +59,7 @@ def apt_repository_versions_api(apt_client): def deb_distribution_factory(apt_distribution_api, gen_object_with_cleanup): """Fixture that generates a deb distribution with cleanup from a given publication.""" - def _deb_distribution_factory(publication=None, repository=None): + def _deb_distribution_factory(publication=None, repository=None, checkpoint=None): """Create a deb distribution. :param publication: The publication the distribution is based on. @@ -70,6 +70,8 @@ def _deb_distribution_factory(publication=None, repository=None): body["publication"] = publication.pulp_href if repository: body["repository"] = repository.pulp_href + if checkpoint is not None: + body["checkpoint"] = checkpoint return gen_object_with_cleanup(apt_distribution_api, body) return _deb_distribution_factory diff --git a/pulp_deb/tests/functional/api/test_checkpoint.py b/pulp_deb/tests/functional/api/test_checkpoint.py new file mode 100644 index 000000000..067b5fadc --- /dev/null +++ b/pulp_deb/tests/functional/api/test_checkpoint.py @@ -0,0 +1,159 @@ +"""Tests for checkpoint distribution and publications.""" + +from datetime import datetime, timedelta +import re +from time import sleep +from urllib.parse import urlparse +import uuid +from aiohttp import ClientResponseError +import pytest +from pulp_deb.tests.functional.constants import DEB_PACKAGE_RELPATH +from pulp_deb.tests.functional.utils import get_local_package_absolute_path + + +@pytest.fixture(scope="class") +def setup( + deb_repository_factory, + deb_publication_factory, + deb_distribution_factory, + deb_package_factory, + apt_repository_api, +): + def create_publication(repo, checkpoint): + package_upload_params = { + "file": str(get_local_package_absolute_path(DEB_PACKAGE_RELPATH)), + "relative_path": DEB_PACKAGE_RELPATH, + "distribution": str(uuid.uuid4()), + "component": str(uuid.uuid4()), + "repository": repo.pulp_href, + } + deb_package_factory(**package_upload_params) + + repo = apt_repository_api.read(repo.pulp_href) + return deb_publication_factory(repo, checkpoint=checkpoint) + + repo = deb_repository_factory() + distribution = deb_distribution_factory(repository=repo, checkpoint=True) + + pubs = [] + pubs.append(create_publication(repo, False)) + sleep(1) + pubs.append(create_publication(repo, True)) + sleep(1) + pubs.append(create_publication(repo, False)) + sleep(1) + pubs.append(create_publication(repo, True)) + sleep(1) + pubs.append(create_publication(repo, False)) + + return pubs, distribution + + +@pytest.fixture +def checkpoint_url(distribution_base_url): + def _checkpoint_url(distribution, timestamp): + distro_base_url = distribution_base_url(distribution.base_url) + return f"{distro_base_url}{_format_checkpoint_timestamp(timestamp)}/" + + return _checkpoint_url + + +def _format_checkpoint_timestamp(timestamp): + return datetime.strftime(timestamp, "%Y%m%dT%H%M%SZ") + + +class TestCheckpointDistribution: + + def test_base_path_lists_checkpoints(self, setup, http_get, distribution_base_url): + pubs, distribution = setup + + response = http_get(distribution_base_url(distribution.base_url)).decode("utf-8") + + checkpoints_ts = set(re.findall(r"\d{8}T\d{6}Z", response)) + assert len(checkpoints_ts) == 2 + assert _format_checkpoint_timestamp(pubs[1].pulp_created) in checkpoints_ts + assert _format_checkpoint_timestamp(pubs[3].pulp_created) in checkpoints_ts + + def test_no_trailing_slash_is_redirected(self, setup, http_get, distribution_base_url): + """Test checkpoint listing when path doesn't end with a slash.""" + + pubs, distribution = setup + + response = http_get(distribution_base_url(distribution.base_url[:-1])).decode("utf-8") + checkpoints_ts = set(re.findall(r"\d{8}T\d{6}Z", response)) + + assert len(checkpoints_ts) == 2 + assert _format_checkpoint_timestamp(pubs[1].pulp_created) in checkpoints_ts + assert _format_checkpoint_timestamp(pubs[3].pulp_created) in checkpoints_ts + + def test_exact_timestamp_is_served(self, setup, http_get, checkpoint_url): + pubs, distribution = setup + + pub_1_url = checkpoint_url(distribution, pubs[1].pulp_created) + response = http_get(pub_1_url).decode("utf-8") + + assert f"

Index of {urlparse(pub_1_url).path}

" in response + + def test_invalid_timestamp_returns_404(self, setup, http_get, distribution_base_url): + _, distribution = setup + with pytest.raises(ClientResponseError) as exc: + http_get(distribution_base_url(f"{distribution.base_url}invalid_ts/")) + + assert exc.value.status == 404 + + with pytest.raises(ClientResponseError) as exc: + http_get(distribution_base_url(f"{distribution.base_url}20259928T092752Z/")) + + assert exc.value.status == 404 + + def test_non_checkpoint_timestamp_is_redirected(self, setup, http_get, checkpoint_url): + pubs, distribution = setup + # Using a non-checkpoint publication timestamp + pub_3_url = checkpoint_url(distribution, pubs[3].pulp_created) + pub_4_url = checkpoint_url(distribution, pubs[4].pulp_created) + + response = http_get(pub_4_url).decode("utf-8") + assert f"

Index of {urlparse(pub_3_url).path}

" in response + + # Test without a trailing slash + response = http_get(pub_4_url[:-1]).decode("utf-8") + assert f"

Index of {urlparse(pub_3_url).path}

" in response + + def test_arbitrary_timestamp_is_redirected(self, setup, http_get, checkpoint_url): + pubs, distribution = setup + pub_1_url = checkpoint_url(distribution, pubs[1].pulp_created) + arbitrary_url = checkpoint_url(distribution, pubs[1].pulp_created + timedelta(seconds=1)) + + response = http_get(arbitrary_url).decode("utf-8") + assert f"

Index of {urlparse(pub_1_url).path}

" in response + + # Test without a trailing slash + response = http_get(arbitrary_url[:-1]).decode("utf-8") + assert f"

Index of {urlparse(pub_1_url).path}

" in response + + def test_current_timestamp_serves_latest_checkpoint(self, setup, http_get, checkpoint_url): + pubs, distribution = setup + pub_3_url = checkpoint_url(distribution, pubs[3].pulp_created) + now_url = checkpoint_url(distribution, datetime.now()) + + response = http_get(now_url).decode("utf-8") + + assert f"

Index of {urlparse(pub_3_url).path}

" in response + + def test_before_first_timestamp_returns_404(self, setup, http_get, checkpoint_url): + pubs, distribution = setup + pub_0_url = checkpoint_url(distribution, pubs[0].pulp_created) + + with pytest.raises(ClientResponseError) as exc: + http_get(pub_0_url).decode("utf-8") + + assert exc.value.status == 404 + + def test_future_timestamp_returns_404(self, setup, http_get, checkpoint_url): + _, distribution = setup + url = checkpoint_url(distribution, datetime.now() + timedelta(days=1)) + + with pytest.raises(ClientResponseError) as exc: + http_get(url).decode("utf-8") + + assert exc.value.status == 404 diff --git a/pyproject.toml b/pyproject.toml index 0c89eeb13..b37792399 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ requires-python = ">=3.9" dependencies = [ # All things django and asyncio are deliberately left to pulpcore # Example transitive requirements: asgiref, asyncio, aiohttp - "pulpcore>=3.49.0,<3.85", + "pulpcore>=3.74.0,<3.85", "python-debian>=0.1.44,<0.2.0", "python-gnupg>=0.5,<0.6", "jsonschema>=4.6,<5.0",