Add checkpoint support

CHANGES/1250.feature

@@ -0,0 +1 @@
+Enabled the checkpoint feature in pulp_deb.

pulp_deb/app/serializers/publication_serializers.py

@@ -1,4 +1,5 @@
 from rest_framework.serializers import BooleanField, ValidationError
+from rest_framework import serializers
 from pulpcore.plugin.models import Publication
 from pulpcore.plugin.serializers import (
     RelatedField,
@@ -36,6 +37,7 @@ class AptPublicationSerializer(PublicationSerializer):
     )
     structured = BooleanField(help_text="Activate structured publishing mode.", default=True)
     publish_upstream_release_fields = BooleanField(help_text="", required=False)
+    checkpoint = serializers.BooleanField(required=False)
     signing_service = RelatedField(
         help_text="Sign Release files with this signing key",
         many=False,
@@ -57,6 +59,7 @@ class Meta:
         fields = PublicationSerializer.Meta.fields + (
             "simple",
             "structured",
+            "checkpoint",
             "signing_service",
             "publish_upstream_release_fields",
         )
@@ -75,7 +78,8 @@ class AptDistributionSerializer(DistributionSerializer):
         queryset=Publication.objects.exclude(complete=False),
         allow_null=True,
     )
+    checkpoint = serializers.BooleanField(required=False)
 
     class Meta:
-        fields = DistributionSerializer.Meta.fields + ("publication",)
+        fields = DistributionSerializer.Meta.fields + ("publication", "checkpoint")
         model = AptDistribution

pulp_deb/app/tasks/publishing.py

@@ -82,6 +82,7 @@ def publish(
     repository_version_pk,
     simple,
     structured,
+    checkpoint=False,
     signing_service_pk=None,
     publish_upstream_release_fields=None,
 ):
@@ -92,6 +93,7 @@ def publish(
         repository_version_pk (str): Create a publication from this repository version.
         simple (bool): Create a simple publication with all packages contained in default/all.
         structured (bool): Create a structured publication with releases and components.
+        checkpoint (bool): Whether to create a checkpoint publication.
         signing_service_pk (str): Use this SigningService to sign the Release files.
 
     """
@@ -115,7 +117,9 @@ def publish(
         )
     )
     with tempfile.TemporaryDirectory(".") as temp_dir:
-        with AptPublication.create(repo_version, pass_through=False) as publication:
+        with AptPublication.create(
+            repo_version, pass_through=False, checkpoint=checkpoint
+        ) as publication:
             publication.simple = simple
             publication.structured = structured
             publication.signing_service = signing_service

pulp_deb/app/viewsets/publication.py

@@ -211,21 +211,25 @@ def create(self, request):
         repository_version = serializer.validated_data.get("repository_version")
         simple = serializer.validated_data.get("simple")
         structured = serializer.validated_data.get("structured")
+        checkpoint = serializer.validated_data.get("checkpoint")
         signing_service = serializer.validated_data.get("signing_service")
         publish_upstream_release_fields = serializer.validated_data.get(
             "publish_upstream_release_fields"
         )
 
+        kwargs = {
+            "repository_version_pk": repository_version.pk,
+            "simple": simple,
+            "structured": structured,
+            "signing_service_pk": getattr(signing_service, "pk", None),
+            "publish_upstream_release_fields": publish_upstream_release_fields,
+        }
+        if checkpoint:
+            kwargs["checkpoint"] = True
         result = dispatch(
             func=tasks.publish,
             shared_resources=[repository_version.repository],
-            kwargs={
-                "repository_version_pk": repository_version.pk,
-                "simple": simple,
-                "structured": structured,
-                "signing_service_pk": getattr(signing_service, "pk", None),
-                "publish_upstream_release_fields": publish_upstream_release_fields,
-            },
+            kwargs=kwargs,
         )
         return OperationPostponedResponse(result, request)
 

pulp_deb/tests/conftest.py

@@ -59,7 +59,7 @@ def apt_repository_versions_api(apt_client):
 def deb_distribution_factory(apt_distribution_api, gen_object_with_cleanup):
     """Fixture that generates a deb distribution with cleanup from a given publication."""
 
-    def _deb_distribution_factory(publication=None, repository=None):
+    def _deb_distribution_factory(publication=None, repository=None, checkpoint=None):
         """Create a deb distribution.
 
         :param publication: The publication the distribution is based on.
@@ -70,6 +70,8 @@ def _deb_distribution_factory(publication=None, repository=None):
             body["publication"] = publication.pulp_href
         if repository:
             body["repository"] = repository.pulp_href
+        if checkpoint is not None:
+            body["checkpoint"] = checkpoint
         return gen_object_with_cleanup(apt_distribution_api, body)
 
     return _deb_distribution_factory

pulp_deb/tests/functional/api/test_checkpoint.py

@@ -0,0 +1,159 @@
+"""Tests for checkpoint distribution and publications."""
+
+from datetime import datetime, timedelta
+import re
+from time import sleep
+from urllib.parse import urlparse
+import uuid
+from aiohttp import ClientResponseError
+import pytest
+from pulp_deb.tests.functional.constants import DEB_PACKAGE_RELPATH
+from pulp_deb.tests.functional.utils import get_local_package_absolute_path
+
+
+@pytest.fixture(scope="class")
+def setup(
+    deb_repository_factory,
+    deb_publication_factory,
+    deb_distribution_factory,
+    deb_package_factory,
+    apt_repository_api,
+):
+    def create_publication(repo, checkpoint):
+        package_upload_params = {
+            "file": str(get_local_package_absolute_path(DEB_PACKAGE_RELPATH)),
+            "relative_path": DEB_PACKAGE_RELPATH,
+            "distribution": str(uuid.uuid4()),
+            "component": str(uuid.uuid4()),
+            "repository": repo.pulp_href,
+        }
+        deb_package_factory(**package_upload_params)
+
+        repo = apt_repository_api.read(repo.pulp_href)
+        return deb_publication_factory(repo, checkpoint=checkpoint)
+
+    repo = deb_repository_factory()
+    distribution = deb_distribution_factory(repository=repo, checkpoint=True)
+
+    pubs = []
+    pubs.append(create_publication(repo, False))
+    sleep(1)
+    pubs.append(create_publication(repo, True))
+    sleep(1)
+    pubs.append(create_publication(repo, False))
+    sleep(1)
+    pubs.append(create_publication(repo, True))
+    sleep(1)
+    pubs.append(create_publication(repo, False))
+
+    return pubs, distribution
+
+
+@pytest.fixture
+def checkpoint_url(distribution_base_url):
+    def _checkpoint_url(distribution, timestamp):
+        distro_base_url = distribution_base_url(distribution.base_url)
+        return f"{distro_base_url}{_format_checkpoint_timestamp(timestamp)}/"
+
+    return _checkpoint_url
+
+
+def _format_checkpoint_timestamp(timestamp):
+    return datetime.strftime(timestamp, "%Y%m%dT%H%M%SZ")
+
+
+class TestCheckpointDistribution:
+
+    def test_base_path_lists_checkpoints(self, setup, http_get, distribution_base_url):
+        pubs, distribution = setup
+
+        response = http_get(distribution_base_url(distribution.base_url)).decode("utf-8")
+
+        checkpoints_ts = set(re.findall(r"\d{8}T\d{6}Z", response))
+        assert len(checkpoints_ts) == 2
+        assert _format_checkpoint_timestamp(pubs[1].pulp_created) in checkpoints_ts
+        assert _format_checkpoint_timestamp(pubs[3].pulp_created) in checkpoints_ts
+
+    def test_no_trailing_slash_is_redirected(self, setup, http_get, distribution_base_url):
+        """Test checkpoint listing when path doesn't end with a slash."""
+
+        pubs, distribution = setup
+
+        response = http_get(distribution_base_url(distribution.base_url[:-1])).decode("utf-8")
+        checkpoints_ts = set(re.findall(r"\d{8}T\d{6}Z", response))
+
+        assert len(checkpoints_ts) == 2
+        assert _format_checkpoint_timestamp(pubs[1].pulp_created) in checkpoints_ts
+        assert _format_checkpoint_timestamp(pubs[3].pulp_created) in checkpoints_ts
+
+    def test_exact_timestamp_is_served(self, setup, http_get, checkpoint_url):
+        pubs, distribution = setup
+
+        pub_1_url = checkpoint_url(distribution, pubs[1].pulp_created)
+        response = http_get(pub_1_url).decode("utf-8")
+
+        assert f"<h1>Index of {urlparse(pub_1_url).path}</h1>" in response
+
+    def test_invalid_timestamp_returns_404(self, setup, http_get, distribution_base_url):
+        _, distribution = setup
+        with pytest.raises(ClientResponseError) as exc:
+            http_get(distribution_base_url(f"{distribution.base_url}invalid_ts/"))
+
+        assert exc.value.status == 404
+
+        with pytest.raises(ClientResponseError) as exc:
+            http_get(distribution_base_url(f"{distribution.base_url}20259928T092752Z/"))
+
+        assert exc.value.status == 404
+
+    def test_non_checkpoint_timestamp_is_redirected(self, setup, http_get, checkpoint_url):
+        pubs, distribution = setup
+        # Using a non-checkpoint publication timestamp
+        pub_3_url = checkpoint_url(distribution, pubs[3].pulp_created)
+        pub_4_url = checkpoint_url(distribution, pubs[4].pulp_created)
+
+        response = http_get(pub_4_url).decode("utf-8")
+        assert f"<h1>Index of {urlparse(pub_3_url).path}</h1>" in response
+
+        # Test without a trailing slash
+        response = http_get(pub_4_url[:-1]).decode("utf-8")
+        assert f"<h1>Index of {urlparse(pub_3_url).path}</h1>" in response
+
+    def test_arbitrary_timestamp_is_redirected(self, setup, http_get, checkpoint_url):
+        pubs, distribution = setup
+        pub_1_url = checkpoint_url(distribution, pubs[1].pulp_created)
+        arbitrary_url = checkpoint_url(distribution, pubs[1].pulp_created + timedelta(seconds=1))
+
+        response = http_get(arbitrary_url).decode("utf-8")
+        assert f"<h1>Index of {urlparse(pub_1_url).path}</h1>" in response
+
+        # Test without a trailing slash
+        response = http_get(arbitrary_url[:-1]).decode("utf-8")
+        assert f"<h1>Index of {urlparse(pub_1_url).path}</h1>" in response
+
+    def test_current_timestamp_serves_latest_checkpoint(self, setup, http_get, checkpoint_url):
+        pubs, distribution = setup
+        pub_3_url = checkpoint_url(distribution, pubs[3].pulp_created)
+        now_url = checkpoint_url(distribution, datetime.now())
+
+        response = http_get(now_url).decode("utf-8")
+
+        assert f"<h1>Index of {urlparse(pub_3_url).path}</h1>" in response
+
+    def test_before_first_timestamp_returns_404(self, setup, http_get, checkpoint_url):
+        pubs, distribution = setup
+        pub_0_url = checkpoint_url(distribution, pubs[0].pulp_created)
+
+        with pytest.raises(ClientResponseError) as exc:
+            http_get(pub_0_url).decode("utf-8")
+
+        assert exc.value.status == 404
+
+    def test_future_timestamp_returns_404(self, setup, http_get, checkpoint_url):
+        _, distribution = setup
+        url = checkpoint_url(distribution, datetime.now() + timedelta(days=1))
+
+        with pytest.raises(ClientResponseError) as exc:
+            http_get(url).decode("utf-8")
+
+        assert exc.value.status == 404

pyproject.toml

@@ -29,7 +29,7 @@ requires-python = ">=3.9"
 dependencies = [
   # All things django and asyncio are deliberately left to pulpcore
   # Example transitive requirements: asgiref, asyncio, aiohttp
-  "pulpcore>=3.49.0,<3.85",
+  "pulpcore>=3.74.0,<3.85",
   "python-debian>=0.1.44,<0.2.0",
   "python-gnupg>=0.5,<0.6",
   "jsonschema>=4.6,<5.0",