From 608e825a866f4f1f53eccc457850b098a0320fc6 Mon Sep 17 00:00:00 2001
From: Moustafa Moustafa <28809043+Moustafa-Moustafa@users.noreply.github.com>
Date: Tue, 28 Jan 2025 13:29:32 -0600
Subject: [PATCH] Add checkpoint support
Added the checkpoint field to AptPublicationSerializer and AptDistributionSerializer.
Enabled creating checkpoint publications.
closes #1250
---
CHANGES/1250.feature | 1 +
.../serializers/publication_serializers.py | 6 +-
pulp_deb/app/tasks/publishing.py | 6 +-
pulp_deb/app/viewsets/publication.py | 18 +-
pulp_deb/tests/conftest.py | 4 +-
.../tests/functional/api/test_checkpoint.py | 159 ++++++++++++++++++
pyproject.toml | 2 +-
7 files changed, 185 insertions(+), 11 deletions(-)
create mode 100644 CHANGES/1250.feature
create mode 100644 pulp_deb/tests/functional/api/test_checkpoint.py
diff --git a/CHANGES/1250.feature b/CHANGES/1250.feature
new file mode 100644
index 000000000..85737a745
--- /dev/null
+++ b/CHANGES/1250.feature
@@ -0,0 +1 @@
+Enabled the checkpoint feature in pulp_deb.
\ No newline at end of file
diff --git a/pulp_deb/app/serializers/publication_serializers.py b/pulp_deb/app/serializers/publication_serializers.py
index 74e9059cb..a7d8c2cf5 100644
--- a/pulp_deb/app/serializers/publication_serializers.py
+++ b/pulp_deb/app/serializers/publication_serializers.py
@@ -1,4 +1,5 @@
from rest_framework.serializers import BooleanField, ValidationError
+from rest_framework import serializers
from pulpcore.plugin.models import Publication
from pulpcore.plugin.serializers import (
RelatedField,
@@ -36,6 +37,7 @@ class AptPublicationSerializer(PublicationSerializer):
)
structured = BooleanField(help_text="Activate structured publishing mode.", default=True)
publish_upstream_release_fields = BooleanField(help_text="", required=False)
+ checkpoint = serializers.BooleanField(required=False)
signing_service = RelatedField(
help_text="Sign Release files with this signing key",
many=False,
@@ -57,6 +59,7 @@ class Meta:
fields = PublicationSerializer.Meta.fields + (
"simple",
"structured",
+ "checkpoint",
"signing_service",
"publish_upstream_release_fields",
)
@@ -75,7 +78,8 @@ class AptDistributionSerializer(DistributionSerializer):
queryset=Publication.objects.exclude(complete=False),
allow_null=True,
)
+ checkpoint = serializers.BooleanField(required=False)
class Meta:
- fields = DistributionSerializer.Meta.fields + ("publication",)
+ fields = DistributionSerializer.Meta.fields + ("publication", "checkpoint")
model = AptDistribution
diff --git a/pulp_deb/app/tasks/publishing.py b/pulp_deb/app/tasks/publishing.py
index f4b612669..489526ae3 100644
--- a/pulp_deb/app/tasks/publishing.py
+++ b/pulp_deb/app/tasks/publishing.py
@@ -82,6 +82,7 @@ def publish(
repository_version_pk,
simple,
structured,
+ checkpoint=False,
signing_service_pk=None,
publish_upstream_release_fields=None,
):
@@ -92,6 +93,7 @@ def publish(
repository_version_pk (str): Create a publication from this repository version.
simple (bool): Create a simple publication with all packages contained in default/all.
structured (bool): Create a structured publication with releases and components.
+ checkpoint (bool): Whether to create a checkpoint publication.
signing_service_pk (str): Use this SigningService to sign the Release files.
"""
@@ -115,7 +117,9 @@ def publish(
)
)
with tempfile.TemporaryDirectory(".") as temp_dir:
- with AptPublication.create(repo_version, pass_through=False) as publication:
+ with AptPublication.create(
+ repo_version, pass_through=False, checkpoint=checkpoint
+ ) as publication:
publication.simple = simple
publication.structured = structured
publication.signing_service = signing_service
diff --git a/pulp_deb/app/viewsets/publication.py b/pulp_deb/app/viewsets/publication.py
index f0d5dc567..3c0624132 100644
--- a/pulp_deb/app/viewsets/publication.py
+++ b/pulp_deb/app/viewsets/publication.py
@@ -211,21 +211,25 @@ def create(self, request):
repository_version = serializer.validated_data.get("repository_version")
simple = serializer.validated_data.get("simple")
structured = serializer.validated_data.get("structured")
+ checkpoint = serializer.validated_data.get("checkpoint")
signing_service = serializer.validated_data.get("signing_service")
publish_upstream_release_fields = serializer.validated_data.get(
"publish_upstream_release_fields"
)
+ kwargs = {
+ "repository_version_pk": repository_version.pk,
+ "simple": simple,
+ "structured": structured,
+ "signing_service_pk": getattr(signing_service, "pk", None),
+ "publish_upstream_release_fields": publish_upstream_release_fields,
+ }
+ if checkpoint:
+ kwargs["checkpoint"] = True
result = dispatch(
func=tasks.publish,
shared_resources=[repository_version.repository],
- kwargs={
- "repository_version_pk": repository_version.pk,
- "simple": simple,
- "structured": structured,
- "signing_service_pk": getattr(signing_service, "pk", None),
- "publish_upstream_release_fields": publish_upstream_release_fields,
- },
+ kwargs=kwargs,
)
return OperationPostponedResponse(result, request)
diff --git a/pulp_deb/tests/conftest.py b/pulp_deb/tests/conftest.py
index 2429606eb..9b3460ea5 100644
--- a/pulp_deb/tests/conftest.py
+++ b/pulp_deb/tests/conftest.py
@@ -59,7 +59,7 @@ def apt_repository_versions_api(apt_client):
def deb_distribution_factory(apt_distribution_api, gen_object_with_cleanup):
"""Fixture that generates a deb distribution with cleanup from a given publication."""
- def _deb_distribution_factory(publication=None, repository=None):
+ def _deb_distribution_factory(publication=None, repository=None, checkpoint=None):
"""Create a deb distribution.
:param publication: The publication the distribution is based on.
@@ -70,6 +70,8 @@ def _deb_distribution_factory(publication=None, repository=None):
body["publication"] = publication.pulp_href
if repository:
body["repository"] = repository.pulp_href
+ if checkpoint is not None:
+ body["checkpoint"] = checkpoint
return gen_object_with_cleanup(apt_distribution_api, body)
return _deb_distribution_factory
diff --git a/pulp_deb/tests/functional/api/test_checkpoint.py b/pulp_deb/tests/functional/api/test_checkpoint.py
new file mode 100644
index 000000000..067b5fadc
--- /dev/null
+++ b/pulp_deb/tests/functional/api/test_checkpoint.py
@@ -0,0 +1,159 @@
+"""Tests for checkpoint distribution and publications."""
+
+from datetime import datetime, timedelta
+import re
+from time import sleep
+from urllib.parse import urlparse
+import uuid
+from aiohttp import ClientResponseError
+import pytest
+from pulp_deb.tests.functional.constants import DEB_PACKAGE_RELPATH
+from pulp_deb.tests.functional.utils import get_local_package_absolute_path
+
+
+@pytest.fixture(scope="class")
+def setup(
+ deb_repository_factory,
+ deb_publication_factory,
+ deb_distribution_factory,
+ deb_package_factory,
+ apt_repository_api,
+):
+ def create_publication(repo, checkpoint):
+ package_upload_params = {
+ "file": str(get_local_package_absolute_path(DEB_PACKAGE_RELPATH)),
+ "relative_path": DEB_PACKAGE_RELPATH,
+ "distribution": str(uuid.uuid4()),
+ "component": str(uuid.uuid4()),
+ "repository": repo.pulp_href,
+ }
+ deb_package_factory(**package_upload_params)
+
+ repo = apt_repository_api.read(repo.pulp_href)
+ return deb_publication_factory(repo, checkpoint=checkpoint)
+
+ repo = deb_repository_factory()
+ distribution = deb_distribution_factory(repository=repo, checkpoint=True)
+
+ pubs = []
+ pubs.append(create_publication(repo, False))
+ sleep(1)
+ pubs.append(create_publication(repo, True))
+ sleep(1)
+ pubs.append(create_publication(repo, False))
+ sleep(1)
+ pubs.append(create_publication(repo, True))
+ sleep(1)
+ pubs.append(create_publication(repo, False))
+
+ return pubs, distribution
+
+
+@pytest.fixture
+def checkpoint_url(distribution_base_url):
+ def _checkpoint_url(distribution, timestamp):
+ distro_base_url = distribution_base_url(distribution.base_url)
+ return f"{distro_base_url}{_format_checkpoint_timestamp(timestamp)}/"
+
+ return _checkpoint_url
+
+
+def _format_checkpoint_timestamp(timestamp):
+ return datetime.strftime(timestamp, "%Y%m%dT%H%M%SZ")
+
+
+class TestCheckpointDistribution:
+
+ def test_base_path_lists_checkpoints(self, setup, http_get, distribution_base_url):
+ pubs, distribution = setup
+
+ response = http_get(distribution_base_url(distribution.base_url)).decode("utf-8")
+
+ checkpoints_ts = set(re.findall(r"\d{8}T\d{6}Z", response))
+ assert len(checkpoints_ts) == 2
+ assert _format_checkpoint_timestamp(pubs[1].pulp_created) in checkpoints_ts
+ assert _format_checkpoint_timestamp(pubs[3].pulp_created) in checkpoints_ts
+
+ def test_no_trailing_slash_is_redirected(self, setup, http_get, distribution_base_url):
+ """Test checkpoint listing when path doesn't end with a slash."""
+
+ pubs, distribution = setup
+
+ response = http_get(distribution_base_url(distribution.base_url[:-1])).decode("utf-8")
+ checkpoints_ts = set(re.findall(r"\d{8}T\d{6}Z", response))
+
+ assert len(checkpoints_ts) == 2
+ assert _format_checkpoint_timestamp(pubs[1].pulp_created) in checkpoints_ts
+ assert _format_checkpoint_timestamp(pubs[3].pulp_created) in checkpoints_ts
+
+ def test_exact_timestamp_is_served(self, setup, http_get, checkpoint_url):
+ pubs, distribution = setup
+
+ pub_1_url = checkpoint_url(distribution, pubs[1].pulp_created)
+ response = http_get(pub_1_url).decode("utf-8")
+
+ assert f"Index of {urlparse(pub_1_url).path}
" in response
+
+ def test_invalid_timestamp_returns_404(self, setup, http_get, distribution_base_url):
+ _, distribution = setup
+ with pytest.raises(ClientResponseError) as exc:
+ http_get(distribution_base_url(f"{distribution.base_url}invalid_ts/"))
+
+ assert exc.value.status == 404
+
+ with pytest.raises(ClientResponseError) as exc:
+ http_get(distribution_base_url(f"{distribution.base_url}20259928T092752Z/"))
+
+ assert exc.value.status == 404
+
+ def test_non_checkpoint_timestamp_is_redirected(self, setup, http_get, checkpoint_url):
+ pubs, distribution = setup
+ # Using a non-checkpoint publication timestamp
+ pub_3_url = checkpoint_url(distribution, pubs[3].pulp_created)
+ pub_4_url = checkpoint_url(distribution, pubs[4].pulp_created)
+
+ response = http_get(pub_4_url).decode("utf-8")
+ assert f"Index of {urlparse(pub_3_url).path}
" in response
+
+ # Test without a trailing slash
+ response = http_get(pub_4_url[:-1]).decode("utf-8")
+ assert f"Index of {urlparse(pub_3_url).path}
" in response
+
+ def test_arbitrary_timestamp_is_redirected(self, setup, http_get, checkpoint_url):
+ pubs, distribution = setup
+ pub_1_url = checkpoint_url(distribution, pubs[1].pulp_created)
+ arbitrary_url = checkpoint_url(distribution, pubs[1].pulp_created + timedelta(seconds=1))
+
+ response = http_get(arbitrary_url).decode("utf-8")
+ assert f"Index of {urlparse(pub_1_url).path}
" in response
+
+ # Test without a trailing slash
+ response = http_get(arbitrary_url[:-1]).decode("utf-8")
+ assert f"Index of {urlparse(pub_1_url).path}
" in response
+
+ def test_current_timestamp_serves_latest_checkpoint(self, setup, http_get, checkpoint_url):
+ pubs, distribution = setup
+ pub_3_url = checkpoint_url(distribution, pubs[3].pulp_created)
+ now_url = checkpoint_url(distribution, datetime.now())
+
+ response = http_get(now_url).decode("utf-8")
+
+ assert f"Index of {urlparse(pub_3_url).path}
" in response
+
+ def test_before_first_timestamp_returns_404(self, setup, http_get, checkpoint_url):
+ pubs, distribution = setup
+ pub_0_url = checkpoint_url(distribution, pubs[0].pulp_created)
+
+ with pytest.raises(ClientResponseError) as exc:
+ http_get(pub_0_url).decode("utf-8")
+
+ assert exc.value.status == 404
+
+ def test_future_timestamp_returns_404(self, setup, http_get, checkpoint_url):
+ _, distribution = setup
+ url = checkpoint_url(distribution, datetime.now() + timedelta(days=1))
+
+ with pytest.raises(ClientResponseError) as exc:
+ http_get(url).decode("utf-8")
+
+ assert exc.value.status == 404
diff --git a/pyproject.toml b/pyproject.toml
index 0c89eeb13..b37792399 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -29,7 +29,7 @@ requires-python = ">=3.9"
dependencies = [
# All things django and asyncio are deliberately left to pulpcore
# Example transitive requirements: asgiref, asyncio, aiohttp
- "pulpcore>=3.49.0,<3.85",
+ "pulpcore>=3.74.0,<3.85",
"python-debian>=0.1.44,<0.2.0",
"python-gnupg>=0.5,<0.6",
"jsonschema>=4.6,<5.0",