diff --git a/CHANGES/1083.bugfix b/CHANGES/1083.bugfix new file mode 100644 index 00000000..ac48c67b --- /dev/null +++ b/CHANGES/1083.bugfix @@ -0,0 +1 @@ +Fixed pull-through PEP 658 metadata not being served correctly for certain tools. diff --git a/CHANGES/1087.bugfix b/CHANGES/1087.bugfix new file mode 100644 index 00000000..15fac616 --- /dev/null +++ b/CHANGES/1087.bugfix @@ -0,0 +1 @@ +Fixed pull-through PEP 658 metadata not being saved correctly with the package. diff --git a/docs/user/guides/publish.md b/docs/user/guides/publish.md index aa0bae8e..1f1eae4d 100644 --- a/docs/user/guides/publish.md +++ b/docs/user/guides/publish.md @@ -105,6 +105,10 @@ pulp python distribution update --name foo --remote bar Functionality may not work or may be incomplete. Also, backwards compatibility when upgrading is not guaranteed. +!!! warning + Chaining pull-through indices, having a pull-through point to another pull-through, does not + work. + ## Use the newly created distribution The metadata and packages can now be retrieved from the distribution: diff --git a/pulp_python/app/models.py b/pulp_python/app/models.py index 3ea252e3..b7a5e548 100644 --- a/pulp_python/app/models.py +++ b/pulp_python/app/models.py @@ -25,6 +25,7 @@ from .provenance import Provenance from .utils import ( artifact_to_python_content_data, + artifact_to_metadata_artifact, canonicalize_name, python_content_to_json, PYPI_LAST_SERIAL, @@ -215,7 +216,10 @@ def init_from_artifact_and_relative_path(artifact, relative_path): """Used when downloading package from pull-through cache.""" path = PurePath(relative_path) data = artifact_to_python_content_data(path.name, artifact, domain=get_domain()) - return PythonPackageContent(**data) + artifacts = {path.name: artifact} + if metadata_artifact := artifact_to_metadata_artifact(path.name, artifact): + artifacts[f"{path.name}.metadata"] = metadata_artifact + return PythonPackageContent(**data), artifacts def __str__(self): """ @@ -320,11 +324,25 @@ def get_remote_artifact_url(self, relative_path=None, request=None): """Get url for remote_artifact""" if request and (url := request.query.get("redirect")): # This is a special case for pull-through caching + # To handle PEP 658, it states that if the package has metadata available then it + # should be found at the download URL + ".metadata". Thus if the request url ends with + # ".metadata" then we need to add ".metadata" to the redirect url if not present. + if relative_path: + if relative_path.endswith(".metadata") and not url.endswith(".metadata"): + url += ".metadata" + # Handle special case for bug in pip (TODO file issue in pip) where it appends + # ".metadata" to the redirect url instead of the request url + if url.endswith(".metadata") and not relative_path.endswith(".metadata"): + setattr(self, "_real_relative_path", url.rsplit("/", 1)[1]) return url return super().get_remote_artifact_url(relative_path, request=request) def get_remote_artifact_content_type(self, relative_path=None): - """Return PythonPackageContent.""" + """Return PythonPackageContent, except for metadata artifacts.""" + if hasattr(self, "_real_relative_path"): + relative_path = getattr(self, "_real_relative_path") + if relative_path and relative_path.endswith(".whl.metadata"): + return None return PythonPackageContent class Meta: diff --git a/pulp_python/tests/functional/api/test_full_mirror.py b/pulp_python/tests/functional/api/test_full_mirror.py index f9850ba4..e37f36c2 100644 --- a/pulp_python/tests/functional/api/test_full_mirror.py +++ b/pulp_python/tests/functional/api/test_full_mirror.py @@ -11,8 +11,9 @@ from pypi_simple import ProjectPage from packaging.version import parse -from urllib.parse import urljoin, urlsplit +from urllib.parse import urljoin, urlsplit, urlunsplit from random import sample +from hashlib import sha256 def test_pull_through_install( @@ -182,3 +183,85 @@ def test_pull_through_local_only( r = requests.get(url) assert r.status_code == 404 assert r.text == "pulp-python does not exist." + + +@pytest.mark.parallel +def test_pull_through_metadata(python_remote_factory, python_distribution_factory): + """ + Tests that metadata is correctly served when using pull-through. + + So when requesting the metadata url according to PEP 658 you should just need to add .metadata + to the end of the url path. Since pull-through includes a redirect query parameter we need to + test adding .metadata to the end of the url path vs adding it to the end of redirect query. + """ + remote = python_remote_factory(includes=["pytz"]) + distro = python_distribution_factory(remote=remote.pulp_href) + + url = f"{distro.base_url}simple/pytz/" + project_page = ProjectPage.from_response(requests.get(url), "pytz") + filename1 = "pytz-2023.2-py2.py3-none-any.whl" + filename2 = "pytz-2023.3-py2.py3-none-any.whl" + package1 = next(p for p in project_page.packages if p.filename == filename1) + package2 = next(p for p in project_page.packages if p.filename == filename2) + assert package1.has_metadata + assert package2.has_metadata + + # The correct way to get the metadata url: add to path (uv does this) + parts1 = urlsplit(package1.url) + url1 = urlunsplit((parts1[0], parts1[1], parts1[2] + ".metadata", parts1[3], parts1[4])) + r = requests.get(url1) + assert r.status_code == 200 + assert sha256(r.content).hexdigest() == package1.metadata_digests["sha256"] + + # The incorrect way to get the metadata url: add to end of string (pip does this) + url2 = package2.url + ".metadata" + r = requests.get(url2) + assert r.status_code == 200 + assert sha256(r.content).hexdigest() == package2.metadata_digests["sha256"] + + +@pytest.mark.parallel +def test_pull_through_metadata_with_repo( + python_repo_factory, + python_remote_factory, + python_distribution_factory, + pulpcore_bindings, +): + """Tests that metadata is correctly saved when using pull-through with a repository.""" + remote = python_remote_factory(url=PYPI_URL, includes=["pip"]) + repo = python_repo_factory() + distro = python_distribution_factory(repository=repo.pulp_href, remote=remote.pulp_href) + + pip_url = f"{distro.base_url}simple/pip/" + project_page = ProjectPage.from_response(requests.get(pip_url), "pip") + filename = "pip-26.0.1-py3-none-any.whl" + package = next(p for p in project_page.packages if p.filename == filename) + assert package.has_metadata + assert "?redirect=" in package.url + + # Retrieve the metadata and assert the content was not saved to the repository + parts = urlsplit(package.url) + url = urlunsplit((parts[0], parts[1], parts[2] + ".metadata", parts[3], parts[4])) + r = requests.get(url) + assert r.status_code == 200 + assert sha256(r.content).hexdigest() == package.metadata_digests["sha256"] + project_page = ProjectPage.from_response(requests.get(pip_url), "pip") + package = next(p for p in project_page.packages if p.filename == filename) + assert package.has_metadata + assert "?redirect=" in package.url + + # Now retrieve the package and assert the content was saved with metadata + r = requests.get(package.url) + assert r.status_code == 200 + pa = pulpcore_bindings.ArtifactsApi.list(sha256=package.digests["sha256"]) + assert pa.count == 1 + ma = pulpcore_bindings.ArtifactsApi.list(sha256=package.metadata_digests["sha256"]) + assert ma.count == 1 + + # Check the simple page is updated to point to the local repository + project_page = ProjectPage.from_response(requests.get(pip_url), "pip") + package = next(p for p in project_page.packages if p.filename == filename) + assert "?redirect=" not in package.url + r = requests.get(package.metadata_url) + assert r.status_code == 200 + assert sha256(r.content).hexdigest() == package.metadata_digests["sha256"]