From 744f7bcd08dfd94b651af6a8eb1f5823811ce890 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Fri, 21 Aug 2026 14:16:28 -0400 Subject: [PATCH] Bump bandersnatch to 8.0 and bump required python_version to >=3.12 Generated-by: cursor-grok-4.6 --- CHANGES/+bandersnatch-8.feature | 1 + CHANGES/+python-3.12.feature | 1 + pulp_python/app/tasks/sync.py | 56 +++++++++---------- .../functional/api/test_download_content.py | 2 +- pyproject.toml | 5 +- 5 files changed, 30 insertions(+), 35 deletions(-) create mode 100644 CHANGES/+bandersnatch-8.feature create mode 100644 CHANGES/+python-3.12.feature diff --git a/CHANGES/+bandersnatch-8.feature b/CHANGES/+bandersnatch-8.feature new file mode 100644 index 00000000..5c048429 --- /dev/null +++ b/CHANGES/+bandersnatch-8.feature @@ -0,0 +1 @@ +Upgraded Bandersnatch to 8.0. Full-index syncs now list packages via the PEP 691 Simple JSON API, falling back to HTML `/simple/` when JSON is unavailable. diff --git a/CHANGES/+python-3.12.feature b/CHANGES/+python-3.12.feature new file mode 100644 index 00000000..e7a70b19 --- /dev/null +++ b/CHANGES/+python-3.12.feature @@ -0,0 +1 @@ +Updated minimum required python version to >=3.12. diff --git a/pulp_python/app/tasks/sync.py b/pulp_python/app/tasks/sync.py index dd5e3826..d3865b9e 100644 --- a/pulp_python/app/tasks/sync.py +++ b/pulp_python/app/tasks/sync.py @@ -3,11 +3,10 @@ from functools import partial from urllib.parse import urljoin, urlparse -from aiohttp import ClientError, ClientResponseError +from aiohttp import ClientError from bandersnatch.configuration import BandersnatchConfig from bandersnatch.master import Master from bandersnatch.mirror import Mirror -from lxml.etree import LxmlError from packaging.requirements import Requirement from pypi_simple import IndexPage @@ -169,41 +168,36 @@ def __init__(self, serial, master, workers, deferred_download, python_stage, pro async def determine_packages_to_sync(self): """ - Calling this means that includes wasn't specified, - so try to get all of the packages from Mirror (hopefully PyPi) + Called when includes wasn't specified. List all projects from the remote + via the PEP 691 Simple JSON API, falling back to HTML /simple/. """ - number_xmlrpc_attempts = 3 - for attempt in range(number_xmlrpc_attempts): - logger.info("Attempt {} to get package list from {}".format(attempt, self.master.url)) - try: - if not self.synced_serial: - logger.info("Syncing all packages.") - # First get the current serial, then start to sync. - all_packages = await self.master.all_packages() - self.packages_to_sync.update(all_packages) - self.target_serial = max( - [self.synced_serial] + [int(v) for v in self.packages_to_sync.values()] - ) - else: - logger.info("Syncing based on changelog.") - changed_packages = await self.master.changed_packages(self.synced_serial) - self.packages_to_sync.update(changed_packages) - self.target_serial = max( - [self.synced_serial] + [int(v) for v in self.packages_to_sync.values()] - ) - break - except (ClientError, ClientResponseError, LxmlError): - # Retry if XMLRPC endpoint failed, server might not support it. - continue - else: - logger.info("Failed to get package list using XMLRPC, trying parse simple page.") + logger.info("Syncing all packages from %s", self.master.url) + try: + simple_index = await self.master.fetch_simple_index() + if not isinstance(simple_index, dict) or "projects" not in simple_index: + raise ValueError("Simple JSON index is missing a projects list") + for project in simple_index["projects"]: + name = project.get("name") + if name is None: + continue + # _last-serial is a PyPI extension; default to 0 when absent + self.packages_to_sync[name] = project.get("_last-serial", 0) + self.target_serial = max( + [self.synced_serial or 0] + [int(v) for v in self.packages_to_sync.values()] + ) + except (ClientError, ValueError, TypeError, AttributeError) as exc: + logger.info( + "Failed to list packages via Simple JSON API (%s); " + "falling back to HTML simple index.", + exc, + ) url = urljoin(self.remote.url, "simple/") downloader = self.remote.get_downloader(url=url) result = await downloader.run() with open(result.path) as f: index = IndexPage.from_html(f.read()) - self.packages_to_sync.update({p: 0 for p in index.projects}) - self.target_serial = result.headers.get(PYPI_LAST_SERIAL, 0) + self.packages_to_sync.update({p: 0 for p in index.projects}) + self.target_serial = result.headers.get(PYPI_LAST_SERIAL, 0) self._filter_packages() if self.target_serial: diff --git a/pulp_python/tests/functional/api/test_download_content.py b/pulp_python/tests/functional/api/test_download_content.py index c9820f11..3fadea48 100644 --- a/pulp_python/tests/functional/api/test_download_content.py +++ b/pulp_python/tests/functional/api/test_download_content.py @@ -48,7 +48,7 @@ def test_full_fixtures_to_pulp_sync( ): """ This test checks that Pulp can fully sync another Python Package repository that is not - PyPI. This reads the repository's simple page if XMLRPC isn't supported. + PyPI. This lists projects via the Simple JSON API, falling back to HTML /simple/. """ # Repository we are syncing from is the fixtures (default url) remote = python_remote_factory(includes=[], prereleases=True) diff --git a/pyproject.toml b/pyproject.toml index 53af0178..9ee46c7c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,15 +20,14 @@ classifiers=[ "Framework :: Django", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", ] -requires-python = ">=3.11" +requires-python = ">=3.12" dependencies = [ "pulpcore>=3.105.0,<3.130", "pkginfo>=1.12.0,<1.13.0", - "bandersnatch>=6.6.0,<6.7", + "bandersnatch>=8.0.0,<8.1", "pypi-simple>=1.8.0,<2.0", "pypi-attestations==0.0.28", # API is not stable ]