Bump bandersnatch to 8.0 and bump required python_version to >=3.12

CHANGES/+bandersnatch-8.feature

@@ -0,0 +1 @@
+Upgraded Bandersnatch to 8.0. Full-index syncs now list packages via the PEP 691 Simple JSON API, falling back to HTML `/simple/` when JSON is unavailable.

CHANGES/+python-3.12.feature

@@ -0,0 +1 @@
+Updated minimum required python version to >=3.12.

pulp_python/app/tasks/sync.py

@@ -3,11 +3,10 @@
 from functools import partial
 from urllib.parse import urljoin, urlparse
 
-from aiohttp import ClientError, ClientResponseError
+from aiohttp import ClientError
 from bandersnatch.configuration import BandersnatchConfig
 from bandersnatch.master import Master
 from bandersnatch.mirror import Mirror
-from lxml.etree import LxmlError
 from packaging.requirements import Requirement
 from pypi_simple import IndexPage
 
@@ -169,41 +168,36 @@ def __init__(self, serial, master, workers, deferred_download, python_stage, pro
 
     async def determine_packages_to_sync(self):
         """
-        Calling this means that includes wasn't specified,
-        so try to get all of the packages from Mirror (hopefully PyPi)
+        Called when includes wasn't specified. List all projects from the remote
+        via the PEP 691 Simple JSON API, falling back to HTML /simple/.
         """
-        number_xmlrpc_attempts = 3
-        for attempt in range(number_xmlrpc_attempts):
-            logger.info("Attempt {} to get package list from {}".format(attempt, self.master.url))
-            try:
-                if not self.synced_serial:
-                    logger.info("Syncing all packages.")
-                    # First get the current serial, then start to sync.
-                    all_packages = await self.master.all_packages()
-                    self.packages_to_sync.update(all_packages)
-                    self.target_serial = max(
-                        [self.synced_serial] + [int(v) for v in self.packages_to_sync.values()]
-                    )
-                else:
-                    logger.info("Syncing based on changelog.")
-                    changed_packages = await self.master.changed_packages(self.synced_serial)
-                    self.packages_to_sync.update(changed_packages)
-                    self.target_serial = max(
-                        [self.synced_serial] + [int(v) for v in self.packages_to_sync.values()]
-                    )
-                break
-            except (ClientError, ClientResponseError, LxmlError):
-                # Retry if XMLRPC endpoint failed, server might not support it.
-                continue
-        else:
-            logger.info("Failed to get package list using XMLRPC, trying parse simple page.")
+        logger.info("Syncing all packages from %s", self.master.url)
+        try:
+            simple_index = await self.master.fetch_simple_index()
+            if not isinstance(simple_index, dict) or "projects" not in simple_index:
+                raise ValueError("Simple JSON index is missing a projects list")
+            for project in simple_index["projects"]:
+                name = project.get("name")
+                if name is None:
+                    continue
+                # _last-serial is a PyPI extension; default to 0 when absent
+                self.packages_to_sync[name] = project.get("_last-serial", 0)
+            self.target_serial = max(
+                [self.synced_serial or 0] + [int(v) for v in self.packages_to_sync.values()]
+            )
+        except (ClientError, ValueError, TypeError, AttributeError) as exc:
+            logger.info(
+                "Failed to list packages via Simple JSON API (%s); "
+                "falling back to HTML simple index.",
+                exc,
+            )
             url = urljoin(self.remote.url, "simple/")
             downloader = self.remote.get_downloader(url=url)
             result = await downloader.run()
             with open(result.path) as f:
                 index = IndexPage.from_html(f.read())
-                self.packages_to_sync.update({p: 0 for p in index.projects})
-                self.target_serial = result.headers.get(PYPI_LAST_SERIAL, 0)
+            self.packages_to_sync.update({p: 0 for p in index.projects})
+            self.target_serial = result.headers.get(PYPI_LAST_SERIAL, 0)
 
         self._filter_packages()
         if self.target_serial:

pulp_python/tests/functional/api/test_download_content.py

@@ -48,7 +48,7 @@ def test_full_fixtures_to_pulp_sync(
 ):
     """
     This test checks that Pulp can fully sync another Python Package repository that is not
-    PyPI. This reads the repository's simple page if XMLRPC isn't supported.
+    PyPI. This lists projects via the Simple JSON API, falling back to HTML /simple/.
     """
     # Repository we are syncing from is the fixtures (default url)
     remote = python_remote_factory(includes=[], prereleases=True)

pyproject.toml

@@ -20,15 +20,14 @@ classifiers=[
   "Framework :: Django",
   "Programming Language :: Python",
   "Programming Language :: Python :: 3",
-  "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
   "Programming Language :: Python :: 3.13",
 ]
-requires-python = ">=3.11"
+requires-python = ">=3.12"
 dependencies = [
   "pulpcore>=3.105.0,<3.130",
   "pkginfo>=1.12.0,<1.13.0",
-  "bandersnatch>=6.6.0,<6.7",
+  "bandersnatch>=8.0.0,<8.1",
   "pypi-simple>=1.8.0,<2.0",
   "pypi-attestations==0.0.28",  # API is not stable
 ]