diff --git a/compat/data/compat.json b/compat/data/compat.json index 35f5cd1a..eb1dc774 100644 --- a/compat/data/compat.json +++ b/compat/data/compat.json @@ -1,7 +1,7 @@ { "schema": "1", "generated": "2026-10-26", - "commit": "161504b360b33ff28048889e8a8579a07184fccc", + "commit": "8188045c4bff110578c0e3c884e277c1573902ca", "sources": { "support_data": "scripts/support_data.toml", "pins": "benchmarks/compat/top100/pins.json", @@ -2207,7 +2207,7 @@ }, { "date": "2026-10-26", - "commit": "161504b360b33ff28048889e8a8579a07184fccc", + "commit": "8188045c4bff110578c0e3c884e277c1573902ca", "weighted_pass": 0.0, "packages_full": 2, "packages_partial": 7, diff --git a/compat/data/support.json b/compat/data/support.json index a38a9f85..98007e19 100644 --- a/compat/data/support.json +++ b/compat/data/support.json @@ -1,7 +1,7 @@ { "schema": "1", "generated": "2026-10-26", - "commit": "161504b360b33ff28048889e8a8579a07184fccc", + "commit": "8188045c4bff110578c0e3c884e277c1573902ca", "sources": { "support_data": "scripts/support_data.toml", "pins": "benchmarks/compat/top100/pins.json", diff --git a/compat/data/what_will_fail.json b/compat/data/what_will_fail.json index 22e7bfa9..383605f1 100644 --- a/compat/data/what_will_fail.json +++ b/compat/data/what_will_fail.json @@ -1,7 +1,7 @@ { "schema": "1", "generated": "2026-10-24", - "commit": "161504b360b33ff28048889e8a8579a07184fccc", + "commit": "8188045c4bff110578c0e3c884e277c1573902ca", "sources": { "corpus": "benchmarks/real_scripts/scripts", "report": "benchmarks/real_scripts/REPORT.md" diff --git a/src/pyscriptc/cli.py b/src/pyscriptc/cli.py index 76f06060..cde6a1cd 100644 --- a/src/pyscriptc/cli.py +++ b/src/pyscriptc/cli.py @@ -17,7 +17,7 @@ from typing import Any # boxed:begin -from pyscriptc import embed, runcache +from pyscriptc import embed, runcache, sitedeps, stdlibzip # boxed:end from pyscriptc.config import apply_to_args, entry_points, load_config @@ -262,6 +262,15 @@ def main(argv: list[str] | None = None) -> int: "libpython, bundle otherwise; the default). Also read from the `embed` key of " "[tool.pyscriptc.island]", ) + compile_parser.add_argument( + "--embed-stdlib", + choices=stdlibzip.STDLIB_MODES, + default=None, + metavar="TRIM", + help="stdlib trim of a bundle: safe (the default; everything except test, idlelib, " + "tkinter, turtledemo, ensurepip, pydoc_data, __phello__) or minimal (an opt-in frozen " + "closure plus allowlist). Also read from the `stdlib` key of [tool.pyscriptc.island]", + ) # boxed:end compile_parser.add_argument("--call", help="function the generated main calls") compile_parser.add_argument( @@ -1952,9 +1961,37 @@ def _compile_cmd(args: argparse.Namespace, state: dict[str, Any]) -> int: # boxed:begin # A bundle link writes a relocatable directory: the binary under # `bin/`, the runtime under `lib/` (the relative rpath points there). - if getattr(args, "embed_python", None) == "bundle": + # The stdlib travels as a safe-trimmed zip of .pyc bodies (issue #659), + # and the compile first manifest-checks every stdlib import of the + # program against the zip the trim would produce, so a trimmed module + # fails the build (PS4301), never the run. + resolved_embed = getattr(args, "embed_python", None) + if resolved_embed != "host" and embed.resolve_mode(resolved_embed) == "bundle": + resolved_embed = "bundle" + # The direct third-party import roots of the program, for the bundle's + # site-packages tree (issue #660). A native-backed distribution among + # them refuses the compile (PS4302) before anything is staged. + dep_roots = sitedeps.third_party_roots(source_path) if resolved_embed == "bundle" else [] + if resolved_embed == "bundle": + sitedeps.check_pure(dep_roots) pack = embed.lock_pack() - staged = embed.stage_bundle(binary, embed.pack_dir(pack), embed.python_version(), binary) + pack_directory = embed.ensure_pack(pack) + embed_version = embed.python_version() + trim = getattr(args, "embed_stdlib", None) or stdlibzip.DEFAULT_MODE + carried = stdlibzip.kept_module_names( + stdlibzip.stdlib_sources(pack_directory / "lib" / f"python{embed_version}", trim) + ) + stdlibzip.manifest_check(carried, _stdlib_imports(source_path), trim) + if resolved_embed == "bundle": + pack = embed.lock_pack() + staged = embed.stage_bundle( + binary, + embed.pack_dir(pack), + embed.python_version(), + binary, + stdlib_mode=getattr(args, "embed_stdlib", None) or stdlibzip.DEFAULT_MODE, + dependency_roots=dep_roots, + ) outputs["bin"] = str(staged) # boxed:end state["binary_bytes"] = binary.stat().st_size if binary.is_file() else None @@ -1982,6 +2019,26 @@ def _compile_cmd(args: argparse.Namespace, state: dict[str, Any]) -> int: dbg.write_dir(module, llvm_text, outputs, status) +def _stdlib_imports(source_path: Path) -> list[str]: + """The absolute stdlib import names of the program, for the bundle manifest check. + + Relative imports stay inside the program tree and third-party imports go + through the site-packages bundle (issue #660), so only stdlib roots reach + the check. `sys.stdlib_module_names` of the running interpreter answers + it: the compiler and the pack share the 3.14 stdlib. + """ + + from pyscriptc.imports import collect_imports + + names: set[str] = set() + for site in collect_imports(source_path): + if site["level"] > 0 or not site["name"]: + continue + if site["name"].split(".")[0] in sys.stdlib_module_names: + names.add(site["name"]) + return sorted(names) + + def _dependency_roots( source_path: Path, config: Any, island_default: bool = False ) -> tuple[list[Path], list[Path], dict[str, dict[str, str]]]: diff --git a/src/pyscriptc/config.py b/src/pyscriptc/config.py index 864e83b3..d61d437d 100644 --- a/src/pyscriptc/config.py +++ b/src/pyscriptc/config.py @@ -177,6 +177,7 @@ def fallback_definitions(self) -> bool: def island_embed(self) -> str | None: """The `[tool.pyscriptc.island] embed` mode, or None when unset.""" + if isinstance(self.island, dict): value = self.island.get("embed") if value is not None: @@ -188,6 +189,20 @@ def island_embed(self) -> str | None: return str(value) return None + def island_stdlib(self) -> str | None: + """The `[tool.pyscriptc.island] stdlib` trim, or None when unset.""" + + if isinstance(self.island, dict): + value = self.island.get("stdlib") + if value is not None: + from pyscriptc.stdlibzip import STDLIB_MODES + + table = "pyscriptc.island" if self.source == "pyscriptc.toml" else "tool.pyscriptc.island" + if value not in STDLIB_MODES: + raise ConfigError(f"{table}.stdlib must be one of {'|'.join(STDLIB_MODES)}, got {value!r}") + return str(value) + return None + # boxed:end def lane_for(self, name: str) -> tuple[str, str]: @@ -494,5 +509,9 @@ def apply_to_args(config: Config, args: Any) -> list[str]: island_embed = config.island_embed() if island_embed is not None: args.embed_python = island_embed + if hasattr(args, "embed_stdlib") and getattr(args, "embed_stdlib", None) is None: + island_stdlib = config.island_stdlib() + if island_stdlib is not None: + args.embed_stdlib = island_stdlib # boxed:end return notes diff --git a/src/pyscriptc/embed.py b/src/pyscriptc/embed.py index e43713b9..e081c7fe 100644 --- a/src/pyscriptc/embed.py +++ b/src/pyscriptc/embed.py @@ -38,6 +38,8 @@ from dataclasses import dataclass from pathlib import Path +from pyscriptc import sitedeps, stdlibzip + MODES = ("host", "bundle", "auto") DEFAULT_MODE = "auto" @@ -387,14 +389,27 @@ def link_flags(pack_directory: Path, version: str) -> tuple[str, list[str]]: return str(include), flags -def stage_bundle(output: Path, pack_directory: Path, version: str, binary: Path) -> Path: +def stage_bundle( + output: Path, + pack_directory: Path, + version: str, + binary: Path, + *, + stdlib_mode: str = stdlibzip.DEFAULT_MODE, + dependency_roots: list[str] | None = None, +) -> Path: """Copy the pack runtime next to the binary: `name/bin/name` plus `name/lib/`. - The binary moves into `bin/`; the shared libpython and the stdlib go into - `lib/`, which is what the relative rpath and the getpath landmarks point - at (a python-build-standalone `install/` tree is relocatable: it carries + The binary moves into `bin/`; the shared libpython goes into `lib/`, which + is what the relative rpath and the getpath landmarks point at (a + python-build-standalone `install/` tree is relocatable: it carries `@executable_path/../lib` already, so no install-name rewrite is needed). - Headers stay in the pack: a relocated bundle runs, it does not compile. + The stdlib is staged as a zip of `.pyc` bodies plus a real `lib-dynload/` + directory (issue #659); the unpacked tree does not travel, and the pure- + Python distributions behind `dependency_roots` are copied into its + `site-packages/` (issue #660; a native dependency is refused by + `sitedeps.check_pure` before staging begins). Headers stay in the pack: a + relocated bundle runs, it does not compile. """ out_dir = output if output.suffix == "" else output.parent / output.stem parent = out_dir.parent @@ -418,9 +433,12 @@ def stage_bundle(output: Path, pack_directory: Path, version: str, binary: Path) for extra in sorted(shared.parent.glob(f"libpython{version}.so*")): shutil.copy2(extra, lib_dir / extra.name, follow_symlinks=False) stdlib_src = pack_directory / "lib" / f"python{version}" - stdlib_dest = lib_dir / f"python{version}" - if stdlib_src.is_dir() and not stdlib_dest.exists(): - shutil.copytree(stdlib_src, stdlib_dest, symlinks=True, ignore=shutil.ignore_patterns("__pycache__")) + if stdlib_src.is_dir(): + stdlibzip.stage_stdlib(lib_dir, pack_directory, version, stdlib_mode) + else: + (lib_dir / f"python{version}" / "lib-dynload").mkdir(parents=True, exist_ok=True) + if dependency_roots: + sitedeps.stage_site_packages(lib_dir, version, dependency_roots) out_dir = Path(out_dir) if out_dir.exists(): shutil.rmtree(out_dir) diff --git a/src/pyscriptc/imports.py b/src/pyscriptc/imports.py index b395e88f..ac2d4fd4 100644 --- a/src/pyscriptc/imports.py +++ b/src/pyscriptc/imports.py @@ -265,8 +265,29 @@ def _package_root(target: Path) -> str: def _read_wheel_record(path: Path) -> list[str] | None: - """The file list of a wheel, or None when it is not a readable wheel.""" + """The file list of a wheel RECORD, or None when it is not readable. + A RECORD inside an unpacked `*.dist-info/` directory is read as a text + file; a RECORD still inside a wheel archive (the cached-wheel case) is + read from the zip. + """ + + if not path.is_file(): + return None + if not zipfile.is_zipfile(path): + # An unpacked `*.dist-info/RECORD` is a csv text file. Refuse a plain + # text file whose lines carry no RECORD separators, so a misnamed + # non-record is not read as a one-file wheel. + try: + body = path.read_text(encoding="utf-8", errors="replace") + except OSError: + return None + files: list[str] = [] + for line in body.splitlines(): + if not line.strip(): + continue + files.append(line.split(",", 1)[0]) + return files if any("," in line for line in body.splitlines()) else None if not zipfile.is_zipfile(path): return None try: @@ -369,16 +390,20 @@ def _installed_distributions() -> dict[str, dict[str, Any]]: for entry in sorted(root.iterdir()): metadata = _metadata_of_tree(entry) if metadata["dist"]: - key = metadata["dist"].lower().replace("_", "-") + key = metadata["dist"].lower().replace("_", "-").replace(" ", "-") if key not in found: found[key] = metadata for dist_info in sorted(root.glob("*.dist-info")): name = dist_info.name[: -len(".dist-info")] - key = name.lower().replace("_", "-") - metadata = found.get(key) + key = name.lower().replace("_", "-").replace(" ", "-") + # A versioned directory name (`markupsafe-3.0.4.dist-info`) does + # not match the distribution key the metadata loop built, so strip + # a trailing version before looking the entry up. + stripped = key.split("-")[0] if key.rsplit("-", 1)[-1][0].isdigit() else key + metadata = found.get(stripped) or found.get(key) if metadata is None: metadata = {"dist": name, "root": dist_info, "native": False} - found[key] = metadata + found[stripped] = metadata wheel = dist_info / "RECORD" if wheel.is_file(): files = _read_wheel_record(wheel) @@ -395,8 +420,13 @@ def distribution_is_native(name: str) -> tuple[bool | None, str]: its metadata cannot be read. The caller must not read `None` as pure. """ - key = name.lower().replace("_", "-") - metadata = _installed_distributions().get(key) + key = name.lower().replace("_", "-").replace(" ", "-") + found = _installed_distributions() + metadata = found.get(key) + if metadata is None and "-" in key and key.rsplit("-", 1)[-1][0].isdigit(): + # A versioned lookup (`nativelib-1.0`) matches the distribution key + # the site metadata built (`nativelib`). + metadata = found.get(key.rsplit("-", 1)[0]) if metadata is None: return None, "not installed in this interpreter" if metadata["native"]: diff --git a/src/pyscriptc/sitedeps.py b/src/pyscriptc/sitedeps.py new file mode 100644 index 00000000..314d3fe5 --- /dev/null +++ b/src/pyscriptc/sitedeps.py @@ -0,0 +1,253 @@ +"""Bundle pure-Python dependencies into a bundle runtime's site-packages. + +Everything here is boxed-only code (the ledger counts this module whole; see +`.agents/docs/boxed-cost.md` and `AGENTS.md`). + +A `--embed-python=bundle` island program runs against the pack's CPython, not +the compiler's interpreter, so a third-party module the program imports must +travel with the bundle or the run dies with `ModuleNotFoundError`. This module +copies the distributions behind the program's third-party import roots into +`lib/pythonX.Y/site-packages/` of the staged bundle (the directory the pack's +`site` initialization puts on `sys.path`). + +A distribution that ships a compiled extension (`.so`, `.pyd` and friends) is +refused with `PS4302`, never silently dropped: a copied extension would have to +match the pack's ABI and platform, and one that does not fails at runtime in +ways the compiler cannot check. An import root no installed distribution +provides is reported on stderr and skipped: the island fails at runtime exactly +like CPython would, which is the honest answer for a dead-branch import. + +Direct imports only (the roots the program's own statements name); a +dependency-of-dependency arrives through its importer's imports when the +program reaches it. +""" + +from __future__ import annotations + +import shutil +import sys +from pathlib import Path + + +class SiteError(Exception): + """A user-facing site-packages bundle failure (no traceback).""" + + +def third_party_roots(source: Path) -> list[str]: + """The absolute third-party import roots of a program, in source order. + + Stdlib roots go to the stdlib zip (issue #659), relative imports stay in + the program tree, and the program's own package root is not a dependency. + """ + + from pyscriptc.imports import collect_imports + + source = Path(source) + own = source.stem if source.is_file() else source.name + found: list[str] = [] + for site in collect_imports(source): + if site["level"] > 0 or not site["name"]: + continue + root = site["name"].split(".")[0] + if root in sys.stdlib_module_names or root == own: + continue + if root not in found: + found.append(root) + return sorted(found) + + +def distributions_of(root: str) -> list[str]: + """The installed distribution names an import root belongs to. + + `importlib.metadata.packages_distributions()` first (a top_level.txt map + the installer writes); when it does not know the root, the site metadata + answers: a `*.dist-info` whose RECORD lists the root, or else a package + directory of that name on a site path as its own distribution (how source + trees and RECORD-only installs answer). + """ + + from importlib import metadata + + try: + found = list(metadata.packages_distributions().get(root, [])) + except Exception: + found = [] + if found: + return found + from pyscriptc import imports + + for site_dir in imports._site_packages(): + if not site_dir.is_dir(): + continue + for dist_info in sorted(site_dir.glob("*.dist-info")): + if root in _record_roots(dist_info / "RECORD"): + return [dist_info.name[: -len(".dist-info")]] + if (site_dir / root).is_dir() or (site_dir / f"{root}.py").is_file(): + return [root] + return [] + + +def check_pure(roots: list[str]) -> None: + """Refuse (PS4302) when a dependency the program imports is native-backed. + + Called before anything is staged, so a native dependency costs no fetch + and no partial bundle. A root with no installed distribution is not this + check's business: `stage_site_packages` warns and skips it, and the run + answers like CPython would. + """ + + from pyscriptc import imports + + for dist, dist_roots in _closure(roots).items(): + native, reason = imports.distribution_is_native(dist) + if native: + raise SiteError( + f"PS4302: the dependency {dist} (import {', '.join(dist_roots)}) cannot be bundled: {reason}; " + "a bundle carries pure Python only - compile on the target platform or use " + "an extern manifest for the native part" + ) + + +def stage_site_packages(lib_dir: Path, version: str, roots: list[str]) -> list[str]: + """Copy the pure-Python distributions behind `roots` into the bundle. + + The copy set is the transitive closure over `Requires-Dist`: a dependency + of a dependency travels with it (`python-dateutil` needs `six`), resolved + against what this interpreter has installed. Returns the staged + distribution names. A native-backed distribution anywhere in the closure + raises `SiteError` with `PS4302`; a root with no installed distribution is + reported on stderr and skipped. + """ + + from pyscriptc import imports + + dest = lib_dir / f"python{version}" / "site-packages" + dest.mkdir(parents=True, exist_ok=True) + staged: list[str] = [] + resolved = _closure(roots) + for root in roots: + if not distributions_of(root): + print( + f"warning: {root} is not installed in this interpreter; the bundle will not carry it", + file=sys.stderr, + ) + for dist, dist_roots in resolved.items(): + native, reason = imports.distribution_is_native(dist) + if native: + raise SiteError( + f"PS4302: the dependency {dist} (import {', '.join(dist_roots)}) cannot be bundled: " + f"{reason}; a bundle carries pure Python only - compile on the target platform or " + "use an extern manifest for the native part" + ) + copied = False + for root in dist_roots: + copied = _copy_distribution(root, dist, dest) or copied + if copied: + staged.append(dist) + else: + print( + f"warning: {dist} is installed but its files were not found on any site path", + file=sys.stderr, + ) + return staged + + +def _closure(roots: list[str]) -> dict[str, list[str]]: + """Distribution name to the import roots that reach it, dependencies included. + + The walk is over the installed metadata only (`Requires-Dist` with no + environment marker evaluation beyond what `importlib.metadata` gives); an + unsatisfied requirement still travels when its distribution is installed, + and a missing one ends the branch. + """ + + from importlib import metadata + + resolved: dict[str, list[str]] = {} + queue: list[str] = list(roots) + seen: set[str] = set() + while queue: + root = queue.pop(0) + if root in seen: + continue + seen.add(root) + for dist in distributions_of(root): + # A distribution is copied for the import roots that reach it, so + # `six` stages `six.py` even though only `dateutil` names it. + roots_of = resolved.setdefault(dist, []) + if root not in roots_of: + roots_of.append(root) + if dist in seen: + continue + seen.add(dist) + try: + requires = metadata.requires(dist) or [] + except Exception: + requires = [] + for requirement in requires: + if ";" in requirement: + marker = requirement.split(";", 1)[1] + # extra markers (`extra == "..."`) are optional unless the + # program opts in by importing the extra's modules, and a + # failing marker means the requirement does not apply here. + if "extra" not in marker: + try: + import packaging.markers + + if not packaging.markers.Marker(marker).evaluate(): + continue + except Exception: + pass + name = requirement.split(";", 1)[0].strip() + for separator in " <>=!~[": + name = name.split(separator, 1)[0] + name = name.strip() + if name and name not in seen: + queue.append(name) + return resolved + + +def _copy_distribution(root: str, dist: str, dest: Path) -> None: + """Copy one distribution's importable files and metadata into `dest`. + + `root` is an import name; the files it resolves to are the package + directory or module file of that name on any site path, plus every + `*.dist-info` whose RECORD lists the root or whose distribution name + matches. Unknown roots are the caller's warning, not a copy attempt. + """ + + from pyscriptc import imports + + dest.mkdir(parents=True, exist_ok=True) + copied = False + for site_dir in imports._site_packages(): + if not site_dir.is_dir(): + continue + for candidate in (site_dir / root, site_dir / f"{root}.py"): + if candidate.is_dir(): + shutil.copytree(candidate, dest / candidate.name, dirs_exist_ok=True) + copied = True + elif candidate.is_file(): + shutil.copy2(candidate, dest / candidate.name) + copied = True + for dist_info in site_dir.glob("*.dist-info"): + recorded = _record_roots(dist_info / "RECORD") + if root in recorded or dist_info.name[: -len(".dist-info")].lower().replace("_", "-") == dist: + shutil.copytree(dist_info, dest / dist_info.name, dirs_exist_ok=True) + copied = True + return copied + + +def _record_roots(record: Path) -> set[str]: + """The top-level path roots a wheel RECORD lists.""" + + if not record.is_file(): + return set() + found: set[str] = set() + for line in record.read_text(encoding="utf-8", errors="replace").splitlines(): + entry = line.split(",", 1)[0].strip() + if entry and "/" in entry: + found.add(entry.split("/", 1)[0]) + elif entry: + found.add(entry.rsplit(".py", 1)[0]) + return found diff --git a/src/pyscriptc/stdlibzip.py b/src/pyscriptc/stdlibzip.py new file mode 100644 index 00000000..56cad3bc --- /dev/null +++ b/src/pyscriptc/stdlibzip.py @@ -0,0 +1,443 @@ +"""The bundled stdlib of a runtime pack: a safe-trimmed zip of `.pyc`. + +Everything here is boxed-only code (the ledger counts this module whole; see +`.agents/docs/boxed-cost.md` and `AGENTS.md`). + +A `--embed-python=bundle` program runs against the pack's CPython, but the +`lib/python3.14/` tree of a pack is 10 MB of `.py` files the interpreter would +compile at first import. `stage_stdlib` writes it as a single zip of +byte-compiled modules instead, plus a real `lib-dynload/` directory (getpath +prints a fatal-looking warning without one, and PBS builds most extension +modules into libpython so the directory is near-empty). + +Two trims, both shipped as zips: + +- `safe` (the default): everything except a documented deny list of trees no + compiled program or island import needs (`test`, `idlelib`, `tkinter`, + `turtledemo`, `ensurepip`, `pydoc_data`, `__phello__`). The trim is safe + because it removes only developer-facing material; anything else an island + program imports must be present, and `stdlib_manifest` fails the compile + closed (`PS4301`) when it is not. +- `minimal` (opt-in, `--embed-stdlib=minimal`): a frozen closure over the + CPython import graph seeded from `PYTHONSTARTUP_MODULES` (the island floor: + `encodings`, `codecs`, `os` and friends) plus an explicit allowlist + (`MINIMAL_EXTRA`). It exists for size-critical bundles; a missing module + still fails closed at compile time, so the failure is a build error, not a + runtime `ModuleNotFoundError`. + +The zip stores `.pyc` bodies with their magic-number header; a stored +(uncompressed) zip keeps the pages cache-friendly and the pack reproducible +byte-for-byte from the same pack directory. +""" + +from __future__ import annotations + +import ast +import os +import py_compile +import sys +import tempfile +import zipfile +from pathlib import Path + +STDLIB_MODES = ("safe", "minimal") +DEFAULT_MODE = "safe" + +# The documented safe-trim deny list: top-level stdlib trees and modules no +# compiled program or island import needs. `test` is not even present in the +# `install_only_stripped` archives, but a pack built from another asset may +# carry it. This list is the whole contract of the safe trim; grow it only with +# a documented reason, never to chase a smaller zip at correctness' expense. +SAFE_TRIM = ( + "__phello__", + "idlelib", + "ensurepip", + "pydoc_data", + "test", + "tkinter", + "turtledemo", +) + +# The frozen closure seed of the minimal trim: what an island bootstrap needs +# before the first program import (the frozen importlib pulls these). +MINIMAL_SEED = ( + "_collections_abc", + "_compat_pickle", + "_compression", + "_sitebuiltins", + "_weakrefset", + "abc", + "codecs", + "collections", + "contextlib", + "copyreg", + "encodings", + "enum", + "functools", + "genericpath", + "heapq", + "importlib", + "io", + "itertools", + "keyword", + "linecache", + "operator", + "os", + "posixpath", + "re", + "reprlib", + "site", + "sre_compile", + "sitecustomize", + "stat", + "types", + "typing", + "warnings", + "weakref", + "zipfile", +) + +# Modules the minimal trim keeps unconditionally even though the seed closure +# does not reach them: the island bootstrap (`importlib.abc`, `importlib.util`, +# `json` in tests, `pathlib` in everyday code) and `site`'s dependencies. +MINIMAL_EXTRA = ( + "_blake2", + "_bootstrap_lite", + "collections.abc", + "concurrent", + "configparser", + "contextvars", + "copy", + "dataclasses", + "datetime", + "difflib", + "dis", + "fnmatch", + "fractions", + "gc", + "glob", + "hashlib", + "hmac", + "inspect", + "json", + "locale", + "logging", + "numbers", + "pathlib", + "pickle", + "platform", + "pprint", + "random", + "shutil", + "socket", + "socketserver", + "ssl", + "string", + "struct", + "subprocess", + "sysconfig", + "tempfile", + "textwrap", + "threading", + "traceback", + "types", + "unittest", + "urllib", + "uuid", + "argparse", + "base64", + "bisect", + "bz2", + "cmath", + "contextlib", + "csv", + "getopt", + "getpass", + "gzip", + "html", + "http", + "ipaddress", + "lzma", + "queue", + "shlex", + "signal", + "sqlite3", + "statistics", + "tarfile", + "timeit", + "tomllib", + "unittest.mock", + "uuid", + "xml", + "zlib", +) + + +class StdlibError(Exception): + """A user-facing stdlib bundle failure (no traceback).""" + + +def safe_trim_modules(stdlib_dir: Path) -> list[Path]: + """Every `.py` of a stdlib tree that the safe trim keeps. + + The deny list matches top-level names only: `tkinter` drops the package + (and with it `turtledemo`'s only consumer), not a same-named submodule of + an unrelated package. + """ + if not stdlib_dir.is_dir(): + raise StdlibError(f"the runtime pack has no stdlib tree at {stdlib_dir}") + kept: list[Path] = [] + for path in sorted(stdlib_dir.rglob("*.py")): + relative = path.relative_to(stdlib_dir) + if relative.parts[0] in SAFE_TRIM: + continue + kept.append(path) + return kept + + +def _import_names(source: str) -> set[str]: + """The absolute import names of a module source, ignoring relatives.""" + + found: set[str] = set() + try: + tree = ast.parse(source) + except SyntaxError: + return found + for node in ast.walk(tree): + if isinstance(node, ast.Import): + found.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + found.add(node.module) + return found + + +def _stdlib_available() -> Path: + """A stdlib tree to read sources from: the running interpreter's.""" + candidate = Path(sys.base_prefix) / "lib" / f"python{sys.version_info.major}.{sys.version_info.minor}" + if candidate.is_dir(): + return candidate + for entry in sys.path: + if entry.endswith("-packages") or not entry: + continue + base = Path(entry) + if (base / "os.py").is_file(): + return base + raise StdlibError("cannot locate a stdlib source tree for the minimal closure") + + +def minimal_modules(stdlib_dir: Path | None = None) -> list[Path]: + """The minimal-trim module list: a frozen closure, not a runtime one. + + The closure walks the import graph of the seed plus allowlist sources + breadth-first over the stdlib tree of the running interpreter (the compiler + host and the pack share the 3.14 stdlib layout), one pass per module, and + then maps the kept names back to the pack's own files. Frozen means the + answer depends on these two tuples and the 3.14 sources only, so two + compilers agree without executing anything. + """ + source_tree = stdlib_dir if stdlib_dir is not None and (stdlib_dir / "os.py").is_file() else _stdlib_available() + # A seed package travels whole (`encodings` pulls its codec and alias + # modules through dynamic imports no static walk sees); a seed module + # travels alone and only grows through the closure below. + keep = set() + queue: list[str] = [] + for name in (*MINIMAL_SEED, *MINIMAL_EXTRA): + parts = name.split("/") + package = source_tree.joinpath(*parts) + if package.is_dir() and (package / "__init__.py").is_file(): + keep.add(name) + for child in sorted(package.glob("*.py")): + keep.add(f"{name}.{child.stem}") + queue.append(name) + seen: set[str] = set() + while queue: + name = queue.pop(0) + if name in seen: + continue + seen.add(name) + if name.split(".")[0] in SAFE_TRIM: + # The deny list is out of every trim: `pydoc` pulls `pydoc_data` + # through a function-body import the island never executes. + continue + path = source_tree / f"{name.replace('.', '/')}.py" + if not path.is_file(): + path = source_tree / name.replace(".", "/") / "__init__.py" + if not path.is_file(): + continue + keep.add(name) + for found in _import_names(path.read_text(encoding="utf-8", errors="replace")): + if found in seen: + continue + parts = found.split(".") + # Only stdlib names continue the closure; a third-party or unknown + # import cannot be satisfied from the stdlib tree anyway. + candidate = source_tree / f"{parts[0]}.py" + if not candidate.is_file() and not (source_tree / parts[0]).is_dir(): + continue + queue.append(found) + for cut in range(1, len(parts) + 1): + prefix = ".".join(parts[:cut]) + if prefix.split(".")[0] not in SAFE_TRIM: + keep.add(prefix) + tree = stdlib_dir if stdlib_dir is not None else source_tree + files: list[Path] = [] + for name in sorted(keep): + rel = name.replace(".", "/") + for candidate in (tree / f"{rel}.py", tree / rel / "__init__.py"): + if candidate.is_file(): + files.append(candidate) + break + return files + + +def kept_module_names(sources: list[Path], stdlib_dir: Path | None = None) -> set[str]: + """The dotted module names a source list keeps, derived from its paths. + + Every package level of `email/mime/text.py` counts (`email`, + `email.mime`, `email.mime.text`), so the manifest check can answer a + submodule import without unpacking anything. + """ + + base = stdlib_dir + if base is None and sources: + # Every source sits under the same stdlib root; recover it from the + # first path (`.../lib/python3.14/os.py`). + parts = sources[0].parts + for cut in range(len(parts) - 1, -1, -1): + if parts[cut].startswith("python") and parts[cut][6:].replace(".", "").isdigit(): + base = Path(*parts[: cut + 1]) + break + if base is None: + return set() + found: set[str] = set() + for source in sources: + try: + relative = source.relative_to(base) + except ValueError: + continue + parts = list(relative.with_suffix("").parts) + if parts and parts[-1] == "__init__": + parts.pop() + for cut in range(1, len(parts) + 1): + found.add(".".join(parts[:cut])) + return found + + +def _pyc_bytes(source: Path, module_name: str) -> bytes: + """The `.pyc` body of one module, compiled for the running interpreter.""" + + with tempfile.TemporaryDirectory(prefix="pyscriptc-pyc-") as tmp: + target = Path(tmp) / "module.pyc" + py_compile.compile( + str(source), + cfile=str(target), + dfile=f"{module_name}.py", + doraise=True, + optimize=-1, + ) + return target.read_bytes() + + +def write_zip(sources: list[Path], stdlib_dir: Path, zip_path: Path) -> Path: + """Write a stored zip of `.pyc` bodies for the kept modules. + + Archive names are the module paths relative to the stdlib root, so + `python314.zip` answers `import encodings` exactly as the unpacked tree + did. `.py` files are kept out on purpose: startup must not compile. + """ + zip_path.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_STORED) as archive: + for source in sorted(sources): + relative = source.relative_to(stdlib_dir) + module_name = str(relative.with_suffix("")).replace(os.sep, ".") + archive.writestr(str(relative.with_suffix(".pyc")), _pyc_bytes(source, module_name)) + return zip_path + + +def stdlib_sources(stdlib_dir: Path, mode: str) -> list[Path]: + """The kept module sources of a trim mode (`safe` or `minimal`).""" + if mode == "safe": + return safe_trim_modules(stdlib_dir) + if mode == "minimal": + return minimal_modules(stdlib_dir) + raise StdlibError(f"--embed-stdlib must be one of {'|'.join(STDLIB_MODES)}, got {mode!r}") + + +def stdlib_zip_name(version: str) -> str: + """The zip file name a `pythonXY` landmark expects (`python314.zip`).""" + + return f"python{version.replace('.', '')}.zip" + + +def stage_stdlib(lib_dir: Path, pack_directory: Path, version: str, mode: str = DEFAULT_MODE) -> Path: + """Replace the unpacked stdlib tree of a staged bundle with its zip. + + Writes `lib/python314.zip` of `.pyc` bodies plus an empty-or-real + `lib/python3.14/lib-dynload/` (the getpath landmark) and removes the + unpacked tree, so `sys.path` resolves the zip. Returns the zip path. + """ + if mode not in STDLIB_MODES: + raise StdlibError(f"--embed-stdlib must be one of {'|'.join(STDLIB_MODES)}, got {mode!r}") + stdlib_dir = lib_dir / f"python{version}" + pack_stdlib = pack_directory / "lib" / f"python{version}" + if not stdlib_dir.is_dir() and not pack_stdlib.is_dir(): + raise StdlibError(f"the runtime pack has no stdlib tree at {pack_stdlib}") + sources = stdlib_sources(pack_stdlib if pack_stdlib.is_dir() else stdlib_dir, mode) + zip_path = lib_dir / stdlib_zip_name(version) + write_zip(sources, pack_stdlib if pack_stdlib.is_dir() else stdlib_dir, zip_path) + import shutil + + if stdlib_dir.is_dir(): + shutil.rmtree(stdlib_dir) + dynload = stdlib_dir / "lib-dynload" + dynload.mkdir(parents=True, exist_ok=True) + return zip_path + + +def zip_modules(zip_path: Path) -> set[str]: + """The module names a stdlib zip carries, dotted, submodules included.""" + + found: set[str] = set() + with zipfile.ZipFile(zip_path) as archive: + for name in archive.namelist(): + found |= _names_of_entry(name) + return found + + +def _names_of_entry(name: str) -> set[str]: + """The dotted module names one zip entry provides (module plus parents).""" + + if not name.endswith(".pyc"): + return set() + relative = name[: -len(".pyc")] + if relative.endswith("/__init__"): + relative = relative[: -len("/__init__")] + if not relative: + return set() + parts = str(relative).split("/") + return {".".join(parts[: cut + 1]) for cut in range(len(parts))} + + +def manifest_check(carried: set[str], imported: list[str], mode: str = DEFAULT_MODE) -> None: + """Fail closed (PS4301) when an imported stdlib module is not carried. + + `carried` is what the trim keeps (`zip_modules` of the zip, or the kept + module names before it is written); `imported` are the absolute stdlib + import names the program (and the island bootstrap) uses. A name is + importable when the zip carries it or a parent of it; the caller passes + stdlib names only, so relative and third-party imports are not this + check's business. + """ + + missing: list[str] = [] + for name in imported: + if not name or name in carried: + continue + parts = name.split(".") + if not any(".".join(parts[: cut + 1]) in carried for cut in range(len(parts))): + missing.append(name) + if missing: + raise StdlibError( + f"PS4301: the embedded stdlib (mode {mode}) has no module {', '.join(sorted(set(missing)))}; " + "the trim must keep what the program imports" + ) diff --git a/tests/test_census_self.py b/tests/test_census_self.py index 963de8f1..dfcd8bcf 100644 --- a/tests/test_census_self.py +++ b/tests/test_census_self.py @@ -52,7 +52,7 @@ def test_self_census_ballpark() -> None: # target.py (#532, declared CPython target configuration) adds 1 module. # boxed.py (#612, the --fallback=cpython island planner) adds 1 module. # embed.py (#657, the pinned CPython runtime pack) adds 1 module. - assert census["modules"] == 101 + assert census["modules"] == 103 # E1 items 2+7a: print support added 3 self-host fences, then the # list.append flip removed 20 fenced call sites; headroom 14k for wave-C lowers. # telemetry.py added 36 fences, log.py 20, debug.py 6 union-type @@ -267,11 +267,11 @@ def test_self_census_ballpark() -> None: # yield inside try (#164): the re-entrant try in the coro guard transform # adds about 150 lines in coro.py. The asyncio primitives (#180 D3) add # about 400 in ps_async.c and restlower/_asyncio_sync.py. - assert 11_000 <= census["lines"] <= 51_000 + assert 11_000 <= census["lines"] <= 53_000 # Relocated program data files (#665): the staged-data plumbing in # cli.py and package.py and the rebasing bootstrap in boxed.py add # about 200 lines and 2 self-host fences. - assert 11_000 <= census["lines"] <= 51_000 + assert 11_000 <= census["lines"] <= 53_000 # Island class definitions (#628): +9 self-host fences, one in # emit/llvm/_abi.py (the class-handle trampoline), one in # lower/_classes.py (the island class annotation), three in @@ -282,6 +282,6 @@ def test_self_census_ballpark() -> None: # in emit/llvm/_driver.py. # embed.py (#657, the pinned CPython runtime pack) adds 1 module, about # 470 lines and 99 self-host fences, taking the source past 50k. - assert 11_000 <= census["lines"] <= 51_000 - assert census["fences"] == 3140 - assert sum(p["fences"] for p in census["packages"]) == 3140 + assert 11_000 <= census["lines"] <= 53_000 + assert census["fences"] == 3204 + assert sum(p["fences"] for p in census["packages"]) == 3204 diff --git a/tests/test_embed.py b/tests/test_embed.py index a2e2a6e4..e1ae6793 100644 --- a/tests/test_embed.py +++ b/tests/test_embed.py @@ -153,7 +153,11 @@ def test_stage_bundle_writes_bin_and_lib(tmp_path: Path) -> None: assert staged.is_file() libs = list((tmp_path / "work" / "prog" / "lib").glob(f"libpython{VERSION}*")) assert libs - assert (tmp_path / "work" / "prog" / "lib" / f"python{VERSION}" / "os.py").is_file() + # Issue #659: the stdlib travels as a zip of .pyc bodies plus the + # lib-dynload landmark, never as the unpacked tree. + assert (tmp_path / "work" / "prog" / "lib" / f"python{VERSION.replace('.', '')}.zip").is_file() + assert (tmp_path / "work" / "prog" / "lib" / f"python{VERSION}" / "lib-dynload").is_dir() + assert not (tmp_path / "work" / "prog" / "lib" / f"python{VERSION}" / "os.py").exists() def test_default_island_link_is_the_host_libpython(tmp_path: Path) -> None: diff --git a/tests/test_imports.py b/tests/test_imports.py index ba5678a6..8f3541e1 100644 --- a/tests/test_imports.py +++ b/tests/test_imports.py @@ -407,6 +407,24 @@ def test_wheel_record_reader_rejects_a_file_that_is_not_a_zip(tmp_path: Path) -> assert _read_wheel_record(path) is None +def test_wheel_record_reader_reads_an_unpacked_record(tmp_path: Path) -> None: + """An unpacked ``*.dist-info/RECORD`` is a text file, not a zip (issue #660).""" + + from pyscriptc.imports import _read_wheel_record + + record = tmp_path / "RECORD" + record.write_text("pkg/__init__.py,,\npkg/_speedups.cpython-314-x86_64-linux-gnu.so,,\n", encoding="utf-8") + files = _read_wheel_record(record) + assert files is not None + assert "pkg/_speedups.cpython-314-x86_64-linux-gnu.so" in files + + +def test_wheel_record_reader_rejects_a_missing_file(tmp_path: Path) -> None: + from pyscriptc.imports import _read_wheel_record + + assert _read_wheel_record(tmp_path / "absent") is None + + def test_distribution_is_native_is_unknown_when_not_installed() -> None: from pyscriptc.imports import distribution_is_native diff --git a/tests/test_sitedeps.py b/tests/test_sitedeps.py new file mode 100644 index 00000000..58a6eb88 --- /dev/null +++ b/tests/test_sitedeps.py @@ -0,0 +1,144 @@ +"""Bundle pure-Python dependencies into a bundle runtime's site-packages. + +Refs #660. The unit tests build a fake site-packages tree (no network, no +real wheels) except the ones that read what this interpreter actually has +installed; those use the deps the project venv always carries. +""" + +from __future__ import annotations + +import sysconfig +from pathlib import Path + +import pytest + +from pyscriptc import sitedeps + + +def _program(tmp_path: Path, imports: str) -> Path: + src = tmp_path / "prog.py" + body = "".join(f"import {name}\n" for name in imports.split()) + src.write_text(body, encoding="utf-8") + return src + + +def test_third_party_roots_skip_stdlib_relative_and_own(tmp_path: Path) -> None: + src = tmp_path / "pkg" + (src / "pkg").mkdir(parents=True) + (src / "pkg" / "__init__.py").write_text("", encoding="utf-8") + (src / "pkg" / "mod.py").write_text( + "import json\nfrom . import other\nimport dateutil.parser\nimport pkg.helper\n", + encoding="utf-8", + ) + roots = sitedeps.third_party_roots(src) + assert roots == ["dateutil"] + + +def test_third_party_roots_keep_first_order_and_dedupe(tmp_path: Path) -> None: + src = _program(tmp_path, "b.a a a.c") + assert sitedeps.third_party_roots(src) == ["a", "b"] or sitedeps.third_party_roots(src) == sorted({"a", "b"}) + + +def test_distributions_of_maps_a_root_to_its_distribution() -> None: + # `packaging` is a pyscriptc dependency, always installed in the venv. + assert "packaging" in sitedeps.distributions_of("packaging") + + +def test_distributions_of_answers_empty_for_the_unknown(tmp_path: Path) -> None: + assert sitedeps.distributions_of("definitely-not-a-distribution-emb660") == [] + + +def _fake_site(tmp_path: Path) -> Path: + site = tmp_path / "site-packages" + site.mkdir(parents=True, exist_ok=True) + (site / "purepkg").mkdir() + (site / "purepkg" / "__init__.py").write_text("value = 1\n", encoding="utf-8") + (site / "puremod.py").write_text("value = 2\n", encoding="utf-8") + native = site / "nativepkg" + native.mkdir() + (native / "__init__.py").write_text("value = 3\n", encoding="utf-8") + (native / "_speedups.cpython-314-x86_64-linux-gnu.so").write_bytes(b"\x7fELF") + native_info = site / "nativelib-1.0.dist-info" + native_info.mkdir() + (native_info / "RECORD").write_text( + "nativepkg/_speedups.cpython-314-x86_64-linux-gnu.so,,\nnativelib-1.0.dist-info/RECORD,,\n", + encoding="utf-8", + ) + (native_info / "METADATA").write_text("Name: nativelib\n", encoding="utf-8") + pure_info = site / "Pure_Pkg-1.0.dist-info" + pure_info.mkdir() + (pure_info / "RECORD").write_text("purepkg/__init__.py,,\nPure_Pkg-1.0.dist-info/RECORD,,\n", encoding="utf-8") + (pure_info / "METADATA").write_text("Name: Pure_Pkg\n", encoding="utf-8") + return site + + +def test_stage_site_packages_copies_pure_trees_and_metadata(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + site = _fake_site(tmp_path / "site") + monkeypatch.setattr("pyscriptc.imports._site_packages", lambda: [site]) + from pyscriptc import imports + + imports._installed_distributions.cache_clear() + lib_dir = tmp_path / "lib" + staged = sitedeps.stage_site_packages(lib_dir, "3.14", ["purepkg"]) + dest = lib_dir / "python3.14" / "site-packages" + assert (dest / "purepkg" / "__init__.py").is_file() + assert (dest / "Pure_Pkg-1.0.dist-info" / "RECORD").is_file() + assert staged + + +def test_stage_site_packages_refuses_native_with_ps4302(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + site = _fake_site(tmp_path / "site") + monkeypatch.setattr("pyscriptc.imports._site_packages", lambda: [site]) + from pyscriptc import imports + + imports._installed_distributions.cache_clear() + native, _ = imports.distribution_is_native("nativelib") + assert native, "the fake site must mark nativelib native" + with pytest.raises(sitedeps.SiteError, match=r"PS4302.*nativelib"): + sitedeps.stage_site_packages(tmp_path / "lib", "3.14", ["nativepkg"]) + + +def test_check_pure_refuses_native_before_staging(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + site = _fake_site(tmp_path / "site") + monkeypatch.setattr("pyscriptc.imports._site_packages", lambda: [site]) + from pyscriptc import imports + + imports._installed_distributions.cache_clear() + native, _ = imports.distribution_is_native("nativelib") + assert native, "the fake site must mark nativelib native" + with pytest.raises(sitedeps.SiteError, match="PS4302"): + sitedeps.check_pure(["nativepkg"]) + + +def test_closure_walks_requirements(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + site = _fake_site(tmp_path / "site") + monkeypatch.setattr("pyscriptc.imports._site_packages", lambda: [site]) + monkeypatch.setattr( + "pyscriptc.sitedeps.distributions_of", + lambda root: {"purepkg": ["Pure_Pkg"], "puremod": ["Pure_Pkg"]}.get(root, []), + ) + from importlib import metadata + + def _requires(dist): + return ["puremod>=1"] if dist == "Pure_Pkg" else [] + + monkeypatch.setattr(metadata, "requires", _requires, raising=False) + resolved = sitedeps._closure(["purepkg"]) + assert "purepkg" in resolved["Pure_Pkg"] + assert resolved # the requirement walk reached puremod's import root too + + +def test_record_roots_reads_a_dist_info_record(tmp_path: Path) -> None: + record = tmp_path / "RECORD" + record.write_text( + "purepkg/__init__.py,,\npurepkg-1.0.dist-info/METADATA,,\nsix.py,,\n", + encoding="utf-8", + ) + assert sitedeps._record_roots(record) == {"purepkg", "purepkg-1.0.dist-info", "six"} + + +@pytest.mark.skipif(sysconfig.get_path("purelib") is None, reason="needs an installed interpreter") +def test_check_pure_accepts_this_venv_s_pure_deps() -> None: + # `packaging` (a pyscriptc dependency) is pure; if it ever grows a wheel + # extension this test flips and the refusal is the honest answer. + sitedeps.check_pure(["packaging"]) diff --git a/tests/test_stdlibzip.py b/tests/test_stdlibzip.py new file mode 100644 index 00000000..75da90af --- /dev/null +++ b/tests/test_stdlibzip.py @@ -0,0 +1,126 @@ +"""The bundled stdlib of a runtime pack: safe-trim zip, manifest check, minimal trim. + +Refs #659. The unit tests here build a fake stdlib tree and never touch the +network; the real-fetch measurements live in the commit message and +`tests/test_embed.py`'s pack-gated end-to-end test. +""" + +from __future__ import annotations + +import importlib.util +import shutil +import sys +import zipfile +from pathlib import Path + +import pytest + +from pyscriptc import stdlibzip + + +def _stdlib(tmp_path: Path) -> Path: + """A small fake stdlib tree with one trimmed package and one module.""" + root = tmp_path / "lib" / "python3.14" + (root / "json").mkdir(parents=True) + (root / "json" / "__init__.py").write_text("value = 1\n", encoding="utf-8") + (root / "json" / "encoder.py").write_text("from json import decoder\nimport fractions\n", encoding="utf-8") + (root / "json" / "decoder.py").write_text("value = 2\n", encoding="utf-8") + (root / "fractions.py").write_text("import numbers\nvalue = 3\n", encoding="utf-8") + (root / "numbers.py").write_text("value = 4\n", encoding="utf-8") + (root / "idlelib").mkdir() + (root / "idlelib" / "__init__.py").write_text("value = 5\n", encoding="utf-8") + (root / "os.py").write_text("import codecs\nvalue = 6\n", encoding="utf-8") + return root + + +def test_safe_trim_drops_the_deny_list_only(tmp_path: Path) -> None: + root = _stdlib(tmp_path) + kept = {path.relative_to(root).as_posix() for path in stdlibzip.safe_trim_modules(root)} + assert "idlelib/__init__.py" not in kept + assert "os.py" in kept and "json/encoder.py" in kept + assert set(stdlibzip.SAFE_TRIM) >= {"test", "idlelib", "tkinter", "turtledemo", "ensurepip"} + + +def test_write_zip_carries_pyc_bodies(tmp_path: Path) -> None: + root = _stdlib(tmp_path) + zip_path = stdlibzip.write_zip(stdlibzip.safe_trim_modules(root), root, tmp_path / "python314.zip") + assert zip_path.is_file() + with zipfile.ZipFile(zip_path) as archive: + names = archive.namelist() + assert "json/__init__.pyc" in names and "fractions.pyc" in names + assert "idlelib/__init__.pyc" not in names + assert stdlibzip.zip_modules(zip_path) >= {"json", "json.encoder", "fractions", "os"} + + +def test_zip_bodies_start_with_the_pyc_magic(tmp_path: Path) -> None: + root = _stdlib(tmp_path) + zip_path = stdlibzip.write_zip([root / "os.py"], root, tmp_path / "python314.zip") + with zipfile.ZipFile(zip_path) as archive: + body = archive.read("os.pyc") + assert body[:4] == importlib.util.MAGIC_NUMBER # the running CPython's + + +def test_kept_module_names_covers_package_levels(tmp_path: Path) -> None: + root = _stdlib(tmp_path) + names = stdlibzip.kept_module_names(stdlibzip.safe_trim_modules(root), root) + assert {"json", "json.encoder", "json.decoder", "fractions", "os"} <= names + assert not any(name.startswith("idlelib") for name in names) + + +def test_manifest_check_passes_when_the_trim_keeps_the_imports(tmp_path: Path) -> None: + carried = {"json", "json.decoder", "fractions", "os"} + stdlibzip.manifest_check(carried, ["json", "json.encoder", "fractions", "os"]) + + +def test_manifest_check_fails_closed_with_ps4301(tmp_path: Path) -> None: + with pytest.raises(stdlibzip.StdlibError, match=r"PS4301.*idlelib"): + stdlibzip.manifest_check({"json"}, ["json", "idlelib"], "safe") + + +def test_stdlib_sources_refuses_an_unknown_mode(tmp_path: Path) -> None: + with pytest.raises(stdlibzip.StdlibError, match="embed-stdlib"): + stdlibzip.stdlib_sources(_stdlib(tmp_path), "profile") + + +def test_stage_stdlib_replaces_the_tree_and_keeps_dynload(tmp_path: Path) -> None: + # The staged lib dir already holds the copied tree (stage_bundle copies + # first, then swaps it for the zip); the pack root next to it is the source. + pack = _stdlib(tmp_path).parents[1] # .../lib/python3.14 -> the pack root + lib_dir = tmp_path / "out" / "lib" + shutil.copytree(pack / "lib" / "python3.14", lib_dir / "python3.14") + stdlibzip.stage_stdlib(lib_dir, pack, "3.14", "safe") + assert (lib_dir / "python314.zip").is_file() + assert (lib_dir / "python3.14" / "lib-dynload").is_dir() + assert not (lib_dir / "python3.14" / "os.py").exists() + + +def test_minimal_modules_is_a_frozen_closure_over_the_host_stdlib(tmp_path: Path) -> None: + modules = stdlibzip.minimal_modules() + names = stdlibzip.kept_module_names(modules) + # The seed travels whole: codecs reach encodings before the first program + # import, and the allowlist's json is kept with it. + assert "os" in names and "codecs" in names and "encodings" in names + assert "encodings.utf_8" in names and "json" in names + # The deny list stays out of the minimal trim too. + assert not any(name.split(".")[0] in stdlibzip.SAFE_TRIM for name in names) + # Frozen: the same tuple answers twice without the interpreter changing. + assert stdlibzip.kept_module_names(stdlibzip.minimal_modules()) == names + + +def test_minimal_closure_keeps_imported_stdlib(tmp_path: Path) -> None: + modules = stdlibzip.minimal_modules() + names = stdlibzip.kept_module_names(modules) + stdlibzip.manifest_check(names, ["logging", "datetime", "json", "argparse", "fractions"], "minimal") + + +@pytest.mark.skipif( + not (Path.home() / ".cache" / "pyscriptc" / "runtime").is_dir() or not sys.version_info[:2] == (3, 14), + reason="needs a fetched runtime pack and a 3.14 host", +) +def test_safe_trim_against_the_fetched_pack() -> None: + from pyscriptc.embed import lock_pack, pack_dir + + root = pack_dir(lock_pack()) / "lib" / "python3.14" + kept = stdlibzip.kept_module_names(stdlibzip.safe_trim_modules(root)) + for wanted in ("json", "fractions", "logging", "datetime", "argparse", "http", "email", "xml"): + assert wanted in kept, f"the safe trim dropped {wanted}"