diff --git a/MANIFEST.in b/MANIFEST.in index 6d1c13aa..63ce2e21 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,13 @@ include traincheck/static_analyzer/func_level/optim_func_level.log -include traincheck/static_analyzer/func_level/nn_func_level.log \ No newline at end of file +include traincheck/static_analyzer/func_level/nn_func_level.log + +# Source distributions must retain the tests' imports and provenance fixtures. +recursive-include tests *.py *.json *.md + +# These are source-checkout utilities, not additional installed wheel packages. +recursive-include tools/benchmarks *.py *.json *.md +recursive-include tools/fresh_validation *.py *.json *.md +recursive-include tools/release *.py *.json *.md + +include docs/releasing.md +include docs/releases/v0.2.0.md diff --git a/docs/releases/v0.2.0.md b/docs/releases/v0.2.0.md new file mode 100644 index 00000000..cb09a15d --- /dev/null +++ b/docs/releases/v0.2.0.md @@ -0,0 +1,69 @@ +# TrainCheck 0.2.0 + +This beta release adds parallel offline inference/checking, accelerates trace +queries, and fixes instrumentation and inference correctness issues. It includes +the reviewed changes from PRs #139–#148, consolidated on main by #151. + +## Highlights + +- Process-based offline inference and checking via `--num-workers`, with + deterministic inference sampling across tested worker counts and schedules. +- Bounded example reservoirs during construction and merging, with validated + worker configuration and accurate represented-population counts. +- Context/call indexes, cached process/thread identities, and indexed time-window + queries for offline invariant inference. +- Exception-safe recorder suppression and atomic buffer delivery to prevent + recorder recursion and duplicate or lost batches. +- Preserved TorchScript builtin mappings and robust dynamic type-name handling. +- Independent pruning state for each precondition solve. +- Typed fresh-run validation utilities with explicit activation, coverage, + unsupported, skipped, and failed states. +- Correct standard console-entrypoint metadata for all four commands. + +## Measured performance + +These are complete offline-inference timings on a saved trace from a 10-step, +6,163,712-parameter TorchTitan Llama debug training run: 573,494 events, batch size +1, sequence length 256. Inference ran on labmach's CPU with one worker/native +thread and six enabled relations. + +| Experiment | Before | After | Scope | +| --- | ---: | ---: | --- | +| Process/thread identity caching | 1,385.656 s | 98.551 s | One baseline-first paired run; 14.06x | +| Time-window indexing | 101.796 s | 76.748 s | Medians across three alternating paired runs; 24.61% less wall time | + +Outputs match within each comparison. The experiments use different corrected +baselines; do not multiply their speedups or claim that either measures training, +trace-collection overhead, distributed scaling, or model quality. +See [identity-cache evidence](../measurements/inference-identity-cache.md) and +[time-window evidence](../measurements/inference-time-range-index.md) for exact +source/input hashes, memory costs, commands, and limitations. + +## Compatibility and limits + +Python 3.10 or newer remains required. Standard pip-installed command names are +unchanged. Packaging checks cover both wheel and source distributions. + +Reservoir-cap and pruning-state corrections can change inferred evidence or +predicates. This release is not solely an output-preserving performance update. +Trace indexes assume stable dataframe values, dtypes, and row order; create a new +`TracePandas` after unsupported in-place mutation. + +Fresh-run validation reports use schema version 2, retaining unknown totals as +`null` with explicit observation/error states. The Transformer study still +contains two healthy false alarms associated with embedded object identities. +TorchScript compatibility uses private PyTorch APIs and reports missing support; +it does not establish general Megatron/checkpoint compatibility. + +Benchmark and fresh-validation utilities are available in the repository/source +distribution, not installed as part of the wheel. Legacy untyped bodies are not +all strictly checked. Three unchanged repository-wide formatting issues remain +outside the release-preparation changes. + +## Validation + +The consolidated production/harness stack passed 168 regression tests. Release +preparation additionally checks real installed-package entrypoints in independent +interpreters, negative metadata/location contracts, required static-analysis +tables, source-distribution rebuilds, distribution metadata, and command help. +See [release procedure](../releasing.md). diff --git a/docs/releasing.md b/docs/releasing.md new file mode 100644 index 00000000..3ff41f59 --- /dev/null +++ b/docs/releasing.md @@ -0,0 +1,67 @@ +# Releasing TrainCheck + +Release from a reviewed commit that includes the version in `pyproject.toml`. +The Git tag must be `v` followed by that exact version. Keep tags immutable and +preserve configured commit/tag signing. Publication and merging require the +maintainer authorization applicable to the task. + +## Build and inspect + +Use Python 3.10 or newer and an isolated build environment with `build` and +`twine`. From a clean checkout: + +```sh +python -m build --outdir /absolute/path/to/release-dist +python -m twine check --strict /absolute/path/to/release-dist/* +``` + +The default build creates the source distribution first, then builds the wheel +from it. Inspect both archives: include the two required +`traincheck/static_analyzer/func_level/*_func_level.log` tables and new production +modules; exclude traces, checkpoints, credentials, and local experiment outputs. +The source distribution also includes test fixtures and supporting tools. + +Install the wheel into a separate Python environment, leave the source checkout, +and run the installed-release validator with that environment's Python: + +```sh +python /absolute/path/to/checkout/tools/release/verify_installed.py --expected-version VERSION +python -m pip check +traincheck-collect --help +traincheck-infer --help +traincheck-check --help +traincheck-onlinecheck --help +``` + +The validator checks installed metadata and import location, then loads each +console entrypoint in a fresh isolated interpreter. It intentionally rejects +source/editable installs. Validate an installation rebuilt from the source +distribution too. Run relevant tests and configured checks; state the actual +Python/framework versions and any dependency/environment limitations. + +Record source commit, artifact filenames, SHA-256 hashes, build/runtime versions, +commands, test results, and known limits in the agreed experiment evidence +directory. Publish exactly the inspected files. Version/metadata or included-file +changes require rebuilding; do not silently replace previously published bytes. + +## GitHub and PyPI + +After the reviewed release commit is ready, create a signed annotated tag and +verify it before pushing. Create the GitHub release using the version's notes and +attach the inspected wheel, source archive, and checksums. + +For local PyPI uploads, configure a project-scoped token through the normal Twine +credential mechanism, such as the maintainer's private `~/.pypirc` or an OS +keyring. Never put credentials in this repository, command arguments, logs, or +chat. A private configuration uses a `[pypi]` section with +`username = __token__` and its token in `password`. + +```sh +python -m twine upload --non-interactive /absolute/path/to/release-dist/* +``` + +Do not retry an uncertain upload blindly. Query +`https://pypi.org/pypi/traincheck/VERSION/json` and compare both filenames and +SHA-256 digests with the recorded artifacts. Confirm the GitHub tag points to the +intended commit and attached asset hashes match. A successful local build or +GitHub release alone does not establish PyPI publication. diff --git a/pyproject.toml b/pyproject.toml index fa95fc77..c5b4e7da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ include = ["traincheck", "traincheck.*"] [project] requires-python = ">= 3.10" name = "traincheck" -version = "0.1.2" +version = "0.2.0" description = "Training with Confidence: Catching Silent DL Training Bugs with Automated Proactive Checks" authors = [ { name = "Yuxuan Jiang", email = "jyuxuan@umich.edu" }, @@ -53,7 +53,7 @@ classifiers = [ Homepage = "https://github.com/OrderLab/TrainCheck" [project.scripts] -traincheck-collect = "traincheck:collect_trace.main" -traincheck-infer = "traincheck:infer_engine.main" -traincheck-check = "traincheck:checker.main" -traincheck-onlinecheck = "traincheck:checker_online.main" +traincheck-collect = "traincheck.collect_trace:main" +traincheck-infer = "traincheck.infer_engine:main" +traincheck-check = "traincheck.checker:main" +traincheck-onlinecheck = "traincheck.checker_online:main" diff --git a/tests/test_release_installation.py b/tests/test_release_installation.py new file mode 100644 index 00000000..e84a9167 --- /dev/null +++ b/tests/test_release_installation.py @@ -0,0 +1,74 @@ +"""Negative metadata contracts for the installed-release validator.""" + +import importlib.metadata +import tempfile +import unittest +from pathlib import Path +from unittest.mock import PropertyMock, patch + +from tools.release.verify_installed import console_entries, verify_installed + + +class InstalledReleaseValidationTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary_directory.cleanup) + self.prefix = Path(self.temporary_directory.name).resolve() + metadata = self.prefix / "traincheck-0.2.0.dist-info" + metadata.mkdir() + (metadata / "METADATA").write_text( + "Metadata-Version: 2.1\nName: traincheck\nVersion: 0.2.0\n" + ) + self.metadata = metadata + self.distribution = importlib.metadata.Distribution.at(metadata) + + def test_missing_console_command_is_rejected(self) -> None: + (self.metadata / "entry_points.txt").write_text( + "[console_scripts]\ntraincheck-infer = traincheck.infer_engine:main\n" + ) + with self.assertRaisesRegex(ValueError, "Unexpected console command set"): + console_entries(self.distribution) + + def test_missing_entrypoint_metadata_is_rejected(self) -> None: + with self.assertRaisesRegex(ValueError, "Unexpected console command set"): + console_entries(self.distribution) + + def test_duplicate_command_is_rejected(self) -> None: + entry = importlib.metadata.EntryPoint( + name="traincheck-infer", + value="traincheck.infer_engine:main", + group="console_scripts", + ) + # Distribution providers can expose repeated entries even when the + # filesystem metadata parser rejects duplicate INI keys itself. + with patch.object( + type(self.distribution), + "entry_points", + new_callable=PropertyMock, + return_value=importlib.metadata.EntryPoints([entry, entry]), + ): + with self.assertRaisesRegex(ValueError, "Duplicate console entry point"): + console_entries(self.distribution) + + def test_wrong_installed_version_is_rejected(self) -> None: + with patch( + "tools.release.verify_installed.importlib.metadata.distribution", + return_value=self.distribution, + ): + with self.assertRaisesRegex(ValueError, "Expected traincheck 9.9.9"): + verify_installed("9.9.9") + + def test_package_outside_environment_is_rejected(self) -> None: + package = self.prefix / "traincheck" + package.mkdir() + (package / "__init__.py").write_text("") + with patch( + "tools.release.verify_installed.importlib.metadata.distribution", + return_value=self.distribution, + ): + with self.assertRaisesRegex(ValueError, "outside the selected Python"): + verify_installed("0.2.0") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/release/verify_installed.py b/tools/release/verify_installed.py new file mode 100644 index 00000000..e032d96a --- /dev/null +++ b/tools/release/verify_installed.py @@ -0,0 +1,146 @@ +"""Validate an installed release in its Python environment, outside the checkout. + +Run this file with the Python executable from a venv containing the candidate +wheel. Source/editable installations deliberately fail the prefix check. +""" + +import argparse +import importlib.metadata +import json +import subprocess +import sys +from pathlib import Path +from typing import TypedDict + +CONSOLE_COMMANDS = frozenset( + { + "traincheck-collect", + "traincheck-infer", + "traincheck-check", + "traincheck-onlinecheck", + } +) + + +class CommandCheck(TypedDict): + name: str + target: str + + +class VerificationReport(TypedDict): + version: str + environment_prefix: str + package_path: str + commands: list[CommandCheck] + + +# Each command gets a separate interpreter: an earlier command's imports must +# not make a malformed dotted attribute path appear loadable. +_LOAD_ENTRY_POINT = """ +import importlib.metadata +from pathlib import Path +import sys + +distribution = importlib.metadata.distribution("traincheck") +name = sys.argv[1] +expected_version = sys.argv[2] +expected_package = Path(sys.argv[3]).resolve() +if distribution.version != expected_version: + raise RuntimeError("Child process resolved an unexpected package version") +matches = [ + entry for entry in distribution.entry_points + if entry.group == "console_scripts" and entry.name == name +] +if len(matches) != 1: + raise RuntimeError(f"Expected one console entry point for {name!r}") +entry = matches[0] +loaded = entry.load() +if not callable(loaded): + raise TypeError(f"Console entry point {name!r} did not load a callable") +import traincheck +module_file = traincheck.__file__ +if not isinstance(module_file, str): + raise TypeError("Installed traincheck has no filesystem module path") +if Path(module_file).resolve() != expected_package: + raise RuntimeError(f"Unexpected imported package location: {module_file}") +""" + + +def console_entries( + distribution: importlib.metadata.Distribution, +) -> dict[str, importlib.metadata.EntryPoint]: + entries: dict[str, importlib.metadata.EntryPoint] = {} + for entry in distribution.entry_points: + if entry.group != "console_scripts": + continue + if entry.name in entries: + raise ValueError(f"Duplicate console entry point: {entry.name!r}") + entries[entry.name] = entry + if entries.keys() != CONSOLE_COMMANDS: + raise ValueError( + "Unexpected console command set: " + f"expected {sorted(CONSOLE_COMMANDS)!r}, found {sorted(entries)!r}" + ) + return entries + + +def verify_installed(expected_version: str) -> VerificationReport: + distribution = importlib.metadata.distribution("traincheck") + if distribution.version != expected_version: + raise ValueError( + f"Expected traincheck {expected_version}, found {distribution.version}" + ) + prefix = Path(sys.prefix).resolve() + # Distribution.locate_file returns a SimplePath; its string representation + # supplies the filesystem path for this installed-wheel-only validator. + package_path = Path(str(distribution.locate_file("traincheck/__init__.py"))) + package_path = package_path.resolve(strict=True) + if not package_path.is_relative_to(prefix): + raise ValueError( + f"Package is outside the selected Python environment: {package_path}" + ) + commands: list[CommandCheck] = [] + for name, entry in sorted(console_entries(distribution).items()): + result = subprocess.run( + [ + sys.executable, + "-I", + "-B", + "-c", + _LOAD_ENTRY_POINT, + name, + expected_version, + str(package_path), + ], + cwd=prefix, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + if result.returncode != 0: + raise RuntimeError( + f"Console entry point {name!r} failed in a fresh interpreter " + f"(exit {result.returncode}).\n{result.stdout}{result.stderr}" + ) + commands.append({"name": name, "target": entry.value}) + return { + "version": distribution.version, + "environment_prefix": str(prefix), + "package_path": str(package_path), + "commands": commands, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--expected-version", required=True) + args = parser.parse_args() + expected_version: object = args.expected_version + if not isinstance(expected_version, str) or not expected_version: + parser.error("--expected-version must be a nonempty string") + print(json.dumps(verify_installed(expected_version), indent=2)) + + +if __name__ == "__main__": + main()