From 2f034ae5d0fab7e792b7c0a5bdc6a1acfeb8962c Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 18:41:58 +0800 Subject: [PATCH 1/4] fix: validate dynamic type-name metadata before fallback --- .../dynamic-type-names-results.json | 25 +++++ docs/experiments/dynamic-type-names.md | 43 ++++++++ tests/test_dynamic_typename.py | 100 ++++++++++++++++++ traincheck/utils.py | 2 +- 4 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 docs/experiments/dynamic-type-names-results.json create mode 100644 docs/experiments/dynamic-type-names.md create mode 100644 tests/test_dynamic_typename.py diff --git a/docs/experiments/dynamic-type-names-results.json b/docs/experiments/dynamic-type-names-results.json new file mode 100644 index 00000000..b77bb34a --- /dev/null +++ b/docs/experiments/dynamic-type-names-results.json @@ -0,0 +1,25 @@ +{ + "baseline_commit": "9ac61d1140cf8679c24442a174ec12f4de069980", + "python": "3.10.16", + "torch": "2.2.2", + "platform": "macOS-27.0.1-arm64-arm-64bit", + "method": "Identical seven-test suite, candidate module globals, exact parent typename function extracted by AST versus candidate function.", + "before": { + "tests": 7, + "failures": 3, + "errors": 0, + "passed": 4, + "failed_tests": [ + "test_dynamic_typename.DynamicTypenameTests.test_dynamic_callable_metadata_uses_class_name", + "test_dynamic_typename.DynamicTypenameTests.test_non_string_name_values_fall_back_without_truth_testing", + "test_dynamic_typename.DynamicTypenameTests.test_real_pytorch_dynamic_ops_wrapper" + ] + }, + "after": { + "tests": 7, + "failures": 0, + "errors": 0, + "passed": 7, + "failed_tests": [] + } +} diff --git a/docs/experiments/dynamic-type-names.md b/docs/experiments/dynamic-type-names.md new file mode 100644 index 00000000..3b93cd47 --- /dev/null +++ b/docs/experiments/dynamic-type-names.md @@ -0,0 +1,43 @@ +# Dynamic type-name metadata + +## Problem and change + +PyTorch `torch._inductor.virtualized.ops` returns callable values for dynamic +`__qualname__` and `__name__` attributes. TrainCheck rejected the first value but +accepted the second, then asserted that its type name was a string. A fresh +MNIST trace-collection attempt failed before training reached its first step. + +Validate `__name__` as a nonempty string before using it. Otherwise, use the +existing reported-class fallback. This does not call dynamic metadata, truth-test +non-string values, change valid name precedence, or replace the reported +`__class__` with the object's actual type. + +## Validation + +An identical seven-test suite was run against the exact parent `typename` +function extracted from commit `9ac61d1140cf8679c24442a174ec12f4de069980` and against +the changed function, with identical module globals. Python 3.10.16 / PyTorch +2.2.2 on macOS arm64: + +| Result | Parent | Changed | +| --- | ---: | ---: | +| Tests passed | 4 | 7 | +| Tests failed | 3 | 0 | +| Test errors | 0 | 0 | + +The three previously failing tests cover callable dynamic metadata, non-string +names (including an unsafe truth-value implementation), and the real PyTorch +`OpsWrapper`. The remaining tests preserve qualified/simple names, reported-class +proxies, functions, descriptors, tensors, and parameters. Both instrumentation +and runtime modes are exercised for dynamic metadata. + +Reproduce the candidate suite with: + +```sh +python -m unittest tests.test_dynamic_typename -v +``` + +The motivating Linux training run used an existing broader temporary workaround. +Its successful collection is not evidence of an end-to-end run of this narrower +production change; the direct real-object regression test is the evidence here. +This is a compatibility fix, with no performance improvement claim. diff --git a/tests/test_dynamic_typename.py b/tests/test_dynamic_typename.py new file mode 100644 index 00000000..d5f96194 --- /dev/null +++ b/tests/test_dynamic_typename.py @@ -0,0 +1,100 @@ +"""Runtime type names must tolerate dynamic metadata without calling it.""" + +import unittest + +import torch + +from traincheck.utils import typename + + +class DynamicMetadata: + __module__ = "fixtures.dynamic" + + def __getattr__(self, name): + if name in {"__qualname__", "__name__"}: + return lambda: None + raise AttributeError(name) + + +class UnsafeTruthValue: + def __bool__(self): + raise AssertionError("Metadata truth value must not be evaluated") + + +class VisibleType: + pass + + +class ReportedClass: + __module__ = "fixtures.dynamic" + + @property + def __class__(self): + return VisibleType + + +class DynamicTypenameTests(unittest.TestCase): + def test_dynamic_callable_metadata_uses_class_name(self): + for runtime in [False, True]: + self.assertEqual( + typename(DynamicMetadata(), is_runtime=runtime), + "fixtures.dynamic.DynamicMetadata", + ) + + def test_non_string_name_values_fall_back_without_truth_testing(self): + for value in [object(), 17, ["name"], UnsafeTruthValue()]: + + class Metadata: + __module__ = "fixtures.dynamic" + + def __getattr__(self, name): + if name == "__name__": + return value + raise AttributeError(name) + + for runtime in [False, True]: + self.assertEqual( + typename(Metadata(), is_runtime=runtime), + "fixtures.dynamic.Metadata", + ) + + def test_valid_string_names_still_take_precedence(self): + class Metadata: + __module__ = "fixtures.dynamic" + + def __getattr__(self, name): + if name == "__qualname__": + return "Outer.call" + if name == "__name__": + raise AssertionError("Qualified name should take precedence") + raise AttributeError(name) + + self.assertEqual(typename(Metadata()), "fixtures.dynamic.Outer.call") + + def test_valid_unqualified_name_is_preserved(self): + value = DynamicMetadata() + value.__name__ = "exported" + self.assertEqual(typename(value), "fixtures.dynamic.exported") + + def test_reported_class_fallback_is_preserved(self): + self.assertEqual(typename(ReportedClass()), "fixtures.dynamic.VisibleType") + + def test_standard_functions_descriptors_tensors_and_parameters(self): + self.assertEqual(typename(len), "len") + self.assertEqual(typename(str.upper), "str.upper") + tensor = torch.zeros(1) + self.assertEqual(typename(tensor), tensor.type()) + self.assertEqual(typename(torch.nn.Parameter(tensor)), "torch.nn.Parameter") + + def test_real_pytorch_dynamic_ops_wrapper(self): + from torch._inductor.virtualized import ops + + self.assertEqual(typename(ops), "torch._inductor.virtualized.OpsWrapper") + self.assertEqual( + typename(ops, is_runtime=True), + "torch._inductor.virtualized.OpsWrapper", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/traincheck/utils.py b/traincheck/utils.py index 332cfe04..f62971fd 100644 --- a/traincheck/utils.py +++ b/traincheck/utils.py @@ -68,7 +68,7 @@ def typename(o, is_runtime=False): if not last_name: last_name = safe_getattr(o, "__name__", "") is_class_name_qualname = False - if not last_name: + if not isinstance(last_name, str) or not last_name: last_name = safe_getattr(o, "__class__", type(o)).__name__ is_class_name_qualname = False # Handle the typename logic for method descriptors. The `is_class_name_qualname` flag is used as a heuristic to determine if `o` is likely a method descriptor. From e673266115b4e809a2392f2edaaf0dbd96ae1fe6 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 19:01:26 +0800 Subject: [PATCH 2/4] Document dynamic type-name validation on matching Linux runtime --- .../dynamic-type-names-results.json | 27 +++++++++++++++++++ docs/experiments/dynamic-type-names.md | 15 +++++++++++ 2 files changed, 42 insertions(+) diff --git a/docs/experiments/dynamic-type-names-results.json b/docs/experiments/dynamic-type-names-results.json index b77bb34a..b785b380 100644 --- a/docs/experiments/dynamic-type-names-results.json +++ b/docs/experiments/dynamic-type-names-results.json @@ -21,5 +21,32 @@ "errors": 0, "passed": 7, "failed_tests": [] + }, + "linux_followup": { + "fix_commit": "2f034ae5d0fab7e792b7c0a5bdc6a1acfeb8962c", + "utils_sha256": "854c11519e7aa6d0cbbfd79694d2ad4895fb1aaf72e9efbb0068048598b70750", + "runtime": { + "python": "3.11.4 (main, Jul 5 2023, 13:45:01) [GCC 11.2.0]", + "torch": "2.16.0.dev20261012+cpu" + }, + "ops_wrapper_names": { + "instrumentation": "torch._inductor.virtualized.OpsWrapper", + "runtime": "torch._inductor.virtualized.OpsWrapper" + }, + "tests": { + "v3-integration-tests": { + "tests_passed": 29, + "exit_code": 0, + "source_unchanged": true, + "stdout_sha256": "be7f9c929651de7f09f383a8f3b0176ec8d3285f0ad053f4d49e6e837a40309b" + }, + "v3-coverage-tests": { + "tests_passed": 11, + "exit_code": 0, + "source_unchanged": true, + "stdout_sha256": "a33575d7feb6117b55c84c6919d21911e1f76065db033ebbdd9270671c536239" + } + }, + "scope": "Exact PR146 utility in composed source; seven type-name tests included in 29 integration tests. No training or timing claim from these preflight results." } } diff --git a/docs/experiments/dynamic-type-names.md b/docs/experiments/dynamic-type-names.md index 3b93cd47..cfe34ca9 100644 --- a/docs/experiments/dynamic-type-names.md +++ b/docs/experiments/dynamic-type-names.md @@ -41,3 +41,18 @@ The motivating Linux training run used an existing broader temporary workaround. Its successful collection is not evidence of an end-to-end run of this narrower production change; the direct real-object regression test is the evidence here. This is a compatibility fix, with no performance improvement claim. + +## Matching Linux runtime follow-up + +The exact signed utility change was also tested on Python 3.11.4 and PyTorch +2.16.0.dev20261012+cpu. All seven type-name regressions passed within a **29-test +integration suite** (also covering trace integrity, TorchScript, and precondition +state isolation). A separate 11-test checker-observer suite passed. The real +`OpsWrapper` returned `torch._inductor.virtualized.OpsWrapper` in both modes. + +The disposable integration source combined the signed tracing, TorchScript, +cache, sampling, state-isolation, and type-name changes. Its `utils.py` matched +commit `2f034ae5d0fab7e792b7c0a5bdc6a1acfeb8962c` exactly, and all source hashes +were unchanged after testing. Compact evidence includes runtime versions and +log/source hashes. These preflight results do not establish training overhead +or cross-workload invariant generalization. From 2333682cb2f3f48d338ff27a6d70682f0f4e4702 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 19:09:35 +0800 Subject: [PATCH 3/4] Record successful collection with the narrow type-name fix --- .../experiments/dynamic-type-names-results.json | 17 ++++++++++++++++- docs/experiments/dynamic-type-names.md | 17 +++++++++++++---- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/docs/experiments/dynamic-type-names-results.json b/docs/experiments/dynamic-type-names-results.json index b785b380..ab69fc52 100644 --- a/docs/experiments/dynamic-type-names-results.json +++ b/docs/experiments/dynamic-type-names-results.json @@ -47,6 +47,21 @@ "stdout_sha256": "a33575d7feb6117b55c84c6919d21911e1f76065db033ebbdd9270671c536239" } }, - "scope": "Exact PR146 utility in composed source; seven type-name tests included in 29 integration tests. No training or timing claim from these preflight results." + "scope": "Exact PR146 utility in composed source; seven type-name tests included in 29 integration tests. No training or timing claim from these preflight results.", + "collection_smoke": { + "workload": "Bounded benchmark MNIST Net / cached real MNIST / batch2 / CPUfloat32 / seed42", + "completed_steps": 10, + "success": true, + "source_unchanged": true, + "state_shapes_unchanged": true, + "source_stack": "PR144 plus PR142/143/145/146 production fixes; immutable integration-v3, signed74b harness", + "evidence_sha256": { + "phase-result.json": "03a50887287fb8e98398735e738412edd6c356bcc21dc39ee15929d5ffcf324c", + "metrics.json": "d99a6fd45dc534964835937a8a623ad98309052a3771e9750c3f8dfb0bfe0713", + "adaptation.json": "8a533afe8ea099fbb255124ba589f2edf068f1c17d5d3774d73c0ff88a63fa8e", + "invocation.json": "8ecea3da01502bff67024e8aa303f3d7e6ab72e7a7d1191b3aef53d61868fc75" + }, + "scope": "Collection completed with exact narrow PR146 utility in composed stack. Separate later shape-only fault-selector failure does not affect this collection result. No fix-isolated training timing claim." + } } } diff --git a/docs/experiments/dynamic-type-names.md b/docs/experiments/dynamic-type-names.md index cfe34ca9..2a7ab3e6 100644 --- a/docs/experiments/dynamic-type-names.md +++ b/docs/experiments/dynamic-type-names.md @@ -37,10 +37,10 @@ Reproduce the candidate suite with: python -m unittest tests.test_dynamic_typename -v ``` -The motivating Linux training run used an existing broader temporary workaround. -Its successful collection is not evidence of an end-to-end run of this narrower -production change; the direct real-object regression test is the evidence here. -This is a compatibility fix, with no performance improvement claim. +The first successful Linux training pilot used an existing broader temporary +workaround. That pilot is not evidence for this narrower production change; the +follow-up below uses the exact signed fix. This is a compatibility fix, with no +performance improvement claim. ## Matching Linux runtime follow-up @@ -56,3 +56,12 @@ commit `2f034ae5d0fab7e792b7c0a5bdc6a1acfeb8962c` exactly, and all source hashes were unchanged after testing. Compact evidence includes runtime versions and log/source hashes. These preflight results do not establish training overhead or cross-workload invariant generalization. + +The composed source also completed a fresh **10-step MNIST trace collection** +with cached real MNIST, batch size 2, CPU float32, and seed 42. Its source hashes +and model-state shapes stayed unchanged. This supports collection compatibility +in the identified stack; it does not isolate training behavior to this one fix. +The study later stopped at a shape-only negative-control selector because no +shape invariant existed, after collection and healthy checks had succeeded. +That harness issue and its preserved evidence are separate from the type-name +fix. Evidence hashes for the successful collection are included in the JSON. From b64023b0651278ab987d3276aaad9141e8bf4894 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Sun, 25 Oct 2026 21:37:29 +0800 Subject: [PATCH 4/4] Type dynamic name contracts and retain failure assertions --- docs/experiments/dynamic-type-names.md | 34 ++++++++++++++ tests/test_dynamic_typename.py | 65 ++++++++++++++++++++------ traincheck/utils.py | 2 +- 3 files changed, 85 insertions(+), 16 deletions(-) diff --git a/docs/experiments/dynamic-type-names.md b/docs/experiments/dynamic-type-names.md index 2a7ab3e6..a6df3f75 100644 --- a/docs/experiments/dynamic-type-names.md +++ b/docs/experiments/dynamic-type-names.md @@ -65,3 +65,37 @@ The study later stopped at a shape-only negative-control selector because no shape invariant existed, after collection and healthy checks had succeeded. That harness issue and its preserved evidence are separate from the type-name fix. Evidence hashes for the successful collection are included in the JSON. + +## Contract hardening (2026-10-25) + +`typename` now declares its existing `object`/`bool` inputs and string result. +The dynamic-name guards, reported-class fallback, and final string assertions +are unchanged. All dynamic-name test helpers now have explicit annotations. +Two additional tests preserve the negative contract: an invalid fallback class +name still fails the final assertion, and an unexpected metadata exception +propagates unchanged. This does not change `safe_getattr`'s pre-existing framework +compatibility exception handling. + +The composed branch includes the signed #142 and #143 hardening checkpoints. +Validation uses Python 3.10.16/PyTorch 2.2.2 with one OpenMP/MKL/OpenBLAS thread: + +```sh +python -m unittest discover -s tests -p 'test_dynamic_typename.py' +python -m unittest discover -s tests -p 'test_torchscript*.py' +python -m unittest discover -s tests -p 'test_trac*.py' +pre-commit run --files traincheck/utils.py tests/test_dynamic_typename.py traincheck/instrumentor/tracer.py traincheck/instrumentor/dumper.py traincheck/instrumentor/torchscript_compat.py tests/test_torchscript_compat.py tests/test_torchscript_contracts.py tests/test_tracing_contracts.py +mypy --follow-imports=silent --ignore-missing-imports --check-untyped-defs --disallow-untyped-defs --disallow-any-generics --warn-return-any traincheck/instrumentor/torchscript_compat.py tests/test_dynamic_typename.py tests/test_torchscript_contracts.py tests/test_tracing_contracts.py +``` + +All 37 tests passed: 9 dynamic-name, 13 TorchScript (including six fresh-process +controls), and 15 tracing tests. All pinned hooks and the stronger scoped check +passed using mypy 1.9. The existing general utility/serialization modules are +not claimed to be strictly typed in full; see the #143 evidence for the remaining +legacy tracer/dumper diagnostics. No runtime benchmark was rerun for this +annotation/contract update, so earlier performance numbers retain their original +revision and workload scope. + +With `--check-untyped-defs`, `utils.py` reports the same single pre-existing +`threading.__excepthook__` stub diagnostic at both original head +`2333682cb2f3f48d338ff27a6d70682f0f4e4702` and this update; no new diagnostic +was introduced by the `typename` annotation. diff --git a/tests/test_dynamic_typename.py b/tests/test_dynamic_typename.py index d5f96194..1ff4d61a 100644 --- a/tests/test_dynamic_typename.py +++ b/tests/test_dynamic_typename.py @@ -10,14 +10,14 @@ class DynamicMetadata: __module__ = "fixtures.dynamic" - def __getattr__(self, name): + def __getattr__(self, name: str) -> object: if name in {"__qualname__", "__name__"}: return lambda: None raise AttributeError(name) class UnsafeTruthValue: - def __bool__(self): + def __bool__(self) -> bool: raise AssertionError("Metadata truth value must not be evaluated") @@ -28,26 +28,28 @@ class VisibleType: class ReportedClass: __module__ = "fixtures.dynamic" - @property - def __class__(self): - return VisibleType + def __getattribute__(self, name: str) -> object: + if name == "__class__": + return VisibleType + attribute: object = super().__getattribute__(name) + return attribute class DynamicTypenameTests(unittest.TestCase): - def test_dynamic_callable_metadata_uses_class_name(self): + def test_dynamic_callable_metadata_uses_class_name(self) -> None: for runtime in [False, True]: self.assertEqual( typename(DynamicMetadata(), is_runtime=runtime), "fixtures.dynamic.DynamicMetadata", ) - def test_non_string_name_values_fall_back_without_truth_testing(self): + def test_non_string_name_values_fall_back_without_truth_testing(self) -> None: for value in [object(), 17, ["name"], UnsafeTruthValue()]: class Metadata: __module__ = "fixtures.dynamic" - def __getattr__(self, name): + def __getattr__(self, name: str) -> object: if name == "__name__": return value raise AttributeError(name) @@ -58,11 +60,11 @@ def __getattr__(self, name): "fixtures.dynamic.Metadata", ) - def test_valid_string_names_still_take_precedence(self): + def test_valid_string_names_still_take_precedence(self) -> None: class Metadata: __module__ = "fixtures.dynamic" - def __getattr__(self, name): + def __getattr__(self, name: str) -> object: if name == "__qualname__": return "Outer.call" if name == "__name__": @@ -71,22 +73,55 @@ def __getattr__(self, name): self.assertEqual(typename(Metadata()), "fixtures.dynamic.Outer.call") - def test_valid_unqualified_name_is_preserved(self): + def test_valid_unqualified_name_is_preserved(self) -> None: value = DynamicMetadata() - value.__name__ = "exported" + setattr(value, "__name__", "exported") self.assertEqual(typename(value), "fixtures.dynamic.exported") - def test_reported_class_fallback_is_preserved(self): + def test_reported_class_fallback_is_preserved(self) -> None: self.assertEqual(typename(ReportedClass()), "fixtures.dynamic.VisibleType") - def test_standard_functions_descriptors_tensors_and_parameters(self): + def test_standard_functions_descriptors_tensors_and_parameters(self) -> None: self.assertEqual(typename(len), "len") self.assertEqual(typename(str.upper), "str.upper") tensor = torch.zeros(1) self.assertEqual(typename(tensor), tensor.type()) self.assertEqual(typename(torch.nn.Parameter(tensor)), "torch.nn.Parameter") - def test_real_pytorch_dynamic_ops_wrapper(self): + def test_invalid_reported_class_name_retains_final_assertion(self) -> None: + class InvalidName: + def __getattribute__(self, name: str) -> object: + if name == "__module__": + return "fixtures.dynamic" + if name in {"__qualname__", "__name__"}: + return None + if name == "__class__": + return InvalidClassName() + raise AttributeError(name) + + class InvalidClassName: + __name__ = 17 + + for runtime in (False, True): + with self.assertRaisesRegex( + AssertionError, "prefix and last_name should be str" + ): + typename(InvalidName(), is_runtime=runtime) + + def test_unexpected_metadata_failure_propagates(self) -> None: + error = ValueError("metadata unavailable") + + class FailingMetadata: + def __getattr__(self, name: str) -> object: + if name == "__qualname__": + raise error + raise AttributeError(name) + + with self.assertRaises(ValueError) as caught: + typename(FailingMetadata()) + self.assertIs(caught.exception, error) + + def test_real_pytorch_dynamic_ops_wrapper(self) -> None: from torch._inductor.virtualized import ops self.assertEqual(typename(ops), "torch._inductor.virtualized.OpsWrapper") diff --git a/traincheck/utils.py b/traincheck/utils.py index f62971fd..30de83bc 100644 --- a/traincheck/utils.py +++ b/traincheck/utils.py @@ -47,7 +47,7 @@ def typename_compile(o): return "compile_stage" -def typename(o, is_runtime=False): +def typename(o: object, is_runtime: bool = False) -> str: if isinstance(o, torch.nn.Parameter): return "torch.nn.Parameter" if isinstance(o, torch.Tensor):