From 77ea19be77578fb68c654ea861f05a1a7c6267dc Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Tue, 27 Oct 2026 17:36:02 +0800 Subject: [PATCH 1/6] Skip provably discarded native tensor method metadata --- tests/test_native_metadata_contracts.py | 224 ++++++++++++++++++++ tests/test_native_metadata_methods.py | 192 +++++++++++++++++ tools/benchmarks/tensor_metadata.py | 90 ++++++++ traincheck/instrumentor/dumper.py | 5 + traincheck/instrumentor/metadata_methods.py | 150 +++++++++++++ 5 files changed, 661 insertions(+) create mode 100644 tests/test_native_metadata_contracts.py create mode 100644 tests/test_native_metadata_methods.py create mode 100644 tools/benchmarks/tensor_metadata.py create mode 100644 traincheck/instrumentor/metadata_methods.py diff --git a/tests/test_native_metadata_contracts.py b/tests/test_native_metadata_contracts.py new file mode 100644 index 00000000..0035145e --- /dev/null +++ b/tests/test_native_metadata_contracts.py @@ -0,0 +1,224 @@ +"""Adversarial lookup and configuration contracts for tensor method skipping.""" + +import json +import subprocess +import sys +import textwrap +import types +import unittest +from contextlib import ExitStack +from unittest.mock import patch + +import torch + +from traincheck.instrumentor import dumper + + +class NativeMetadataContractTests(unittest.TestCase): + def setUp(self) -> None: + contexts = ExitStack() + self.addCleanup(contexts.close) + for name in ( + "skip_attrs_due_to_errs", + "skip_type_due_to_errs", + "skip_type_due_to_recursion", + ): + contexts.enter_context(patch.object(dumper, name, {})) + + def test_parameter_metaclass_properties_are_never_evaluated(self) -> None: + visits: list[str] = [] + + def namespace(owner: object) -> object: + visits.append("namespace") + raise AssertionError("Metaclass namespace property must not execute") + + def inheritance(owner: object) -> object: + visits.append("inheritance") + raise AssertionError("Metaclass MRO property must not execute") + + original = type(torch.nn.Parameter) + replacement: type[object] = type( + "SpoofedParameterMeta", + (original,), + {"__dict__": property(namespace), "__mro__": property(inheritance)}, + ) + value = torch.nn.Parameter(torch.ones(1)) + try: + setattr(torch.nn.Parameter, "__class__", replacement) + result = dumper.convert_var_to_dict( + value, False, {"backward": None, "requires_grad": None} + ) + finally: + setattr(torch.nn.Parameter, "__class__", original) + self.assertEqual(result, {"requires_grad": True}) + self.assertEqual(visits, []) + + def test_changed_parameter_inheritance_observes_new_descriptor(self) -> None: + # Restoring __bases__ creates another MRO tuple; isolate this global class + # mutation so later tests still exercise the ordinary Parameter fast path. + fixture = textwrap.dedent( + """ import json + import torch + from unittest.mock import patch + from traincheck.instrumentor import dumper + + class ReplacementBase(torch.Tensor): + pass + + def replacement(value: torch.Tensor) -> int: + return 37 + + setattr(ReplacementBase, "add", property(replacement)) + value = torch.nn.Parameter(torch.ones(1)) + original: object = type.__getattribute__(torch.nn.Parameter, "__bases__") + try: + setattr(torch.nn.Parameter, "__bases__", (ReplacementBase,)) + with patch.object( + dumper, "is_discardable_tensor_method", return_value=False + ): + baseline = dumper.convert_var_to_dict(value, False, {"add": None}) + result = dumper.convert_var_to_dict(value, False, {"add": None}) + finally: + setattr(torch.nn.Parameter, "__bases__", original) + print(json.dumps({"baseline": baseline, "candidate": result})) + """ + ) + completed = subprocess.run( + [sys.executable, "-c", fixture], + check=False, + capture_output=True, + text=True, + timeout=30, + ) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual( + json.loads(completed.stdout), + {"baseline": {"add": 37}, "candidate": {"add": 37}}, + ) + + def test_instance_dictionary_key_equality_matches_baseline(self) -> None: + visits: list[object] = [] + + class CollidingKey: + def __hash__(self) -> int: + return hash("add") + + def __eq__(self, other: object) -> bool: + visits.append(other) + return False + + value = torch.ones(1) + storage: object = object.__getattribute__(value, "__dict__") + assert isinstance(storage, dict) + dict.__setitem__(storage, CollidingKey(), 1) + with patch.object(dumper, "is_discardable_tensor_method", return_value=False): + expected = dumper.convert_var_to_dict(value, False, {"add": None}) + expected_visits = list(visits) + visits.clear() + actual = dumper.convert_var_to_dict(value, False, {"add": None}) + self.assertEqual(actual, expected) + self.assertTrue(expected_visits) + self.assertTrue(all(name == "add" for name in expected_visits)) + self.assertEqual(visits, expected_visits) + + def test_earlier_getter_can_replace_lookup_hook(self) -> None: + visits: list[str] = [] + + def lookup(value: torch.Tensor, name: str) -> object: + visits.append(name) + if name == "add": + return 41 + return object.__getattribute__(value, name) + + def prepare(value: torch.Tensor) -> int: + setattr(torch.Tensor, "__getattribute__", lookup) + return 1 + + value = torch.ones(1) + with patch.object(torch.Tensor, "__getattribute__", object.__getattribute__): + with patch.object(torch.Tensor, "prepare", property(prepare), create=True): + result = dumper.convert_var_to_dict( + value, False, {"prepare": None, "add": None} + ) + self.assertEqual(result, {"prepare": 1, "add": 41}) + self.assertEqual(visits, ["add"]) + + def test_replaced_method_failure_retains_backoff(self) -> None: + visits: list[str] = [] + + def failure(value: torch.Tensor) -> object: + visits.append("add") + raise ValueError("metadata unavailable") + + value = torch.ones(1) + with patch.object(torch.Tensor, "add", property(failure)): + first = dumper.convert_var_to_dict(value, False, {"add": None}) + second = dumper.convert_var_to_dict(value, False, {"add": None}) + self.assertEqual(first, {}) + self.assertEqual(second, {}) + self.assertEqual(visits, ["add"]) + self.assertIn("add", dumper.skip_attrs_due_to_errs[str(torch.Tensor)]) + + def test_retained_type_set_subclass_preserves_membership_calls(self) -> None: + visits: list[object] = [] + + class ObservedTypes(set[type[object]]): + def __contains__(self, item: object) -> bool: + visits.append(item) + return super().__contains__(item) + + retained = ObservedTypes(dumper.primitive_types) + retained.add(types.BuiltinFunctionType) + value = torch.ones(1) + with patch.object(dumper, "primitive_types", retained): + with patch.object( + dumper, "is_discardable_tensor_method", return_value=False + ): + expected = dumper.convert_var_to_dict(value, False, {"add": None}) + expected_visits = list(visits) + visits.clear() + actual = dumper.convert_var_to_dict(value, False, {"add": None}) + self.assertEqual(actual, expected) + self.assertEqual(expected_visits, [types.BuiltinFunctionType]) + self.assertEqual(visits, expected_visits) + + def test_nonempty_receiver_preserves_primitive_metaclass_comparisons(self) -> None: + visits: list[object] = [] + + class CollidingMeta(type): + def __hash__(cls) -> int: + return hash(types.BuiltinFunctionType) + + def __eq__(cls, other: object) -> bool: + visits.append(other) + return False + + class ConfiguredType(metaclass=CollidingMeta): + pass + + retained: set[type[object]] = set(dumper.primitive_types) + retained.add(ConfiguredType) + value = torch.ones(1) + value.__dict__["unrelated"] = 1 + visits.clear() + with patch.object(dumper, "primitive_types", retained): + with patch.object( + dumper, "is_discardable_tensor_method", return_value=False + ): + expected = dumper.convert_var_to_dict(value, False, {"add": None}) + expected_visits = list(visits) + visits.clear() + actual = dumper.convert_var_to_dict(value, False, {"add": None}) + self.assertEqual(actual, expected) + self.assertTrue(expected_visits) + self.assertTrue( + all( + value_type is types.BuiltinFunctionType + for value_type in expected_visits + ) + ) + self.assertEqual(visits, expected_visits) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_native_metadata_methods.py b/tests/test_native_metadata_methods.py new file mode 100644 index 00000000..6ee5dcf8 --- /dev/null +++ b/tests/test_native_metadata_methods.py @@ -0,0 +1,192 @@ +"""Tensor metadata fast paths preserve dynamic lookup and warning contracts.""" + +import types +import unittest +import warnings +from concurrent.futures import ThreadPoolExecutor +from contextlib import ExitStack +from unittest.mock import patch + +import torch + +from traincheck.instrumentor import dumper + + +class TensorSubclass(torch.Tensor): + def __getattribute__(self, name: str) -> object: + if name == "add": + return 91 + return super().__getattribute__(name) + + +class NativeMetadataMethodTests(unittest.TestCase): + def setUp(self) -> None: + contexts = ExitStack() + self.addCleanup(contexts.close) + for name in ( + "skip_attrs_due_to_errs", + "skip_type_due_to_errs", + "skip_type_due_to_recursion", + ): + contexts.enter_context(patch.object(dumper, name, {})) + + def test_exact_tensors_preserve_values_identity_order_and_state(self) -> None: + for value in (torch.ones(2, 3), torch.nn.Parameter(torch.ones(2, 3))): + with self.subTest(receiver=type(value)): + stable = torch.zeros(1) + value.__dict__["payload"] = stable + config: dict[str, object] = { + "add": None, + "dtype": None, + "backward": None, + "shape": None, + "payload": None, + "requires_grad": None, + } + rng_before = torch.get_rng_state().clone() + state_before = value.detach().clone() + with patch.object( + dumper, "is_discardable_tensor_method", return_value=False + ): + expected = dumper.convert_var_to_dict(value, False, config) + actual = dumper.convert_var_to_dict(value, False, config) + self.assertEqual(actual, expected) + self.assertEqual(list(actual), list(expected)) + self.assertEqual(actual["_TRAINCHECK_payload_ID"], id(stable)) + self.assertTrue(torch.equal(rng_before, torch.get_rng_state())) + self.assertTrue(torch.equal(state_before, value)) + + def test_instance_shadow_is_live_across_conversions(self) -> None: + value = torch.ones(1) + config: dict[str, object] = {"add": None} + self.assertEqual(dumper.convert_var_to_dict(value, False, config), {}) + value.__dict__["add"] = 12 + self.assertEqual(dumper.convert_var_to_dict(value, False, config), {"add": 12}) + value.__dict__["add"] = 13 + self.assertEqual(dumper.convert_var_to_dict(value, False, config), {"add": 13}) + del value.__dict__["add"] + self.assertEqual(dumper.convert_var_to_dict(value, False, config), {}) + + def test_property_can_shadow_a_later_method(self) -> None: + def prepare(value: torch.Tensor) -> int: + value.__dict__["add"] = 17 + return 1 + + with patch.object(torch.Tensor, "prepare", property(prepare), create=True): + value = torch.ones(1) + self.assertEqual( + dumper.convert_var_to_dict( + value, False, {"prepare": None, "add": None} + ), + {"prepare": 1, "add": 17}, + ) + + def test_property_can_replace_a_later_class_method(self) -> None: + def prepare(value: torch.Tensor) -> int: + setattr(torch.Tensor, "add", 19) + return 1 + + # The outer patch restores the original method even if conversion fails. + with patch.object(torch.Tensor, "add", torch.Tensor.add): + with patch.object(torch.Tensor, "prepare", property(prepare), create=True): + self.assertEqual( + dumper.convert_var_to_dict( + torch.ones(1), False, {"prepare": None, "add": None} + ), + {"prepare": 1, "add": 19}, + ) + + def test_custom_getattribute_and_subclasses_retain_lookup(self) -> None: + def lookup(value: torch.Tensor, name: str) -> object: + if name == "add": + return 23 + return object.__getattribute__(value, name) + + value = torch.ones(1) + with patch.object(torch.Tensor, "__getattribute__", lookup): + self.assertEqual( + dumper.convert_var_to_dict(value, False, {"add": None}), {"add": 23} + ) + subclass = value.as_subclass(TensorSubclass) + self.assertEqual( + dumper.convert_var_to_dict(subclass, False, {"add": None}), {"add": 91} + ) + + def test_foreign_descriptor_keeps_failed_attribute_backoff(self) -> None: + with patch.object(torch.Tensor, "foreign", str.upper, create=True): + self.assertEqual( + dumper.convert_var_to_dict(torch.ones(1), False, {"foreign": None}), {} + ) + self.assertIn("foreign", dumper.skip_attrs_due_to_errs[str(torch.Tensor)]) + + def test_named_traincheck_callable_is_retained(self) -> None: + # Any substring match is retained by the existing metadata classifier. + name = "prefix_TRAINCHECK_callback" + with patch.object(torch.Tensor, name, torch.Tensor.add, create=True): + value = torch.ones(1) + result = dumper.convert_var_to_dict(value, False, {name: None}) + self.assertEqual(list(result), [name]) + callback = result[name] + self.assertIsInstance(callback, types.BuiltinFunctionType) + assert isinstance(callback, types.BuiltinFunctionType) + self.assertIs(callback.__self__, value) + + def test_retained_callable_types_are_not_skipped(self) -> None: + value = torch.ones(1) + retained = dumper.primitive_types | { + types.BuiltinFunctionType, + types.MethodType, + } + with patch.object(dumper, "primitive_types", retained): + result = dumper.convert_var_to_dict( + value, False, {"add": None, "backward": None} + ) + self.assertEqual(list(result), ["add", "backward"]) + for callback in result.values(): + self.assertIsInstance( + callback, (types.BuiltinFunctionType, types.MethodType) + ) + + def test_property_warning_mutation_does_not_leak_to_next_attribute(self) -> None: + visits: list[str] = [] + + def first(value: torch.Tensor) -> object: + visits.append("first") + warnings.warn("first", UserWarning) + warnings.simplefilter("error", RuntimeWarning) + return len + + def later(value: torch.Tensor) -> int: + visits.append("later") + warnings.warn("later", RuntimeWarning) + return 29 + + with patch.object(torch.Tensor, "first", property(first), create=True): + with patch.object(torch.Tensor, "later", property(later), create=True): + with warnings.catch_warnings(record=True) as observed: + warnings.simplefilter("always") + filters = list(warnings.filters) + result = dumper.convert_var_to_dict( + torch.ones(1), + False, + {"first": None, "add": None, "later": None}, + ) + self.assertEqual(warnings.filters, filters) + self.assertEqual(observed, []) + self.assertEqual(result, {"later": 29}) + self.assertEqual(visits, ["first", "later"]) + + def test_concurrent_receivers_do_not_share_shadow_state(self) -> None: + def convert(index: int) -> dict[str, object]: + value = torch.ones(1) + if index % 2: + value.__dict__["add"] = index + return dumper.convert_var_to_dict(value, False, {"add": None}) + + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(convert, range(20))) + self.assertEqual(results, [{"add": i} if i % 2 else {} for i in range(20)]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/benchmarks/tensor_metadata.py b/tools/benchmarks/tensor_metadata.py new file mode 100644 index 00000000..54d735d1 --- /dev/null +++ b/tools/benchmarks/tensor_metadata.py @@ -0,0 +1,90 @@ +"""Unprofiled full tensor metadata conversion, with a separate getter count. + +Run this same script with PYTHONPATH pointing at each frozen source revision. +This is a component benchmark; it does not measure training or GPU overhead. +""" + +import argparse +import hashlib +import json +import platform +import time +from pathlib import Path +from unittest.mock import patch + +import torch + +from traincheck.instrumentor import dumper + + +def run(iterations: int) -> dict[str, object]: + if iterations < 1: + raise ValueError("iterations must be positive") + torch.set_num_threads(1) + tensors = [torch.arange(256, dtype=torch.float32).reshape(16, 16)] + tensors.append(torch.nn.Parameter(tensors[0].clone())) + before = [value.detach().clone() for value in tensors] + rng_before = torch.get_rng_state().clone() + for value in tensors: + dumper.convert_var_to_dict(value, include_tensor_data=False) + + fields = 0 + start = time.perf_counter() + for _ in range(iterations): + for value in tensors: + fields += len(dumper.convert_var_to_dict(value, include_tensor_data=False)) + seconds = time.perf_counter() - start + + # Diagnostics are outside the ordinary timing and do not mutate source. + getter_calls = 0 + original_getter = dumper.safe_getattr + + def count_getter(obj: object, name: str) -> object: + nonlocal getter_calls + getter_calls += 1 + return original_getter(obj, name) + + with patch.object(dumper, "safe_getattr", count_getter): + for value in tensors: + dumper.convert_var_to_dict(value, include_tensor_data=False) + + if not torch.equal(rng_before, torch.get_rng_state()): + raise AssertionError("Metadata conversion changed the torch RNG") + for value, saved in zip(tensors, before): + if not torch.equal(value, saved): + raise AssertionError("Metadata conversion changed tensor values") + source = Path(dumper.__file__).resolve() + return { + "kind": "component_not_training", + "python": platform.python_version(), + "torch": torch.__version__, + "platform": platform.platform(), + "device": "cpu", + "torch_threads": torch.get_num_threads(), + "iterations": iterations, + "conversions": iterations * len(tensors), + "seconds": seconds, + "recorded_fields": fields, + "separate_diagnostic_getter_calls": getter_calls, + "diagnostic_conversions": len(tensors), + "rng_and_tensor_values_unchanged": True, + "dumper_path": str(source), + "dumper_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "limitations": [ + "Recorded field counts alone do not establish metadata equivalence.", + "No claim about end-to-end training or GPU speedup.", + "No isolated memory cost measurement.", + ], + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--iterations", type=int, default=1000) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + args.output.write_text(json.dumps(run(args.iterations), indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/traincheck/instrumentor/dumper.py b/traincheck/instrumentor/dumper.py index c8d7ce3a..39812eb6 100644 --- a/traincheck/instrumentor/dumper.py +++ b/traincheck/instrumentor/dumper.py @@ -17,6 +17,7 @@ RECURSION_ERR_THRESHOLD, TYPE_ERR_THRESHOLD, ) +from traincheck.instrumentor.metadata_methods import is_discardable_tensor_method # if torch.cuda.is_available(): from traincheck.instrumentor.proxy_wrapper.hash import tensor_hash @@ -425,6 +426,10 @@ def convert_var_to_dict( ): continue + # Verified ordinary method binding has no metadata to retain. Properties, + # shadowed methods, and custom lookup retain per-attribute warning guards. + if is_discardable_tensor_method(var, attr_name, primitive_types): + continue attr = safe_getattr(var, attr_name) if attr is NOT_FOUND: if not ( diff --git a/traincheck/instrumentor/metadata_methods.py b/traincheck/instrumentor/metadata_methods.py new file mode 100644 index 00000000..380f9b25 --- /dev/null +++ b/traincheck/instrumentor/metadata_methods.py @@ -0,0 +1,150 @@ +"""Recognize native tensor method bindings discarded by metadata conversion.""" + +import types +from collections.abc import Mapping + +import torch + +_TYPE_NAMESPACE: Mapping[str, object] = vars(type) +_OBJECT_GETATTRIBUTE = vars(object)["__getattribute__"] +_ABSENT = object() +_TENSOR_TYPE = torch.Tensor +_PARAMETER_TYPE = torch.nn.Parameter + + +def _type_descriptors() -> ( + tuple[types.MemberDescriptorType, types.GetSetDescriptorType] +): + mro = _TYPE_NAMESPACE["__mro__"] + namespace = _TYPE_NAMESPACE["__dict__"] + assert isinstance(mro, types.MemberDescriptorType) + assert isinstance(namespace, types.GetSetDescriptorType) + return mro, namespace + + +_MRO_DESCRIPTOR, _NAMESPACE_DESCRIPTOR = _type_descriptors() + + +def _class_layout( + cls: type[object], +) -> tuple[tuple[object, ...], tuple[Mapping[object, object], ...]]: + # Type's native descriptors bypass custom metaclass properties as well as + # custom __getattribute__. These mapping proxies remain live as attributes + # change. A changed MRO is detected by identity before using any proxy. + mro: object = _MRO_DESCRIPTOR.__get__(cls) + assert isinstance(mro, tuple) + namespaces: list[Mapping[object, object]] = [] + for owner in mro: + assert isinstance(owner, type) + namespace: object = _NAMESPACE_DESCRIPTOR.__get__(owner) + assert isinstance(namespace, types.MappingProxyType) + namespaces.append(namespace) + return mro, tuple(namespaces) + + +_TENSOR_MRO, _TENSOR_NAMESPACES = _class_layout(_TENSOR_TYPE) +_PARAMETER_MRO, _PARAMETER_NAMESPACES = _class_layout(_PARAMETER_TYPE) + + +def _native_tensor_dictionary() -> types.GetSetDescriptorType | None: + # Absence/replacement before import disables this optional fast path. A + # genuine native descriptor remains valid for its owner after a user replaces + # the public __dict__ attribute; invoking it reads actual storage directly. + candidate = _TENSOR_NAMESPACES[0].get("__dict__", _ABSENT) + candidate_type = type(candidate) + if candidate_type is not types.GetSetDescriptorType: + return None + assert isinstance(candidate, types.GetSetDescriptorType) + if candidate.__name__ == "__dict__" and candidate.__objclass__ is _TENSOR_TYPE: + return candidate + return None + + +_TENSOR_DICTIONARY = _native_tensor_dictionary() + + +def is_discardable_tensor_method( + receiver: object, name: str, retained_types: set[type[object]] +) -> bool: + """Whether normal lookup can only bind an otherwise discarded method. + + This is deliberately restricted to exact Tensor/Parameter instances with + empty native dictionaries. Subclasses, custom lookup, altered inheritance, + instance state and non-method descriptors retain the original getter path. + Each decision rereads current namespaces and dictionary storage, including + after an earlier getter mutates the receiver or its class. No application + property, descriptor or lookup hook is invoked while deciding. + + ``retained_types`` is the converter's live primitive-type configuration: a + bound method promoted to primitive data must still be recorded. Names with + TrainCheck metadata anywhere in them are likewise never discarded. + """ + name_type = type(name) + retained_set_type = type(retained_types) + if name_type is not str or "_TRAINCHECK" in name or retained_set_type is not set: + return False + receiver_type = type(receiver) + if receiver_type is _TENSOR_TYPE: + mro, namespaces = _TENSOR_MRO, _TENSOR_NAMESPACES + elif receiver_type is _PARAMETER_TYPE: + mro, namespaces = _PARAMETER_MRO, _PARAMETER_NAMESPACES + else: + return False + if _TENSOR_DICTIONARY is None or _MRO_DESCRIPTOR.__get__(receiver_type) is not mro: + return False + + getattribute: object = _ABSENT + descriptor: object = _ABSENT + for namespace in namespaces: + if "__getattr__" in namespace: + return False + if getattribute is _ABSENT: + getattribute = namespace.get("__getattribute__", _ABSENT) + if descriptor is _ABSENT: + descriptor = namespace.get(name, _ABSENT) + if getattribute is not _OBJECT_GETATTRIBUTE: + return False + + descriptor_type = type(descriptor) + if descriptor_type is types.FunctionType: + bound_type: type[object] = types.MethodType + elif descriptor_type is types.MethodDescriptorType: + bound_type = types.BuiltinFunctionType + elif descriptor_type is types.WrapperDescriptorType: + bound_type = types.MethodWrapperType + elif descriptor_type is types.ClassMethodDescriptorType: + bound_type = types.BuiltinFunctionType + else: + return False + if descriptor_type is not types.FunctionType: + assert isinstance( + descriptor, + ( + types.MethodDescriptorType, + types.WrapperDescriptorType, + types.ClassMethodDescriptorType, + ), + ) + descriptor_owner = descriptor.__objclass__ + for owner in mro: + if descriptor_owner is owner: + break + else: + # A foreign descriptor such as str.upper would fail to bind, and + # the original getter must retain its failure/backoff behavior. + return False + + state: object = _TENSOR_DICTIONARY.__get__(receiver, receiver_type) + assert isinstance(state, dict) + # Even membership checks on a nonempty dictionary can run user key equality. + # Falling back here preserves lookup multiplicity, shadowing and side effects. + if dict.__len__(state) != 0: + return False + # An ordinary membership test could invoke custom metaclass equality, then + # invoke it again after falling back. Snapshot the exact set without user + # iteration hooks, inspect entries without hashing/comparing them, and leave + # custom metaclass configurations to the original classifier unchanged. + for retained in tuple(retained_types): + if type(retained) is not type or retained is bound_type: + return False + return True From ce1e29fa5a6e62b428e54e0b89472ac9e651e0d1 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Tue, 27 Oct 2026 17:46:16 +0800 Subject: [PATCH 2/6] Record native metadata fast-path CPU measurements and limits --- .../native-tensor-methods-20261027.md | 177 ++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 docs/measurements/native-tensor-methods-20261027.md diff --git a/docs/measurements/native-tensor-methods-20261027.md b/docs/measurements/native-tensor-methods-20261027.md new file mode 100644 index 00000000..09c5e404 --- /dev/null +++ b/docs/measurements/native-tensor-methods-20261027.md @@ -0,0 +1,177 @@ +# Avoiding discarded tensor method metadata + +TrainCheck's metadata converter reads hundreds of tensor methods and then discards +their bound callable values. Each read also enters and exits a warning-suppression +context. The candidate recognizes a narrowly verified ordinary method binding and +skips both operations. Recorded metadata fields, JSON encoding, tensor hashing, +and warning handling for properties remain unchanged. + +## Measured results + +These are ordinary, unprofiled CPU runs on labmach. They are not H100 results or +inference timings. Each primary workload ran three training steps in eight fresh +processes, ordered baseline/candidate/candidate/baseline twice. Four measurements +per revision contribute to each median. + +| Workload and timing scope | Baseline median | Candidate median | Time reduction | +| --- | ---: | ---: | ---: | +| MNIST, API-traced training body | 276.949 ms | 217.152 ms | 21.59% | +| Small Transformer, API-traced training body | 676.312 ms | 516.164 ms | 23.68% | +| 2,000 complete tensor metadata conversions | 1.626168 s | 1.158417 s | 28.76% | + +Reduction is `(baseline - candidate) / baseline`; training-body speedups are +1.275x and 1.310x. Training timings include `start_step` and the training body, +excluding imports, setup, metrics/state evidence, queue drainage and shutdown. +Component timings measure `convert_var_to_dict(..., include_tensor_data=False)`; +they exclude tensor hashing and are not a fraction of training time. + +Raw three-step training-body times in milliseconds, listed by pair number: + +| Pair | MNIST baseline | MNIST candidate | Transformer baseline | Transformer candidate | +| --- | ---: | ---: | ---: | ---: | +| 1 | 277.677 | 217.590 | 679.095 | 515.303 | +| 2 | 276.435 | 214.336 | 675.916 | 519.917 | +| 3 | 276.054 | 216.714 | 675.490 | 517.025 | +| 4 | 277.462 | 217.629 | 676.709 | 509.738 | + +An untimed component diagnostic counted **1,192 to 60 guarded attribute reads** +for two full conversions, a **94.97% reduction**. Both versions retained the same +28 field names per object. All eight timed component runs emitted 56,000 fields. +The separate diagnostic found no emitted warnings and preserved warning-filter +contents/identity, the warning handler, Python/NumPy/Torch RNG and tensor values. +This count does not establish what fraction of the H100 workload is eligible. + +Single-pair secondary runs with **both API and subclass model tracking** enabled +measured MNIST 659.491 to 642.414 ms (2.59% reduction), and Transformer 758.467 to +620.978 ms (18.13%). Each retained 64 VAR records; API counts were 520 and 1,340. +These are preliminary single-pair results, not repeated performance estimates. + +Fallbacks have a cost. A separate eight-repeat helper benchmark measured 84,900 +eligible method reads at 78.556 to 48.702 ms, but 450 property reads at 0.400 to +0.690 ms. Property-heavy or subclass-heavy workloads may benefit less or regress. +The primary training results above include this fallback cost. There is no claim +that warning management has been eliminated, or that full instrumentation now +has low overhead. Against single plain controls, API-traced medians still cost +9.16x to 7.18x for MNIST and 152.14x to 116.11x for the tiny Transformer; these +ratios use one plain run per workload and are not repeated plain comparisons. + +## Sources, workload and environment + +- Baseline: `6dccb47d0a8bc192fbe7cbbea700b66106bc2381` (main). +- Measured candidate: signed `77ea19be77578fb68c654ea861f05a1a7c6267dc`. +- No production dependencies on the open CUDA-hashing, selective-policy, + checking-workload or coverage PRs. The optional H100 experiment composes those + existing study changes separately. +- Lab host: `yuxuan-Precision-3660`, Linux 6.1.0-1027-oem x86-64, Python 3.11.4, + Torch `2.16.0.dev20261012+cpu`, NumPy 2.1.0. One Torch, interop, OpenMP, MKL and + OpenBLAS thread. CUDA hidden; deterministic algorithms enabled; float32. +- Primary mode: API tracing, no model tracker, `use_full_instr=False`, every + training step, no warmup, production tensor-hash configuration. No profiler + or diagnostic wrappers during ordinary timings. +- MNIST: exact benchmark `Net` class, 1,199,882 parameters, first six cached + training examples, batch two, original normalization, NLL/Adadelta(lr=1). +- Transformer: exact benchmark `PositionalEncoding`/`TransformerModel` classes, + 21,052 parameters, two layers, width 32, feed-forward width 64, two heads, + sequence 16, batch two, 98-token WikiText prefix and 60-token vocabulary; + NLL/backward/clipped manual SGD. This is a bounded adaptation, not the full + RNN benchmark script or the 6.2M-parameter TorchTitan Llama experiment. +- Construction/data seed 42; training RNG reset to 43 after setup. Both variants + use the unchanged `tools/fresh_validation/benchmark_workload.py` adapter. + Every run records exact model-definition and dataset SHA-256 values. +- Plain controls import TrainCheck and its global Module setter hook but install + neither API wrappers nor tracked parameters. They are state/loss controls, + not an independently installed framework-only benchmark. + +The benchmark-directory Git HEAD was unavailable, so manifests identify the +actual source files and extracted class definitions by content hashes. Source +snapshots were hashed before and after every process and remained unchanged. +The runs have 90-second, 4 GiB observed-RSS and 256 MiB trace caps. Overall memory +overhead was not isolated; no memory improvement is claimed. + +## Behavior and compatibility + +All 16 primary runs exactly match the recorded plain-control initial/final model +state, gradients, optimizer state (where present), stored module values including +the Transformer's cached mask, Python/NumPy/Torch RNG states, and loss trajectory. +Tensor evidence includes names, dtypes, shapes and SHA-256 of bytes; raw checkpoint +tensors were not archived. Both secondary combined pairs also match their controls. + +Every primary pair has the same API sequence/count map and complete PRE/POST +groups: 520 MNIST and 1,338 Transformer records. Recursive raw comparisons retain +every field and report differences only in timestamps, process/thread/call/object +IDs and `_TRAINCHECK_*_ID` values. Cross-process raw traces are **not byte-equal**; +IDs were not deleted or remapped to claim equality. These results do not prove +cross-run object-identity equivalence or frozen-invariant checker parity. +Same-object regression tests separately exercise recorded identities and order. +Raw API trace sizes also match in all eight runs per workload: 351,073 bytes for +MNIST and 1,014,134 bytes for the Transformer. + +The fast path accepts exact Tensor/Parameter instances only, with empty native +dictionaries and ordinary lookup. It checks live class namespaces, descriptor +ownership, inheritance identity, and retained-type configuration at each decision. +Only original MRO identities and live namespace proxies are cached; object values +and mutable lookup results are not. Changed inheritance disables the shortcut, +including after bases are restored. Properties, subclasses, shadowed attributes, +custom lookup/configuration and `_TRAINCHECK` fields use the existing path. A +getter may mutate later attributes or warning filters without bypassing restoration. + +Validation: + +- 17 new contract tests; 106 focused tests pass on Python 3.10.16 / Torch 2.2.2. +- 24 metadata tests pass on the matching lab Python/Torch runtime. +- Full local suite: 326 passed, two failures independently reproduced on unchanged + main: `test_traincheck_internal_attr_hidden` and `test_inject_eval_loop`. +- Changed-file pinned pre-commit passes. All four new Python files pass stricter + scoped mypy with untyped-body, untyped-definition and generic checks. +- All-files Ruff/mypy/isort pass. All-files Black finds existing trailing + whitespace in `config/config.py` and `instrumentor/proxy_wrapper/hash.py` on + both revisions; hook-created unrelated edits were restored and excluded. + +## Evidence and reproduction + +Durable local evidence: `outputs/metadata-fastpath-20261027/` in the canonical +checkout. Lab evidence: `/home/yuxuan/experiments/metadata-fastpath-20261027/`. +Large traces and machine-specific experiment scripts are intentionally outside +Git. `PROTOCOL.md`, `training_benchmark.py`, `run_comparison.py`, and +`compare_training.py` record the exact bounded training recipe. The downloaded +`training-evidence/` contains every run's manifests, state evidence, timings, +raw traces and strict comparison reports; `component/` contains all component +timings and separate diagnostics. + +| Artifact | SHA-256 | +| --- | --- | +| Baseline source archive | `7aae9422ca4b026c6f4c5056e53c198cb4e2b1acc323fbbdd1a651efdd27e09e` | +| Candidate source archive | `c2e394d2043da3a5a305312f0147c236b2311a99e071bcf20343db63ba8ab4d9` | +| Final training worker | `8a8b351a8baf0a98215461855cc72bfded7dbc94a667f27af789170a2deedda6` | +| Downloaded training evidence archive | `d406868f9c24d598f5d91350f0eb3360ed2cb3cf9713fb20b0e7f0e6a7898381` | + +The committed component benchmark can run against either frozen source tree: + +```sh +PYTHONPATH="$baseline_source" PYTHONDONTWRITEBYTECODE=1 "$python" \ + "$candidate_source/tools/benchmarks/tensor_metadata.py" \ + --iterations 1000 --output baseline.json +PYTHONPATH="$candidate_source" PYTHONDONTWRITEBYTECODE=1 "$python" \ + "$candidate_source/tools/benchmarks/tensor_metadata.py" \ + --iterations 1000 --output candidate.json +``` + +Check the emitted `dumper_path` and SHA-256. Repeat in ABBA order with fresh output +paths. The initial local pilot without explicit `PYTHONPATH` loaded an editable +installation from another checkout; it is retained as `local-component-wrong-import.json` +and excluded from every reported comparison. + +For the full CPU recipe on labmach, use the saved harness in a fresh study copy +with frozen sources and the same prepared plain controls: + +```sh +python=/home/yuxuan/experiments/codex-inference-runtime-20261025/venv/bin/python +"$python" "$study/run_comparison.py" --root "$study" \ + --candidate-commit 77ea19be77578fb68c654ea861f05a1a7c6267dc +``` + +The H100 selective-Llama package is prepared under `gpu/`, with three ordinary +AB/BA pairs and a holdout. It has **not been submitted**: the shared Expanse session +is unavailable. GPU timings, actual eligibility counts, raw trace integrity and +the 722 frozen invariants' activation/results still need validation. CPU gains +must not be applied numerically to the earlier H100 profile. From 287b3a2cb97f28e8f7e5b21210781d9c65541f2d Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Tue, 27 Oct 2026 18:11:00 +0800 Subject: [PATCH 3/6] Support Python 3.12 native MRO descriptors --- tests/test_native_metadata_contracts.py | 23 ++++++++++++++++++++- traincheck/instrumentor/metadata_methods.py | 11 ++++++---- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/tests/test_native_metadata_contracts.py b/tests/test_native_metadata_contracts.py index 0035145e..d9f44071 100644 --- a/tests/test_native_metadata_contracts.py +++ b/tests/test_native_metadata_contracts.py @@ -11,7 +11,7 @@ import torch -from traincheck.instrumentor import dumper +from traincheck.instrumentor import dumper, metadata_methods class NativeMetadataContractTests(unittest.TestCase): @@ -25,6 +25,27 @@ def setUp(self) -> None: ): contexts.enter_context(patch.object(dumper, name, {})) + def test_native_type_metadata_matches_runtime_storage(self) -> None: + # Run on each supported Python runtime: __mro__ changes native descriptor + # representation in CPython 3.12, but its lookup contract is the same. + mro, namespace = metadata_methods._type_descriptors() + self.assertIs(mro.__get__(torch.Tensor), torch.Tensor.__mro__) + self.assertIs(mro.__get__(torch.nn.Parameter), torch.nn.Parameter.__mro__) + tensor_namespace: object = namespace.__get__(torch.Tensor) + self.assertIsInstance(tensor_namespace, types.MappingProxyType) + assert isinstance(tensor_namespace, types.MappingProxyType) + self.assertIs(tensor_namespace["__dict__"], vars(torch.Tensor)["__dict__"]) + + def test_application_descriptor_is_rejected_as_native_type_metadata(self) -> None: + def application_property(owner: object) -> object: + raise AssertionError("Application descriptor must never execute") + + invalid = dict(vars(type)) + invalid["__mro__"] = property(application_property) + with patch.object(metadata_methods, "_TYPE_NAMESPACE", invalid): + with self.assertRaises(AssertionError): + metadata_methods._type_descriptors() + def test_parameter_metaclass_properties_are_never_evaluated(self) -> None: visits: list[str] = [] diff --git a/traincheck/instrumentor/metadata_methods.py b/traincheck/instrumentor/metadata_methods.py index 380f9b25..860b42fc 100644 --- a/traincheck/instrumentor/metadata_methods.py +++ b/traincheck/instrumentor/metadata_methods.py @@ -12,12 +12,15 @@ _PARAMETER_TYPE = torch.nn.Parameter -def _type_descriptors() -> ( - tuple[types.MemberDescriptorType, types.GetSetDescriptorType] -): +def _type_descriptors() -> tuple[ + types.MemberDescriptorType | types.GetSetDescriptorType, + types.GetSetDescriptorType, +]: mro = _TYPE_NAMESPACE["__mro__"] namespace = _TYPE_NAMESPACE["__dict__"] - assert isinstance(mro, types.MemberDescriptorType) + # CPython 3.12 exposes type.__mro__ as a getset descriptor; 3.10/3.11 + # expose a member descriptor. Both read native storage without user hooks. + assert isinstance(mro, (types.MemberDescriptorType, types.GetSetDescriptorType)) assert isinstance(namespace, types.GetSetDescriptorType) return mro, namespace From 1ed7f2375e28f2521b788d1544491046d1f865cd Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Tue, 27 Oct 2026 18:27:04 +0800 Subject: [PATCH 4/6] Checkpoint matched H100 timing and metadata eligibility results --- .../native-tensor-methods-20261027.md | 10 +- .../native-tensor-methods-h100-20261027.md | 145 ++++++++++++++++++ 2 files changed, 150 insertions(+), 5 deletions(-) create mode 100644 docs/measurements/native-tensor-methods-h100-20261027.md diff --git a/docs/measurements/native-tensor-methods-20261027.md b/docs/measurements/native-tensor-methods-20261027.md index 09c5e404..8fa40347 100644 --- a/docs/measurements/native-tensor-methods-20261027.md +++ b/docs/measurements/native-tensor-methods-20261027.md @@ -170,8 +170,8 @@ python=/home/yuxuan/experiments/codex-inference-runtime-20261025/venv/bin/python --candidate-commit 77ea19be77578fb68c654ea861f05a1a7c6267dc ``` -The H100 selective-Llama package is prepared under `gpu/`, with three ordinary -AB/BA pairs and a holdout. It has **not been submitted**: the shared Expanse session -is unavailable. GPU timings, actual eligibility counts, raw trace integrity and -the 722 frozen invariants' activation/results still need validation. CPU gains -must not be applied numerically to the earlier H100 profile. +The subsequent [H100 selective-Llama comparison](native-tensor-methods-h100-20261027.md) +measures three ordinary AB/BA pairs and a holdout independently of these CPU runs. +It also records a Python 3.12 import compatibility fix and separate eligibility +counts. The CPU figures above retain their original measured revision and scope; +they are not extrapolated to the H100 workload. diff --git a/docs/measurements/native-tensor-methods-h100-20261027.md b/docs/measurements/native-tensor-methods-h100-20261027.md new file mode 100644 index 00000000..6625cc80 --- /dev/null +++ b/docs/measurements/native-tensor-methods-h100-20261027.md @@ -0,0 +1,145 @@ +# Native tensor method optimization on H100 + +Ordinary job **55325956** measures the metadata optimization on the native +TorchTitan Llama debug workload. The median paired steady-step reduction is +**17.779% (1.2162x)**. Including cold steps, the complete ten-step training body +takes **12.568243 to 10.764749 seconds, 14.350% less time**. + +This is **selective API trace collection**, with no concurrent online checker +and no model tracker. The one-GPU experiment retains native FSDP but does not +establish multi-GPU scaling. CPU results are reported separately in +[the original measurement](native-tensor-methods-20261027.md). + +## Ordinary training measurements + +Each fresh process trains the same 6,163,712-parameter model with seed 42 for ten +steps, batch one and sequence length 256. Each table entry is the median of +synchronized native steps 3–10. Pair order was fixed before measurement. + +| Pair / order | Baseline | Candidate | Time reduction | Speedup | +| --- | ---: | ---: | ---: | ---: | +| 1 / AB | 1,000.882 ms | 821.738 ms | 17.899% | 1.2180x | +| 2 / BA | 997.812 ms | 821.305 ms | 17.689% | 1.2149x | +| 3 / AB | 998.726 ms | 821.159 ms | 17.779% | 1.2162x | + +Percentages use the baseline duration as denominator. The median paired reduction +is 17.779%; the separate ratio of run medians is 998.726 to 821.305 ms, or 17.765%. +The full-body 14.350% reduction includes steps 1–2. Body timings exclude imports, +initialization, state evidence, outer loop housekeeping, queue drain and shutdown. +Every raw step/phase duration is retained; no profiled run contributes to this table. + +A single plain-control run has a 44.178 ms steady-step median. Relative to it, +the baseline/candidate cost is **22.61x / 18.59x**, or **+2,160.68% / +1,759.07%**. +These plain ratios have only one control repetition. The optimization helps, but +selective collection remains expensive on this small model. + +A separate seed-43, 20-step baseline/candidate holdout has medians over steps 3–20 +of 996.772 to 822.673 ms (17.466% less). This is one validation pair, excluded from +the three-pair performance estimate. + +## Avoided work, measured separately + +Diagnostic job **55325959** counts actual helper decisions during steps 3–10. +Its wrappers retain no tensor references and preserve the original return values +and exceptions. Counters cover both calling threads and reconcile with getters. + +| Scope | Helper calls | Discarded method bindings skipped | Remaining guarded reads | +| --- | ---: | ---: | ---: | +| All threads | 1,940,048 | 1,698,000 (87.52%) | 242,048 | +| Window owner | 1,243,312 | 1,100,304 (88.50%) | 143,008 | +| Other trace-producing thread | 696,736 | 597,696 (85.79%) | 99,040 | + +Exact Tensor receivers account for all 1,698,000 skips, from 1,788,000 helper calls +(94.97% eligible). Other receivers have 152,048 calls and no skips. Exact Parameter +calls are **not observed**; this is not a measured zero eligibility percentage. + +The prior accepted owner-thread profile counted 1,243,312 `safe_getattr` calls. +The matching denominator here is the new **owner-thread** helper count, not the +1,940,048 all-thread count. These are avoided getter/warning-context counts, not +percentages of total runtime. The remaining 242,048 reads keep the original +per-attribute warning restoration. No global warning suppression was introduced. + +The diagnostic perturbs its window by 1.38557x (9.10619 versus 6.57215 seconds) +relative to its ordinary candidate control; those durations are excluded from +performance claims. Both corpora have 416,364 rows, 208,182 complete calls, +142,621,017 bytes and zero raw audit issues. Every per-step/API/event-kind count +and per-step byte total matches. This establishes volume/integrity parity, not +byte equality of identity-bearing records. The collector checks wrapper identity +at restoration but did not record pre-install binding identity; that runtime +assertion remains unobserved rather than being inferred as passed. + +## Application and checking validation + +Independent analysis verifies all nine phase commands, successful exits, ordered +step counts, finite timings, source stability, environment and selective policy. +Every seed-42 run exactly matches the plain control's complete recorded initial +and final model/gradient/buffer/RNG fingerprints, losses, model configuration, +parameter count and parameter types. The seed-43 candidate matches its selective +baseline on the same fields. Tensor evidence records names, shapes, dtypes and +byte hashes; it is not an archived full optimizer-state checkpoint. No recorded +IDs, arrays, hashes or values were normalized to obtain state parity. + +All eight selective phases use the same 722 frozen invariants and identical +compiled policy bytes. Raw trace audits and observed/direct checking of the +baseline/candidate reference and holdout corpora are running separately on +labmach. Invariant activation and checker parity are **pending**, not implied by +the timing/state or diagnostic-volume checks above. + +## Revision, environment and reproduction + +- Baseline: composed `8cc73ab35590caa1a5d7286b16462d045b6d2da0`. +- Candidate: only `dumper.py` and new `metadata_methods.py` from signed + `287b3a2cb97f28e8f7e5b21210781d9c65541f2d`, overlaid on that baseline. + CUDA hashing, selective policy and workload source remain identical. +- TorchTitan: `2210414902d84d1219fd63ab8efeb24375a2a1a1`. +- Expanse, one NVIDIA H100 80GB HBM3; Python 3.12.3, + Torch `2.16.0.dev20261012+cu130`, CUDA 13.0. Eight allocated CPUs, 48 GiB RAM, + Torch/BLAS threads fixed by the saved launcher. Jobs run on allocated compute. +- Ordinary job completed in 4:35; diagnostic in 0:56. Resource accounting is + saved. Peak batch memory covers entire jobs, not isolated metadata overhead; + no per-variant memory improvement is claimed. + +The first job, **55325946**, failed before candidate training because CPython 3.12 +exposes `type.__mro__` as a native getset descriptor rather than the member +descriptor used in 3.10/3.11. The fix accepts both explicit native types while +retaining validation and rejecting application properties. Failure was reproduced +locally before fixing. Afterwards 67 affected tests pass on Python 3.12, 108 +focused tests on Python 3.10, and strict scoped typing/pinned checks pass. The full +local suite has 328 passes and the same two failures reproduced on unchanged main. +The failed job is preserved separately and excluded from all timing estimates. + +Local artifacts are under `outputs/metadata-fastpath-20261027/`. Lab validation +uses its isolated `validation-job-55325956` directory. Full source inventories, +frozen inputs, commands, raw traces, per-step timings, failure logs and validators +remain outside Git; the public report contains no trace or credential payloads. + +| Artifact | SHA-256 | +| --- | --- | +| Composed baseline source archive | `e65d03f567850a94a3fd1c1e5f1870bb97363694803484534854df4ea497e37d` | +| Corrected candidate source archive | `c32333538188e69e8b856e65cb7e4a2bf5090b1d177762e0f1448e1a062a4fa8` | +| Ordinary results archive | `06503d2b2ad0887ffe23950455d8d6f52372a32d10b7392d985f024f394cb311` | +| Diagnostic results archive | `f9369d3d448806554d1e1f07f2df576ba65d688bdfc88fb91495bd29068bf2df` | + +From the local artifact directory: + +```sh +python analyze_gpu_results.py \ + --ordinary gpu/job-55325956/results/ordinary \ + --output gpu/job-55325956/ordinary-analysis.json +python gpu-diagnostic/validate_hit_counts_v2.py \ + --results gpu/job-55325959/results \ + --output gpu/job-55325959/hit-count-validation.json +python gpu-diagnostic/summarize_hit_counts.py \ + --validation gpu/job-55325959/hit-count-validation.json \ + --counter gpu/job-55325959/results/diagnostic/counters/hit-counts.json \ + --prior-profile ../checking-overhead-20261027/final-component-profile-analysis.json \ + --output gpu/job-55325959/hit-count-analysis.json +``` + +The ordinary analyzer has 11 regression tests, including failed/missing phases, +negative speedups, policy/hash changes and malformed/nonfinite data. The counter +and post-download validators test concurrency, exceptions, restoration, empty +windows and provenance failures. New reusable helpers pass strict typing and the +repository's pinned formatting/lint checks. Exact submission commands are saved +in `gpu/README.md` and `gpu-diagnostic/README.md`; they verify packages and enforce +time/trace limits before running the isolated phases. From 37f22e58954da403105c2c5510b7c7b6b3162ee6 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Tue, 27 Oct 2026 18:51:41 +0800 Subject: [PATCH 5/6] Preserve native method binding lifetimes during metadata conversion --- .../native-tensor-methods-20261027.md | 8 +- .../native-tensor-methods-h100-20261027.md | 28 ++- tests/test_metadata_lifetimes.py | 184 ++++++++++++++++++ tests/test_native_metadata_contracts.py | 39 +++- tests/test_native_metadata_methods.py | 4 +- traincheck/instrumentor/dumper.py | 23 ++- traincheck/instrumentor/metadata_methods.py | 83 ++++---- 7 files changed, 310 insertions(+), 59 deletions(-) create mode 100644 tests/test_metadata_lifetimes.py diff --git a/docs/measurements/native-tensor-methods-20261027.md b/docs/measurements/native-tensor-methods-20261027.md index 8fa40347..5a4a6919 100644 --- a/docs/measurements/native-tensor-methods-20261027.md +++ b/docs/measurements/native-tensor-methods-20261027.md @@ -1,4 +1,10 @@ -# Avoiding discarded tensor method metadata +# Historical CPU measurement: skipping tensor method metadata + +**Superseded candidate:** these numbers measure `77ea19b`. Later lifetime tests +found that skipping a bound method changes references observable by subsequent +properties. The revised implementation retains real native binding and the +original classifier, bypassing only the guarded getter. Its fresh measurements +must be reported separately; the figures below are not claims about that revision. TrainCheck's metadata converter reads hundreds of tensor methods and then discards their bound callable values. Each read also enters and exits a warning-suppression diff --git a/docs/measurements/native-tensor-methods-h100-20261027.md b/docs/measurements/native-tensor-methods-h100-20261027.md index 6625cc80..d9bd3076 100644 --- a/docs/measurements/native-tensor-methods-h100-20261027.md +++ b/docs/measurements/native-tensor-methods-h100-20261027.md @@ -1,4 +1,10 @@ -# Native tensor method optimization on H100 +# Historical H100 measurement: skipping tensor method metadata + +**Superseded candidate:** this report measures `287b3a2`. Additional lifetime tests +found that skipping a binding can retain a previous temporary value/type or release +a receiver/function too early. The revised implementation returns the real native +binding and runs the original classifier. It requires fresh performance and trace +validation; none of the numbers below are relabeled as results for the revision. Ordinary job **55325956** measures the metadata optimization on the native TorchTitan Llama debug workload. The median paired steady-step reduction is @@ -80,10 +86,22 @@ byte hashes; it is not an archived full optimizer-state checkpoint. No recorded IDs, arrays, hashes or values were normalized to obtain state parity. All eight selective phases use the same 722 frozen invariants and identical -compiled policy bytes. Raw trace audits and observed/direct checking of the -baseline/candidate reference and holdout corpora are running separately on -labmach. Invariant activation and checker parity are **pending**, not implied by -the timing/state or diagnostic-volume checks above. +compiled policy bytes. All 16 raw/parser audit and checking phases completed on +labmach for the baseline/candidate reference and holdout corpora. Every corpus +has 719 active rules (718 passes and one violation), three dormant rules, and no +errors, unknowns or skips. Observed/direct checker results match for all 722 rules +within each corpus. Cross-variant per-rule outcomes and decision counts match: +114,373 decisions per reference corpus and 218,713 per holdout corpus. + +The shared violation is rule 614, `cuda.stream() always calls +StreamContext.__init__()`, whose frozen child argument contains a literal CUDA +stream handle. Dormant rules 216/276 require a reference worker thread ID; rule +717 has an unequal-thread precondition. Rules were not filtered or reinferred. +Cross-variant complete checker results match structurally for 721/722 rules; +rule 614's failure evidence retains different IDs/timestamps. Raw traces are not +byte-equal. All eight raw/parser audits reconcile records and report zero issues. +The completed validation archive SHA-256 is +`535ea73ffa22b9c00214495d8df8a9cb8b005022509dd55f1ac83956c5fcd2ff`. ## Revision, environment and reproduction diff --git a/tests/test_metadata_lifetimes.py b/tests/test_metadata_lifetimes.py new file mode 100644 index 00000000..a16596cc --- /dev/null +++ b/tests/test_metadata_lifetimes.py @@ -0,0 +1,184 @@ +"""Native metadata bindings preserve observable temporary-object lifetimes.""" + +from __future__ import annotations + +import gc +import sys +import types +import unittest +import weakref +from contextlib import ExitStack +from typing import Literal, TypeAlias +from unittest.mock import patch + +import torch + +from traincheck.instrumentor import dumper + +LifetimeCase: TypeAlias = Literal[ + "previous_value", "previous_type", "receiver_reference", "function_reference" +] + + +def _generic_binding(receiver: object, name: str) -> None: + # A plain function adds no retained call arguments, unlike a mock object. + return None + + +def _lifetime_case(case: LifetimeCase) -> dict[str, object]: + tensor = torch.ones(1) + refs: list[weakref.ReferenceType[object]] = [] + + class Ephemeral: + pass + + def prepare(receiver: object) -> object: + value = Ephemeral() + refs.append(weakref.ref(value)) + return value + + def prepare_class(receiver: object) -> object: + class TemporaryClass: + pass + + refs.append(weakref.ref(TemporaryClass)) + return TemporaryClass() + + def observe_alive(receiver: object) -> bool: + return refs[-1]() is not None + + def observe_class(receiver: object) -> bool: + gc.collect() + return refs[-1]() is not None + + def observe_count(receiver: object) -> int: + return sys.getrefcount(receiver) + + def observe_function(receiver: object) -> bool: + delattr(torch.Tensor, "metadata_lifetime_method") + return refs[-1]() is not None + + def method(receiver: object) -> None: + pass + + attrs: dict[str, object] + config: dict[str, object] + if case == "previous_value": + attrs = { + "metadata_lifetime_prepare": property(prepare), + "metadata_lifetime_observe": property(observe_alive), + } + config = { + "metadata_lifetime_prepare": None, + "add": None, + "metadata_lifetime_observe": None, + } + elif case == "previous_type": + attrs = { + "metadata_lifetime_prepare": property(prepare_class), + "metadata_lifetime_observe": property(observe_class), + } + config = { + "metadata_lifetime_prepare": None, + "add": None, + "metadata_lifetime_observe": None, + } + elif case == "receiver_reference": + attrs = {"metadata_lifetime_observe": property(observe_count)} + config = {"add": None, "metadata_lifetime_observe": None} + else: + assert case == "function_reference" + refs.append(weakref.ref(method)) + attrs = { + "metadata_lifetime_method": method, + "metadata_lifetime_observe": property(observe_function), + } + config = {"metadata_lifetime_method": None, "metadata_lifetime_observe": None} + names = list(attrs) + for name in names: + if hasattr(torch.Tensor, name): + raise RuntimeError(f"Lifetime fixture attribute already exists: {name}") + try: + for name, value in attrs.items(): + setattr(torch.Tensor, name, value) + # Only class storage and the converter's real binding may retain method. + attrs.clear() + del value + del method + return dumper.convert_var_to_dict(tensor, False, config) + finally: + for name in names: + if name in vars(torch.Tensor): + delattr(torch.Tensor, name) + + +class MetadataLifetimeTests(unittest.TestCase): + def setUp(self) -> None: + contexts = ExitStack() + self.addCleanup(contexts.close) + for name in ( + "skip_attrs_due_to_errs", + "skip_type_due_to_errs", + "skip_type_due_to_recursion", + ): + contexts.enter_context(patch.object(dumper, name, {})) + + def _assert_lifetime_parity(self, case: LifetimeCase) -> dict[str, object]: + with patch.object(dumper, "tensor_method_binding", _generic_binding): + expected = _lifetime_case(case) + actual = _lifetime_case(case) + self.assertEqual(actual, expected) + self.assertEqual(list(actual), list(expected)) + return expected + + def test_previous_attribute_is_released_before_later_property(self) -> None: + expected = self._assert_lifetime_parity("previous_value") + self.assertIs(expected["metadata_lifetime_observe"], False) + + def test_previous_attribute_type_is_released_before_later_property(self) -> None: + expected = self._assert_lifetime_parity("previous_type") + self.assertIs(expected["metadata_lifetime_observe"], False) + + def test_bound_method_retains_receiver_during_later_property(self) -> None: + expected = self._assert_lifetime_parity("receiver_reference") + count = expected["metadata_lifetime_observe"] + self.assertIs(type(count), int) + assert isinstance(count, int) + self.assertGreater(count, 0) + + def test_bound_method_retains_removed_function_during_later_property(self) -> None: + expected = self._assert_lifetime_parity("function_reference") + self.assertIs(expected["metadata_lifetime_observe"], True) + + def test_function_instance_get_override_is_not_descriptor_protocol(self) -> None: + calls: list[str] = [] + + def method(receiver: object) -> None: + pass + + def instance_get(*args: object, **kwargs: object) -> int: + calls.append("instance __get__") + return 17 + + setattr(method, "__get__", instance_get) + name = "metadata_lifetime_descriptor" + if hasattr(torch.Tensor, name): + raise RuntimeError(f"Lifetime fixture attribute already exists: {name}") + with patch.object(torch.Tensor, name, method, create=True): + tensor = torch.ones(1) + config: dict[str, object] = {name: None} + with patch.object(dumper, "tensor_method_binding", _generic_binding): + expected = dumper.convert_var_to_dict(tensor, False, config) + actual = dumper.convert_var_to_dict(tensor, False, config) + self.assertEqual(expected, {}) + self.assertEqual(actual, expected) + binding = dumper.tensor_method_binding(tensor, name) + self.assertIsInstance(binding, types.MethodType) + assert isinstance(binding, types.MethodType) + self.assertIs(binding.__func__, method) + self.assertIs(binding.__self__, tensor) + self.assertEqual(calls, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_native_metadata_contracts.py b/tests/test_native_metadata_contracts.py index d9f44071..7a38041b 100644 --- a/tests/test_native_metadata_contracts.py +++ b/tests/test_native_metadata_contracts.py @@ -1,4 +1,4 @@ -"""Adversarial lookup and configuration contracts for tensor method skipping.""" +"""Adversarial lookup and configuration contracts for native tensor method binding.""" import json import subprocess @@ -25,6 +25,31 @@ def setUp(self) -> None: ): contexts.enter_context(patch.object(dumper, name, {})) + def test_native_binding_allocation_failure_keeps_attribute_backoff(self) -> None: + visits: list[str] = [] + + def exhausted(receiver: object, name: str) -> None: + visits.append(name) + raise MemoryError("native binding allocation failed") + + tensor = torch.ones(1) + with patch.object(dumper, "tensor_method_binding", exhausted): + first = dumper.convert_var_to_dict(tensor, False, {"add": None}) + second = dumper.convert_var_to_dict(tensor, False, {"add": None}) + self.assertEqual(first, {}) + self.assertEqual(second, {}) + self.assertEqual(visits, ["add"]) + self.assertIn("add", dumper.skip_attrs_due_to_errs[str(torch.Tensor)]) + + def test_native_binding_internal_assertion_is_not_hidden(self) -> None: + def broken_contract(receiver: object, name: str) -> None: + raise AssertionError("invalid native descriptor") + + with patch.object(dumper, "tensor_method_binding", broken_contract): + with self.assertRaisesRegex(AssertionError, "invalid native descriptor"): + dumper.convert_var_to_dict(torch.ones(1), False, {"add": None}) + self.assertEqual(dumper.skip_attrs_due_to_errs, {}) + def test_native_type_metadata_matches_runtime_storage(self) -> None: # Run on each supported Python runtime: __mro__ changes native descriptor # representation in CPython 3.12, but its lookup contract is the same. @@ -95,7 +120,7 @@ def replacement(value: torch.Tensor) -> int: try: setattr(torch.nn.Parameter, "__bases__", (ReplacementBase,)) with patch.object( - dumper, "is_discardable_tensor_method", return_value=False + dumper, "tensor_method_binding", return_value=None ): baseline = dumper.convert_var_to_dict(value, False, {"add": None}) result = dumper.convert_var_to_dict(value, False, {"add": None}) @@ -132,7 +157,7 @@ def __eq__(self, other: object) -> bool: storage: object = object.__getattribute__(value, "__dict__") assert isinstance(storage, dict) dict.__setitem__(storage, CollidingKey(), 1) - with patch.object(dumper, "is_discardable_tensor_method", return_value=False): + with patch.object(dumper, "tensor_method_binding", return_value=None): expected = dumper.convert_var_to_dict(value, False, {"add": None}) expected_visits = list(visits) visits.clear() @@ -192,9 +217,7 @@ def __contains__(self, item: object) -> bool: retained.add(types.BuiltinFunctionType) value = torch.ones(1) with patch.object(dumper, "primitive_types", retained): - with patch.object( - dumper, "is_discardable_tensor_method", return_value=False - ): + with patch.object(dumper, "tensor_method_binding", return_value=None): expected = dumper.convert_var_to_dict(value, False, {"add": None}) expected_visits = list(visits) visits.clear() @@ -223,9 +246,7 @@ class ConfiguredType(metaclass=CollidingMeta): value.__dict__["unrelated"] = 1 visits.clear() with patch.object(dumper, "primitive_types", retained): - with patch.object( - dumper, "is_discardable_tensor_method", return_value=False - ): + with patch.object(dumper, "tensor_method_binding", return_value=None): expected = dumper.convert_var_to_dict(value, False, {"add": None}) expected_visits = list(visits) visits.clear() diff --git a/tests/test_native_metadata_methods.py b/tests/test_native_metadata_methods.py index 6ee5dcf8..2054917e 100644 --- a/tests/test_native_metadata_methods.py +++ b/tests/test_native_metadata_methods.py @@ -45,9 +45,7 @@ def test_exact_tensors_preserve_values_identity_order_and_state(self) -> None: } rng_before = torch.get_rng_state().clone() state_before = value.detach().clone() - with patch.object( - dumper, "is_discardable_tensor_method", return_value=False - ): + with patch.object(dumper, "tensor_method_binding", return_value=None): expected = dumper.convert_var_to_dict(value, False, config) actual = dumper.convert_var_to_dict(value, False, config) self.assertEqual(actual, expected) diff --git a/traincheck/instrumentor/dumper.py b/traincheck/instrumentor/dumper.py index 39812eb6..fcb9416b 100644 --- a/traincheck/instrumentor/dumper.py +++ b/traincheck/instrumentor/dumper.py @@ -17,7 +17,7 @@ RECURSION_ERR_THRESHOLD, TYPE_ERR_THRESHOLD, ) -from traincheck.instrumentor.metadata_methods import is_discardable_tensor_method +from traincheck.instrumentor.metadata_methods import tensor_method_binding # if torch.cuda.is_available(): from traincheck.instrumentor.proxy_wrapper.hash import tensor_hash @@ -426,11 +426,22 @@ def convert_var_to_dict( ): continue - # Verified ordinary method binding has no metadata to retain. Properties, - # shadowed methods, and custom lookup retain per-attribute warning guards. - if is_discardable_tensor_method(var, attr_name, primitive_types): - continue - attr = safe_getattr(var, attr_name) + # Native bindings need no warning guard, but still replace attr and run + # the original classifier. Retain the real binding's receiver/function + # lifetimes, including while a later property executes. + attr: object + try: + method = tensor_method_binding(var, attr_name) + except MemoryError: + # Native allocation failure has the same backoff as safe_getattr. + # Release any previous binding; keep internal contract failures loud. + method = None + attr = NOT_FOUND + else: + if method is None: + attr = safe_getattr(var, attr_name) + else: + attr = method if attr is NOT_FOUND: if not ( attr_name == "data" diff --git a/traincheck/instrumentor/metadata_methods.py b/traincheck/instrumentor/metadata_methods.py index 860b42fc..f8e22249 100644 --- a/traincheck/instrumentor/metadata_methods.py +++ b/traincheck/instrumentor/metadata_methods.py @@ -1,7 +1,8 @@ -"""Recognize native tensor method bindings discarded by metadata conversion.""" +"""Bind verified native tensor methods without per-attribute warning contexts.""" import types from collections.abc import Mapping +from typing import TypeAlias import torch @@ -11,6 +12,10 @@ _TENSOR_TYPE = torch.Tensor _PARAMETER_TYPE = torch.nn.Parameter +BoundTensorMethod: TypeAlias = ( + types.MethodType | types.BuiltinFunctionType | types.MethodWrapperType +) + def _type_descriptors() -> tuple[ types.MemberDescriptorType | types.GetSetDescriptorType, @@ -66,10 +71,8 @@ def _native_tensor_dictionary() -> types.GetSetDescriptorType | None: _TENSOR_DICTIONARY = _native_tensor_dictionary() -def is_discardable_tensor_method( - receiver: object, name: str, retained_types: set[type[object]] -) -> bool: - """Whether normal lookup can only bind an otherwise discarded method. +def tensor_method_binding(receiver: object, name: str) -> BoundTensorMethod | None: + """Bind a native method, or return None to use the original guarded getter. This is deliberately restricted to exact Tensor/Parameter instances with empty native dictionaries. Subclasses, custom lookup, altered inheritance, @@ -78,47 +81,43 @@ def is_discardable_tensor_method( after an earlier getter mutates the receiver or its class. No application property, descriptor or lookup hook is invoked while deciding. - ``retained_types`` is the converter's live primitive-type configuration: a - bound method promoted to primitive data must still be recorded. Names with - TrainCheck metadata anywhere in them are likewise never discarded. + Return the real binding and leave classification to the caller: even an + unrecorded method keeps its receiver/function alive until the next attribute + replaces it. Skipping the binding would change observable object lifetimes. """ name_type = type(name) - retained_set_type = type(retained_types) - if name_type is not str or "_TRAINCHECK" in name or retained_set_type is not set: - return False + if name_type is not str or "_TRAINCHECK" in name: + return None receiver_type = type(receiver) if receiver_type is _TENSOR_TYPE: mro, namespaces = _TENSOR_MRO, _TENSOR_NAMESPACES elif receiver_type is _PARAMETER_TYPE: mro, namespaces = _PARAMETER_MRO, _PARAMETER_NAMESPACES else: - return False + return None if _TENSOR_DICTIONARY is None or _MRO_DESCRIPTOR.__get__(receiver_type) is not mro: - return False + return None getattribute: object = _ABSENT descriptor: object = _ABSENT for namespace in namespaces: if "__getattr__" in namespace: - return False + return None if getattribute is _ABSENT: getattribute = namespace.get("__getattribute__", _ABSENT) if descriptor is _ABSENT: descriptor = namespace.get(name, _ABSENT) if getattribute is not _OBJECT_GETATTRIBUTE: - return False + return None descriptor_type = type(descriptor) - if descriptor_type is types.FunctionType: - bound_type: type[object] = types.MethodType - elif descriptor_type is types.MethodDescriptorType: - bound_type = types.BuiltinFunctionType - elif descriptor_type is types.WrapperDescriptorType: - bound_type = types.MethodWrapperType - elif descriptor_type is types.ClassMethodDescriptorType: - bound_type = types.BuiltinFunctionType - else: - return False + if ( + descriptor_type is not types.FunctionType + and descriptor_type is not types.MethodDescriptorType + and descriptor_type is not types.WrapperDescriptorType + and descriptor_type is not types.ClassMethodDescriptorType + ): + return None if descriptor_type is not types.FunctionType: assert isinstance( descriptor, @@ -135,19 +134,33 @@ def is_discardable_tensor_method( else: # A foreign descriptor such as str.upper would fail to bind, and # the original getter must retain its failure/backoff behavior. - return False + return None state: object = _TENSOR_DICTIONARY.__get__(receiver, receiver_type) assert isinstance(state, dict) # Even membership checks on a nonempty dictionary can run user key equality. # Falling back here preserves lookup multiplicity, shadowing and side effects. if dict.__len__(state) != 0: - return False - # An ordinary membership test could invoke custom metaclass equality, then - # invoke it again after falling back. Snapshot the exact set without user - # iteration hooks, inspect entries without hashing/comparing them, and leave - # custom metaclass configurations to the original classifier unchanged. - for retained in tuple(retained_types): - if type(retained) is not type or retained is bound_type: - return False - return True + return None + # Call the descriptor TYPE's native slot. A Python function may shadow its + # own __get__ attribute, which normal descriptor lookup never invokes. + binding: object + if isinstance(descriptor, types.FunctionType): + binding = types.FunctionType.__get__(descriptor, receiver, receiver_type) + elif isinstance(descriptor, types.MethodDescriptorType): + binding = types.MethodDescriptorType.__get__( + descriptor, receiver, receiver_type + ) + elif isinstance(descriptor, types.WrapperDescriptorType): + binding = types.WrapperDescriptorType.__get__( + descriptor, receiver, receiver_type + ) + else: + assert isinstance(descriptor, types.ClassMethodDescriptorType) + binding = types.ClassMethodDescriptorType.__get__( + descriptor, receiver, receiver_type + ) + assert isinstance( + binding, (types.MethodType, types.BuiltinFunctionType, types.MethodWrapperType) + ) + return binding From bccc56fab83d525198286f1855b9dd7f6866135f Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Tue, 27 Oct 2026 19:11:46 +0800 Subject: [PATCH 6/6] Record fresh binding-preserving CPU and H100 measurements --- .../native-tensor-bindings-20261027.md | 238 ++++++++++++++++++ .../native-tensor-methods-20261027.md | 2 + .../native-tensor-methods-h100-20261027.md | 2 + 3 files changed, 242 insertions(+) create mode 100644 docs/measurements/native-tensor-bindings-20261027.md diff --git a/docs/measurements/native-tensor-bindings-20261027.md b/docs/measurements/native-tensor-bindings-20261027.md new file mode 100644 index 00000000..e42bc021 --- /dev/null +++ b/docs/measurements/native-tensor-bindings-20261027.md @@ -0,0 +1,238 @@ +# Native tensor bindings without repeated warning contexts + +The final implementation in signed commit `37f22e58954da403105c2c5510b7c7b6b3162ee6` +reduces ordinary traced training time on the measured CPU and H100 workloads. +It preserves native method binding and the existing metadata classifier, while +avoiding the warning-managed getter for verified native tensor methods. JSON +encoding, tensor hashing, instrumentation selection and the checker are unchanged. + +## Fresh ordinary measurements + +| Workload / timing scope | Baseline | Candidate | Time reduction | +| --- | ---: | ---: | ---: | +| CPU MNIST, complete three-step API-traced body | 286.400 ms | 222.665 ms | 22.25% | +| CPU small Transformer, complete three-step API-traced body | 682.324 ms | 527.587 ms | 22.68% | +| CPU, 2,000 complete metadata conversions | 1.607038 s | 1.190048 s | 25.95% | +| H100 Llama, median of steady-step run medians | 999.855 ms | 842.885 ms | 15.699% | + +Percentages divide saved time by baseline time. CPU measurements have four fresh +processes per variant, ordered ABBA twice. Component conversion is separate from +training, starts after one warmup conversion per object and excludes tensor +hashing; its percentage is not a fraction of total training time. CPU training uses full API collection with model tracking disabled. + +H100 job **55326071** uses the same 6,163,712-parameter TorchTitan Llama debug +model, native FSDP, batch one, sequence 256, and one H100. It collects selectively +using the same 722 frozen invariants and 244-API policy. There is no model tracker +or concurrent online checker. Each fresh process trains ten seed-42 steps; steady +statistics use synchronized steps 3–10. Every measured step remains in the evidence. + +| H100 pair / order | Baseline median step | Candidate median step | Time reduction | +| --- | ---: | ---: | ---: | +| 1 / AB | 999.855 ms | 842.430 ms | 15.745% | +| 2 / BA | 1,000.320 ms | 842.885 ms | 15.738% | +| 3 / AB | 998.851 ms | 843.650 ms | 15.538% | + +The **median paired reduction is 15.738% (1.1868x)**; the table's ratio of run +medians is a separate aggregate, 15.699%. Including cold steps 1–2, median complete +ten-step training-body time is **12.579847 to 10.990844 seconds, 12.631% less**. +Timings exclude +imports, setup, state evidence, outer loop housekeeping, queue drain and shutdown. +The separate seed-43, 20-step holdout has medians of 1,002.488 to 842.039 ms over +steps 3–20 (16.005% less); this is one validation pair, not a fourth primary pair. + +The single H100 plain-control run has a 44.456 ms steady median: baseline/candidate +are still 22.49x/18.96x that time. CPU single plain controls give 8.60x/6.69x for +MNIST and 48.05x/37.15x for the tiny Transformer. Plain controls import TrainCheck +but install neither API wrappers nor tracked parameters. These ratios have only +one plain repetition, and the workloads are small; they are not general overhead +estimates. No multi-GPU scaling or concurrent-checker improvement is established. + +## What changed and what remains + +Ordinary metadata conversion visits hundreds of Tensor methods. Their values +usually contribute no output fields, yet generic lookup enters and exits a warning +context for each one. The shortcut verifies exact Tensor/Parameter type, ordinary +attribute lookup, native descriptor ownership, unchanged inheritance and an empty +native instance dictionary. It then calls the descriptor type's native binding +operation. Properties, custom lookup, subclasses, instance shadowing, foreign +descriptors and TrainCheck-specific fields retain the original guarded getter. + +Only original MRO identities and live namespace proxies are cached. Current +namespaces and native dictionary storage are rechecked for each attribute, so +mutable values and descriptor results are not cached. A changed MRO disables the +shortcut, including after bases are restored. The original classifier handles +live primitive-type configuration exactly once. Native allocation failure retains +NOT_FOUND/backoff behavior; internal assertions remain visible. + +The implementation deliberately retains the real bound object. A prior temporary +value and its type must be released at their ordinary points, while the bound +method must keep its receiver/function alive during a later property. Four +regressions demonstrated that the earlier skip-based candidate changed these +behaviors. All four fail on verbatim `287b3a2` sources on Python 3.10 and 3.12 and +pass with the final binding implementation. A fifth regression verifies that a +function's own `__get__` attribute cannot override Python's native descriptor +protocol. Historical CPU/H100 reports remain separately labeled; their faster +skip-based measurements are not reused as results for the final implementation. + +This does not combine warning contexts around arbitrary properties, cache tensor +metadata, change serialization, or introduce asynchronous dumping. The CPU +component diagnostic reduces guarded getters from 1,192 to 60 per two conversions, +while retaining all 28 field names per object. It preserves warning-filter +contents/identity, the warning handler, tensor values and Python/NumPy/Torch RNG. +Those are operation counts, not percentages of runtime attributed to warnings. + +A separate H100 diagnostic, job **55326074**, counted steps 3–10: + +| Scope | Binding helper calls | Native bindings without guarded getter | Remaining guarded getters | +| --- | ---: | ---: | ---: | +| All threads | 1,940,048 | 1,698,000 (87.52%) | 242,048 | +| Window owner | 1,243,312 | 1,100,304 (88.50%) | 143,008 | +| Other trace-producing thread | 696,736 | 597,696 (85.79%) | 99,040 | + +Exact Tensor receivers account for all native bindings: 1,698,000 of 1,788,000 +calls (94.97%). Exact Parameter receivers are unobserved, not a measured zero +eligibility rate. The 152,048 other-receiver calls all use the fallback. Every +successful shortcut still constructs the real method and runs classification. +The count percentages do not attribute a fraction of total runtime to warnings. + +The collector preserves returned objects and exceptions, verifies bindings before +installation and at restoration, and records zero exceptions escaping the binding +helper or guarded getter. Internal NOT_FOUND handling remains unchanged and is +not a separate exception counter. Its counted window takes 1.39489x its ordinary +control's duration; diagnostic +durations do not enter the performance table. Both corpora have 416,364 rows, +208,182 complete calls and 142,621,017 bytes with zero raw audit issues. Every +per-step/API/event-kind count and per-step byte total matches, as do the complete +recorded state and policy. This proves volume/integrity parity, not raw byte equality. + +## State, traces and checking + +All 16 CPU traced runs match fresh plain controls' recorded initial/final model, +gradient, optimizer and stored module state, RNG and losses. Every CPU MNIST run +has 520 API records / 351,073 bytes; every Transformer run has 1,338 records / +1,014,134 bytes. Every call has one PRE and one POST, with no VAR records. All +eight paired raw comparisons preserve complete ordered event sequences, fields, +arrays and duplicates. Remaining differences are identities and timestamps; +raw equality is false and cross-process identity-graph equivalence is not claimed. + +All nine H100 phases pass independent source, environment, command, policy, +complete recorded state and loss checks. Seed-42 phases match the plain control; +the seed-43 candidate matches its selective baseline. GPU tensor evidence includes +names, shapes, dtypes and byte hashes, rather than archived full optimizer-state +checkpoints. No identity, tensor value or array was removed to establish state +parity. + +Raw comparisons of the first seed-42 selective pair and the seed-43 holdout +uniquely match both complete ordered per-thread streams in each pair. Each variant +has 416,364 records / 142,621,017 bytes in the first selective pair and 715,004 / +244,304,737 in the holdout. These full field comparisons cover those two pairs; +raw/parser integrity audits separately cover all eight selective runs. No fields, array elements or +duplicates are removed. Differences remain in identities/timestamps, constructor +`object_id` and CUDA `cuda_stream`/`native_handle` values; raw equality is false. +Constructor snapshots store literal `id(receiver)` in both source archives. All +800/1,600 stream-metadata observations preserve alias equality and a bijective +observed handle correspondence. No structural/type/shape/array/exception payload +differences occur, including the same two step-one `_BypassDispatchCache` exceptions. +No global cross-process identity graph or inter-thread ordering equality is claimed. + +All eight raw/parser audits and eight checker phases completed successfully for +this exact revision. The first seed-42 selective pair and seed-43 holdout each +execute the same 722 frozen rules. Every cohort has **719 active rules: 718 passes +and one violation; three rules are dormant**. There are no checker errors, unknown +outcomes, unobserved coverage or variable-check skips. Decision counts match across +variants: 114,373 per ten-step corpus and 218,713 per holdout corpus. No variable +precondition is requested, consistent with the API-only scope; this study does +not establish model-tracker or variable-checking coverage. + +Within each corpus, all 722 complete `checker_result` values match structurally +between observed and independent direct checking. The comparator preserves scalar +types, fields, identities and array order; five legacy NaN literals match only at +the same paths. Cross-variant rule outcomes and decision counts match, while +complete checker results match for 721/722 rules. Rule 614 retains fresh-run +identity/timestamp differences in its failure evidence; those fields are not +removed or remapped to claim equality. + +The shared violation is rule 614, `cuda.stream() always calls +StreamContext.__init__()`: its frozen child argument requires CUDA handle +385452624. Actual handles are 386050160/386062656 in the ten-step pair and +386059824/386070112 in the holdout. Both handle aliases agree, all other required +fields match, and no fields are missing or extra. Dormant APIContain rules 216/276 +require reference worker ID 23438301021888; rule 717 requires unequal threads and +has zero decisions. Rules were not reinferred or filtered. This is unchanged +frozen-rule behavior, not evidence of zero violations or universal rule activation. + +Offline validation ran on two separate labmach CPU cores, one native thread per +worker and an 8-GiB process-group cap per lane. Its scheduling and operational +timings are separate from inference-performance claims. Source and frozen input +inventories remained unchanged. All 98 downloaded report files were verified. + +## Sources, environment and validation + +- CPU baseline: `6dccb47d0a8bc192fbe7cbbea700b66106bc2381`. +- H100 baseline: composed `8cc73ab35590caa1a5d7286b16462d045b6d2da0`; candidate + overlays only `dumper.py` and `metadata_methods.py` from signed `37f22e5`. + CUDA hashing, selective policy and workload code are identical across variants. + The production PR targets main and does not depend on the study's other changes. +- TorchTitan: `2210414902d84d1219fd63ab8efeb24375a2a1a1`. +- Frozen invariant SHA-256: + `bad55c222f4eb3406bdfaec527ab1a75eaf66cdbe8f06c42d7f0775f4ee9224b`; + compiled policy SHA-256: + `df4c91296774b2dfd3af09d0a25e21e25f26ff1de93cb97ab959fa77379c3211`. +- Labmach CPU: Python 3.11.4, PyTorch `2.16.0.dev20261012+cpu`, NumPy 2.1.0; + one Torch/native thread, CUDA hidden. MNIST is the existing 1,199,882-parameter + Net with six cached normalized examples, NLL and Adadelta. The Transformer is + 21,052 parameters: two width-32 layers, feed-forward width 64, two heads, + sequence 16, batch two, fixed WikiText prefix/vocabulary, NLL and clipped SGD. + Seeds, data and benchmark-definition hashes are retained in every run manifest. +- Expanse: H100 80GB HBM3, Python 3.12.3, PyTorch + `2.16.0.dev20261012+cu130`, CUDA 13.0. Eight CPUs / 48 GiB allocated; GPU work + runs on compute nodes. Ordinary job completed in 4:36. Batch peak memory covers + the entire job, not individual variants; no memory-cost improvement is claimed. +- Final local full suite: **335 passed, two failures** also reproduced on the + measured main baseline (`test_traincheck_internal_attr_hidden` and + `test_inject_eval_loop`). All 26 focused metadata tests pass on Python 3.10 and + 3.12; the exact composed source passes 33 metadata/classification tests on both. + The matching labmach runtime also passes all 33. +- Pinned changed-file pre-commit, explicit test lint/format checks and strict + scoped mypy pass. The legacy dumper retains its configured mypy coverage rather + than claiming a repository-wide strictness migration. Earlier all-files checking + found only the documented, unchanged Black whitespace failures in `config.py` + and `proxy_wrapper/hash.py`; those unrelated edits remain excluded. + +## Durable evidence and reproduction + +Local artifact root: `outputs/metadata-fastpath-20261027/` in the canonical +checkout. CPU evidence is in `cpu-binding/`; new H100 packages and results are in +`gpu-binding/` and `gpu-binding-diagnostic/`. Final checking evidence is in +`validation-binding/job-55326071/validation-summary.json` and +`focused-attribution.json`; raw comparison reports are under ordinary job 55326071's +`trace-comparison/`. Exact commands, frozen source +inventories, raw traces, timings, diagnostic scripts and failure logs remain +outside Git. Historical studies are preserved separately. No credentials are +included in source or evidence packages. + +| Artifact | SHA-256 | +| --- | --- | +| CPU evidence archive | `7418a17bd9bf05c36119ac56eb23abb28bd5280e552afafbe2a6df39931e8edd` | +| CPU candidate source archive | `61ff85dce4b0e94633e6fc932d6bc051744fe9c17d8625cf83a5d44c7aae5381` | +| H100 candidate source archive | `517c5afa213ec1693d42aa04f9afd61676dd1680f94b2f73734f0759dc3e68b5` | +| H100 ordinary results, job 55326071 | `3754fa9de95c586a4338ff49a5daec3ccac67a4d40689cfb07bf09c6e66a403b` | +| H100 diagnostic results, job 55326074 | `d6ddd8bb9852209d6f12f64e23f7f68a56d3e3d1f5535645b13b79644aa90df4` | +| Complete fresh validation reports | `4323fff37a0085d75de432645025918b3b262ec56b316f2bb19d002690288abd` | + +From the artifact root, rerun saved analyzers: + +```sh +python analyze_cpu_binding.py --root cpu-binding/results --archives cpu-binding \ + --output cpu-binding/verified-analysis.json +python analyze_binding_gpu_results.py \ + --ordinary gpu-binding/job-55326071/results/ordinary \ + --output gpu-binding/job-55326071/ordinary-analysis.json +``` + +For new experiments, use fresh output directories and the frozen packages' README +commands. CPU runs use `cpu-binding/run_study.py --root STUDY --candidate-commit +37f22e58954da403105c2c5510b7c7b6b3162ee6`; H100 submission verifies package checksums +before `sbatch comparison.sbatch`. Source inventories are verified before and after +execution. CPU processes have 90-second / 4-GiB / 256-MiB-trace bounds; H100 phases +and jobs have explicit time/trace/allocation limits. No limit was reached. diff --git a/docs/measurements/native-tensor-methods-20261027.md b/docs/measurements/native-tensor-methods-20261027.md index 5a4a6919..1e73d6f9 100644 --- a/docs/measurements/native-tensor-methods-20261027.md +++ b/docs/measurements/native-tensor-methods-20261027.md @@ -1,5 +1,7 @@ # Historical CPU measurement: skipping tensor method metadata +The final binding-preserving implementation has [fresh CPU and H100 measurements](native-tensor-bindings-20261027.md). + **Superseded candidate:** these numbers measure `77ea19b`. Later lifetime tests found that skipping a bound method changes references observable by subsequent properties. The revised implementation retains real native binding and the diff --git a/docs/measurements/native-tensor-methods-h100-20261027.md b/docs/measurements/native-tensor-methods-h100-20261027.md index d9bd3076..be89ac56 100644 --- a/docs/measurements/native-tensor-methods-h100-20261027.md +++ b/docs/measurements/native-tensor-methods-h100-20261027.md @@ -1,5 +1,7 @@ # Historical H100 measurement: skipping tensor method metadata +The final binding-preserving implementation has [fresh CPU and H100 measurements](native-tensor-bindings-20261027.md). + **Superseded candidate:** this report measures `287b3a2`. Additional lifetime tests found that skipping a binding can retain a previous temporary value/type or release a receiver/function too early. The revised implementation returns the real native