From b8e7af66738bfdc24b92a9a78f199b1e40564249 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Tue, 27 Oct 2026 02:57:33 +0800 Subject: [PATCH 1/5] Skip framework metadata checks for exact callable types --- tests/test_metadata_classification.py | 221 ++++++++++++++++++++++++++ traincheck/instrumentor/dumper.py | 48 +++++- 2 files changed, 262 insertions(+), 7 deletions(-) create mode 100644 tests/test_metadata_classification.py diff --git a/tests/test_metadata_classification.py b/tests/test_metadata_classification.py new file mode 100644 index 00000000..3c0fc7cc --- /dev/null +++ b/tests/test_metadata_classification.py @@ -0,0 +1,221 @@ +"""Record contracts for metadata classification, including custom type behavior.""" + +import unittest +import warnings +from contextlib import ExitStack +from unittest.mock import patch + +import torch + +from traincheck.instrumentor import dumper + + +class AttributeHolder: + def __init__(self, value: object) -> None: + self.value = value + + +class CustomParameter(torch.nn.Parameter): + pass + + +class CustomTensor(torch.Tensor): + pass + + +class ChangingClass: + """An object allowed by Python's isinstance protocol, not an actual tensor.""" + + def __init__(self, error: Exception | None = None) -> None: + self.reads = 0 + self.error = error + + def __getattribute__(self, name: str) -> object: + if name == "__class__": + self.reads += 1 + if self.reads == 1: + return object + if self.error is not None: + raise self.error + return torch.nn.Parameter + return super().__getattribute__(name) + + +class MetadataClassificationTests(unittest.TestCase): + def setUp(self) -> None: + contexts = ExitStack() + self.addCleanup(contexts.close) + for name in ( + "skip_attrs_due_to_errs", + "skip_type_due_to_errs", + "skip_type_due_to_recursion", + ): + contexts.enter_context(patch.object(dumper, name, {})) + + def test_callable_attributes_keep_metadata_values_and_order(self) -> None: + def function() -> None: + return None + + callables: tuple[object, ...] = ( + function, + len, + self.setUp, + object().__str__, + object.__str__, + str.upper, + dict.__dict__["fromkeys"], + ) + for callback in callables: + with self.subTest(callback_type=type(callback)): + holder = AttributeHolder(callback) + holder.__dict__["_TRAINCHECK_callback"] = callback + holder.__dict__["name"] = "observed" + holder.__dict__["dtype"] = torch.float32 + holder.__dict__["shape"] = torch.Size([2, 3]) + record = dumper.convert_var_to_dict( + holder, + include_tensor_data=False, + dump_config={ + "value": None, + "name": None, + "_TRAINCHECK_callback": None, + "dtype": None, + "shape": None, + }, + ) + self.assertEqual( + list(record), ["name", "_TRAINCHECK_callback", "dtype", "shape"] + ) + self.assertEqual(record["name"], "observed") + self.assertIs(record["_TRAINCHECK_callback"], callback) + self.assertEqual(record["dtype"], "torch.float32") + self.assertEqual(record["shape"], (2, 3)) + + def test_tensor_and_parameter_subclasses_keep_identity_and_values(self) -> None: + flagged = torch.tensor([3.0, 4.0]) + flagged.__dict__["_is_param"] = True + values = ( + torch.tensor([3.0, 4.0]), + torch.tensor([3.0, 4.0]).as_subclass(CustomTensor), + torch.nn.Parameter(torch.tensor([3.0, 4.0])), + CustomParameter(torch.tensor([3.0, 4.0])), + flagged, + ) + with patch.dict( + dumper.tensor_dump_format, + dump_tensor_hash=False, + dump_tensor_stats=False, + dump_tensor_full=True, + ): + for value in values: + with self.subTest(tensor_type=type(value)): + holder = AttributeHolder(value) + metadata = dumper.convert_var_to_dict( + holder, False, {"value": None} + ) + self.assertEqual(metadata, {"_TRAINCHECK_value_ID": id(value)}) + before = dumper.convert_var_to_dict(holder, True, {"value": None}) + self.assertEqual( + before, + {"_TRAINCHECK_value_ID": id(value), "value": [3.0, 4.0]}, + ) + self.assertEqual(list(before), ["_TRAINCHECK_value_ID", "value"]) + with torch.no_grad(): + value.add_(1) + after = dumper.convert_var_to_dict(holder, True, {"value": None}) + self.assertEqual( + after, + {"_TRAINCHECK_value_ID": id(value), "value": [4.0, 5.0]}, + ) + self.assertEqual(before["value"], [3.0, 4.0]) + + def test_dynamic_class_keeps_parameter_fallback(self) -> None: + value = ChangingClass() + record = dumper.convert_var_to_dict( + AttributeHolder(value), False, {"value": None} + ) + self.assertEqual(record, {"_TRAINCHECK_value_ID": id(value)}) + self.assertEqual(value.reads, 2) + + def test_dynamic_class_exception_still_propagates(self) -> None: + error = ValueError("dynamic class failed") + value = ChangingClass(error) + with self.assertRaises(ValueError) as caught: + dumper.convert_var_to_dict(AttributeHolder(value), False, {"value": None}) + self.assertIs(caught.exception, error) + + def test_custom_metaclass_does_not_add_classification_side_effects(self) -> None: + visits: list[str] = [] + + class Metaclass(type): + def __hash__(cls) -> int: + visits.append("hash") + if len(visits) > 1: + raise ValueError("unexpected repeated type hashing") + return type.__hash__(cls) + + class Value(metaclass=Metaclass): + pass + + self.assertEqual( + dumper.convert_var_to_dict( + AttributeHolder(Value()), False, {"value": None} + ), + {}, + ) + self.assertEqual(visits, ["hash"]) + + def test_getters_warning_restoration_and_failed_attribute_backoff(self) -> None: + visits: list[str] = [] + + class Getters: + @property + def callback(self) -> object: + visits.append("callback") + warnings.warn("metadata warning", UserWarning) + warnings.simplefilter("error", RuntimeWarning) + return len + + @property + def broken(self) -> object: + visits.append("broken") + raise ValueError("missing metadata") + + @property + def value(self) -> int: + visits.append("value") + warnings.warn("later warning", RuntimeWarning) + return 7 + + with warnings.catch_warnings(record=True) as observed: + warnings.simplefilter("always") + filters = list(warnings.filters) + holder = Getters() + config = {"callback": None, "broken": None, "value": None} + self.assertEqual( + dumper.convert_var_to_dict(holder, False, config), {"value": 7} + ) + self.assertEqual( + dumper.convert_var_to_dict(holder, False, config), {"value": 7} + ) + self.assertEqual(warnings.filters, filters) + self.assertEqual(observed, []) + self.assertEqual(visits, ["callback", "broken", "value", "callback", "value"]) + + def test_dict_subclass_keeps_dynamic_keys_and_missing_fields(self) -> None: + class Metadata(dict[str, object]): + pass + + value = Metadata(first=1, callback=len) + self.assertEqual(dumper.convert_var_to_dict(value, False), {"first": 1}) + value["second"] = 2 + result = dumper.convert_var_to_dict(value, False) + self.assertEqual(result, {"first": 1, "second": 2}) + self.assertEqual(list(result), ["first", "second"]) + self.assertEqual( + dumper.convert_var_to_dict(value, False, {"missing": None}), {} + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/traincheck/instrumentor/dumper.py b/traincheck/instrumentor/dumper.py index 5a852d33..c8d7ce3a 100644 --- a/traincheck/instrumentor/dumper.py +++ b/traincheck/instrumentor/dumper.py @@ -3,8 +3,10 @@ import os import threading import time +import types +from collections.abc import Mapping from queue import Empty, Queue -from typing import Callable, Hashable +from typing import Callable, Hashable, cast import orjson import torch @@ -45,6 +47,20 @@ logger = logging.getLogger(__name__) +# Exact callable types have no tensor/module metadata. Subclasses and arbitrary +# callable objects retain the generic path, including dynamic __class__ behavior. +_METADATA_CALLABLE_TYPES: frozenset[type] = frozenset( + { + types.FunctionType, + types.BuiltinFunctionType, + types.MethodType, + types.MethodWrapperType, + types.WrapperDescriptorType, + types.MethodDescriptorType, + types.ClassMethodDescriptorType, + } +) + def _json_default(o): try: @@ -325,22 +341,29 @@ class NOT_FOUND: pass -def safe_getattr(obj, attr_name): +def safe_getattr(obj: object, attr_name: str) -> object: import warnings try: with warnings.catch_warnings(): warnings.simplefilter("ignore") - attr = getattr(obj, attr_name, NOT_FOUND) + attr: object = getattr(obj, attr_name, NOT_FOUND) if attr is NOT_FOUND: if issubclass(type(obj), dict): - attr = dict.get(obj, attr_name, NOT_FOUND) + # The concrete-type check validates this cast without + # invoking an object's potentially dynamic __class__. + mapping = cast(dict[object, object], obj) + attr = dict.get(mapping, attr_name, NOT_FOUND) return attr except Exception: return NOT_FOUND -def convert_var_to_dict(var, include_tensor_data=True, dump_config=None) -> dict: +def convert_var_to_dict( + var: object, + include_tensor_data: bool = True, + dump_config: Mapping[str, object] | None = None, +) -> dict[str, object]: """ TODO: variables can be nested and thus this should be a recursive function (dump_config supports this). But this function is not recursive yet, so it only dumps the first level of attributes of the variable. @@ -364,7 +387,10 @@ def convert_var_to_dict(var, include_tensor_data=True, dump_config=None) -> dict name for name in dir(var) if not name.startswith("__") ] # dir() won't get called on primitive vars (look at the logic below checking for primitive types) whose dump_config is always None, so no need to check for primitive types here. if issubclass(type(var), dict): - attr_names += [k for k in var.keys() if isinstance(k, str)] + # The concrete-type check excludes objects spoofing __class__; + # mypy cannot narrow through issubclass(type(...), ...). + mapping = cast(dict[object, object], var) + attr_names += [k for k in mapping.keys() if isinstance(k, str)] except Exception as e: if type(var) not in skip_type_due_to_errs: skip_type_due_to_errs[type(var)] = 0 @@ -415,9 +441,17 @@ def convert_var_to_dict(var, include_tensor_data=True, dump_config=None) -> dict continue attr_name = str(attr_name) - if type(attr) in primitive_types: + attr_type = type(attr) + if attr_type in primitive_types: result[attr_name] = attr + elif type(attr_type) is type and attr_type in _METADATA_CALLABLE_TYPES: + # These exact built-in types cannot match the framework branches + # below. Retain explicitly requested TrainCheck metadata unchanged. + # The metaclass guard avoids extra user-defined __hash__/__eq__ calls. + if "_TRAINCHECK" in attr_name: + result[attr_name] = attr + elif isinstance(attr, torch.Tensor): result[f"_TRAINCHECK_{attr_name}_ID"] = id(attr) if include_tensor_data: From c5faf1e0fb66ed3e8d76af3e416baa60c3e4af5e Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Tue, 27 Oct 2026 03:10:46 +0800 Subject: [PATCH 2/5] Record matched API and subclass tracing measurements --- .../metadata-classification-20261027.json | 2505 +++++++++++++++++ .../metadata-classification-20261027.md | 227 ++ 2 files changed, 2732 insertions(+) create mode 100644 docs/measurements/data/metadata-classification-20261027.json create mode 100644 docs/measurements/metadata-classification-20261027.md diff --git a/docs/measurements/data/metadata-classification-20261027.json b/docs/measurements/data/metadata-classification-20261027.json new file mode 100644 index 00000000..379d3193 --- /dev/null +++ b/docs/measurements/data/metadata-classification-20261027.json @@ -0,0 +1,2505 @@ +{ + "artifacts": { + "driver_sha256": "3a89d821238902362b110abbc9d66bb2b87c1b1cef09144bf7407a5400cd72f4", + "evidence_archive_bytes": 24826924, + "evidence_archive_sha256": "1de069eb0dd5d8dd4ce60e758a4d3dae5716fa69bda1f3e30570b983f4e69773", + "remote": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed", + "runner_sha256": "d17315872c4e412d3a2be075fa733eebb9c7d4f8189dff82d53dc6bae92c0818" + }, + "limitations": [ + "CPU only, one native thread, three cold steps and five pairs on a shared host", + "Subclass tracker only; unchanged CPU hashing, tensor dump format, tracing coverage and sample policy", + "Timing excludes setup, diagnostics and final writer drain; process time and post-metrics exit separately retained", + "Post-metrics exit includes Python shutdown and polling delay; it is not a pure writer timer", + "Full raw traces retained; no normalized cross-process trace equivalence claim" + ], + "production_changed_files": [ + "traincheck/instrumentor/dumper.py" + ], + "smoke": [ + { + "logical_hash_input_bytes": 0, + "mode": "plain", + "records": { + "API": 0, + "VAR": 0 + }, + "tensor_hash_fields": 0, + "trace_bytes": { + "API": 0, + "VAR": 0 + }, + "tracked_variables": [], + "training_seconds": 0.01541786640882492 + }, + { + "logical_hash_input_bytes": 0, + "mode": "api", + "records": { + "API": 1338, + "VAR": 0 + }, + "tensor_hash_fields": 0, + "trace_bytes": { + "API": 1014134, + "VAR": 0 + }, + "tracked_variables": [], + "training_seconds": 0.8779591619968414 + }, + { + "logical_hash_input_bytes": 214496, + "mode": "var", + "records": { + "API": 0, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 0, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "training_seconds": 0.08836576342582703 + }, + { + "logical_hash_input_bytes": 214496, + "mode": "combined", + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 1034378, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "training_seconds": 0.9408722668886185 + } + ], + "sources": { + "baseline": { + "adapter_sha256": "a5b5e597d4921a5b4baceeada541a6a1765f13c97ab3bd1c2ef79dea331f0934", + "production_files_verified": 77, + "production_hashes": { + "traincheck/__init__.py": "47fdcb090c4c8c5822769d642dc23a520e8d8ff3e4724583450723e05408164b", + "traincheck/checker.py": "44dc20fc66ca82bb6b71fa02033aae3b7d489ae7afbc1611c44e857f399716b2", + "traincheck/checker_online.py": "2824c9ccd2f684c86884c68f2214a1a4be1d69c6098ce30b817d03f1fe1077d6", + "traincheck/collect_trace.py": "c1db96c798af9689f8ee7f7b0b929d90694d49a234cb217beaf435d4b8d5cfb0", + "traincheck/config/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/config/config.py": "4c940a4d92f5de5fe8d3885e570078e479d9842d9626651fd60483e78f0160a9", + "traincheck/developer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/developer/annotations.py": "0ff28d35e085b40e1e366499feec96c8eb1e13bf3100dcdd7ab304c7f2f6e963", + "traincheck/infer_engine.py": "16bf736b498a5f86583b2f1207d31f37479db72510b04121abcb1158d0141f73", + "traincheck/inference_sampling.py": "d261b3416aba3ead6fa1e9b15091511eb3525137d531e4f0147146e76308976f", + "traincheck/instrumentor/__init__.py": "228b4f6a11cde8118cde9aba2f98b60a839f4331e7fa2b06c37cee99e79fcd50", + "traincheck/instrumentor/caches.py": "320eb1ffa1ce5f5b97fe4675a16d47e9591265779fdc29b94b72cdf5df93f168", + "traincheck/instrumentor/control.py": "3612a39b61e85fb77b40008b7fcd2f7569b246e8985ca64a109ea53733cd29ef", + "traincheck/instrumentor/dumper.py": "cd4a56d4eb95049dcab85bacc1c64d7e55bfe45087fe5096ed247e4d0d7e8373", + "traincheck/instrumentor/proxy_wrapper/__init__.py": "7c71aaf70b978ee04e236497af6f803e4436049bc0acdf18c3edb48bfcece63d", + "traincheck/instrumentor/proxy_wrapper/dumper.py": "6907ac0a198443dd9138a37dc9b5de1232d3d7fb26412f8bba3fb08c85cd222d", + "traincheck/instrumentor/proxy_wrapper/hash.py": "e917bec817df1981c682d8b8645307432c6badb220b9f7c424b4089d9bf97bd7", + "traincheck/instrumentor/proxy_wrapper/proxy.py": "fe241558f0591738391bc57d0666e3b11e04d8579eb220ce5de855abc309baf3", + "traincheck/instrumentor/proxy_wrapper/proxy_basics.py": "f012e97fa51cf77be1806807ef6130b06c9cec6fde8d047d57d45b6ec9fcc6e5", + "traincheck/instrumentor/proxy_wrapper/proxy_config.py": "afb2f8dd04ed10e628ff21339ec8ebef36357fd2a0cc084e99e99e37756cf16b", + "traincheck/instrumentor/proxy_wrapper/proxy_handler.py": "4bab74647d35c64c5e9c702295dc4cb010302bf620401da0fad8539692b95ac7", + "traincheck/instrumentor/proxy_wrapper/proxy_methods.py": "6ae18928aa12e89b33102368c497aff048e3d6bed330eca2118fac051b8e3a97", + "traincheck/instrumentor/proxy_wrapper/proxy_observer.py": "10e83570fbe1f72089e4a594a308f83e045f7d74bb9e49193012645ac906d97b", + "traincheck/instrumentor/proxy_wrapper/proxy_registry.py": "9dfecae65aee4185152d152a5c2fb0e1e1b63339efceab3087df59a5882410a8", + "traincheck/instrumentor/proxy_wrapper/subclass.py": "6152488b768edb35617451424c4e494b8924bf72ab0c21f198bca9dab15588bb", + "traincheck/instrumentor/proxy_wrapper/torch_proxy.py": "d6abd7f2eac207757fd90cbf55e179ae506b25996bed906c5a094564ff6faf2f", + "traincheck/instrumentor/proxy_wrapper/utils.py": "5f058609e89a96a3545561ed82d83f7dd6a153bb0376608654c199691984dc14", + "traincheck/instrumentor/replace_functions.py": "ca4f10b52083a4a5c251db194b7608406cbe36c85372dfdb9df17644f3d20b62", + "traincheck/instrumentor/source_file.py": "dad92c793efb41161d86250e89aa6a9636749564e567df8b9af92a477eb4858d", + "traincheck/instrumentor/torchscript_compat.py": "1b42072c80b29c2432f3ef83d6849365ecc0ae8c015dd2051a5b192ba209f88a", + "traincheck/instrumentor/tracer.py": "be6511d748415b17dc8056e2d120a2901003fcc242cfd2e0a16772432021bd9d", + "traincheck/instrumentor/types.py": "258679aa8d0b40247d900ba2e26ab03b25cb4626b046ab6d724973226c7b5469", + "traincheck/instrumentor/variable.py": "b01219cdd74f011055e0bc2a645ed283dbc47e66a834d5555b814fa815a9eab8", + "traincheck/invariant/DistinctArgumentRelation.py": "6e01540583668e30718ba90c341fd540cd9df53f20525a6cf502fcceee8ed7bc", + "traincheck/invariant/__init__.py": "773e9f767566ffd92658dc0e94107ac840672e446a41ba437a49cee9cc52197a", + "traincheck/invariant/base_cls.py": "8c419a0ea675146900be1b78a47de2da07f27442019b632ff224be870d48d5e6", + "traincheck/invariant/consistency_relation.py": "bc5b14810ea8a5e0e5f3e6c2c6be24d137a444d8609216d00eafd155b4aa3917", + "traincheck/invariant/consistency_transient_vars.py": "a702cd1b6787e8be1904bef9119773c0e037a83017959c771020dbbbbe1549e2", + "traincheck/invariant/contain_relation.py": "b32f819db4d5a11a11318bf7e38c64148f4c9360137ac5f09475b480c34c16a5", + "traincheck/invariant/cover_relation.py": "0eb0e1c30ad5b34a15ce32c7b0d15e894f7a3b9b68bfcaae5b901f5dc6a092c6", + "traincheck/invariant/lead_relation.py": "978a4db32b69dc1a90a84123aa93e52a3caaba97761321dbb844666cb47d4360", + "traincheck/invariant/precondition.py": "c14c7580643780954eab5a4d955b0b5d6a6768a57cd8e5e48328ad0d8f2f8703", + "traincheck/invariant/relation_pool.py": "8ac568cf1924162a791728ed5db8c41868a550f9e0cd5190fc9a19827f55d3e6", + "traincheck/invariant/symbolic_value.py": "28627e2c2a47a146087a96510f07f7deb8463c46371e418afa3fdeff33c7e6cb", + "traincheck/invariant/var_periodic_change_relation.py": "8086df3b9cfe34229b8bd46b18dce22a4b746902e38d1ba4bd02bc6f6c2f2ef4", + "traincheck/onlinechecker/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/onlinechecker/streamhandler_filesystem.py": "b5a298bd06a9408e857ff6721578cb4ae10f227762ba3d66aec685e574cfa511", + "traincheck/onlinechecker/utils.py": "65ba5d8a84b748adf4267800e704df1f4662c0f483534289af20096bf9a78a00", + "traincheck/parallel.py": "a431d169022264109c56b7057ebcdc8a587b698a8670a1257b765189d9e5bf76", + "traincheck/progress.py": "aab7d040324e283c8474aa82bc7550a777aec316131a26e1dc1b6a277333d1ae", + "traincheck/reporting/__init__.py": "299180153357c4e6f48cf75191e915d70cfca5ae6e7b3ac4d45144066b93ddca", + "traincheck/reporting/checker_report.py": "0d59b1753fc348701b49a0eb74e4b1d849a47d8234005ccbbb26e09e3cd0c808", + "traincheck/runner/__init__.py": "287ffa9cab82f01da0e82faea77589dc7d348710c862496458d2d60c364394ba", + "traincheck/runner/runner.py": "89027b488ded8640fddb387dd80d98f059cf40be5f94863966b259acea1da7c7", + "traincheck/script/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/script/num_invs_each_rel.py": "202f13d536929c52defa84cdce00dec2251ea09aabf3ae8ae57067c32d0b844d", + "traincheck/static_analyzer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/config.py": "840460167bd6d26db429076cd4dc227e6c17c4e5fb157172041c48574f44edf7", + "traincheck/static_analyzer/func_level/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/graph_generator/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/graph_generator/analyzer.py": "035684eb7ea4edf8ee0dbf4c6b12b91b428cd6dd57be5fe658253c21c8c420b1", + "traincheck/static_analyzer/graph_generator/anutils.py": "9e62b7ce5fced45a033b76de3d408bdff2fd8561ffa28bc58de82e4105b51d39", + "traincheck/static_analyzer/graph_generator/call_graph_parser.py": "69e2555a1595078a2d2bca174de32cd7d4b0f023c0a20b68a01ca00d1d3b8647", + "traincheck/static_analyzer/graph_generator/main.py": "a568488c50383a67a6ab8feecaf96e29b288861c1dce17481a1b5032d20748a4", + "traincheck/static_analyzer/graph_generator/node.py": "b6a0cea44a28d33f28fee79b7f7073e69914e95565f7a8d62f9e5a5fd6b240cd", + "traincheck/toolkit/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/toolkit/analyze_invariant.py": "706cf8c06062b6df3f1f86691f588c91696cbef6c27698b2f0b290ad8f53ff4b", + "traincheck/toolkit/analyze_trace.py": "48dfc81ef85d96678b5fe466b86ba80d892ecfc497387773fa7ab5995f22e822", + "traincheck/toolkit/detect_anomaly_from_trace_diff.py": "2ffc0723dd3709ccab09cb9b58d76c7bb0558dda09b66c47184e335b815cedd3", + "traincheck/trace/__init__.py": "ec84edd0ff32906c20956166a51b85b3847e22e48432c6a31698080cafbcbc2b", + "traincheck/trace/trace.py": "c4f6fae85de0a9d95591ab88d52ad8acf24f052f42978ec381a53042e1e49e67", + "traincheck/trace/trace_dict.py": "3aec874ef0ce2c34f6332dface604d5f79965976fd3d1c2b692f746f2af4e703", + "traincheck/trace/trace_pandas.py": "953d9cac029f83e015eb22e3bc9c8abc7b63919a0eef18b9794517392117ec24", + "traincheck/trace/trace_polars.py": "64136efcd60a851b0391e708108570b259c2288db6242564b636f69abd1b7b0e", + "traincheck/trace/types.py": "96c1b137f3d72fb05dcf806a4e7ea21fe453ae9cbc3e08352c55e95973fe96e0", + "traincheck/trace/utils.py": "6356156e4c5bbb6c111140302660f36ad1c126e5bc977bd55c81f097022102a3", + "traincheck/utils.py": "71034730f74a15978d37dc9fc9b748133389a69e0ea7f9c9109dd1bc573fd272" + }, + "source_archive_sha256": "ac1a830f490bef1af6b1b15ba49d116ff28a8de2e57f51dd80afc714f4e8d909", + "source_commit": "88b6df2401e187ca6bd81bf0a83bc7c5d9a5cd54", + "source_tree": "5415a3b0e5eb08d119dffa91631d76866e6e1d11" + }, + "candidate": { + "adapter_sha256": "a5b5e597d4921a5b4baceeada541a6a1765f13c97ab3bd1c2ef79dea331f0934", + "production_files_verified": 77, + "production_hashes": { + "traincheck/__init__.py": "47fdcb090c4c8c5822769d642dc23a520e8d8ff3e4724583450723e05408164b", + "traincheck/checker.py": "44dc20fc66ca82bb6b71fa02033aae3b7d489ae7afbc1611c44e857f399716b2", + "traincheck/checker_online.py": "2824c9ccd2f684c86884c68f2214a1a4be1d69c6098ce30b817d03f1fe1077d6", + "traincheck/collect_trace.py": "c1db96c798af9689f8ee7f7b0b929d90694d49a234cb217beaf435d4b8d5cfb0", + "traincheck/config/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/config/config.py": "4c940a4d92f5de5fe8d3885e570078e479d9842d9626651fd60483e78f0160a9", + "traincheck/developer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/developer/annotations.py": "0ff28d35e085b40e1e366499feec96c8eb1e13bf3100dcdd7ab304c7f2f6e963", + "traincheck/infer_engine.py": "16bf736b498a5f86583b2f1207d31f37479db72510b04121abcb1158d0141f73", + "traincheck/inference_sampling.py": "d261b3416aba3ead6fa1e9b15091511eb3525137d531e4f0147146e76308976f", + "traincheck/instrumentor/__init__.py": "228b4f6a11cde8118cde9aba2f98b60a839f4331e7fa2b06c37cee99e79fcd50", + "traincheck/instrumentor/caches.py": "320eb1ffa1ce5f5b97fe4675a16d47e9591265779fdc29b94b72cdf5df93f168", + "traincheck/instrumentor/control.py": "3612a39b61e85fb77b40008b7fcd2f7569b246e8985ca64a109ea53733cd29ef", + "traincheck/instrumentor/dumper.py": "faf87dcfb0aa08dd97bb544aa80120925ec1eed05d03947a1de743fdf5da53e2", + "traincheck/instrumentor/proxy_wrapper/__init__.py": "7c71aaf70b978ee04e236497af6f803e4436049bc0acdf18c3edb48bfcece63d", + "traincheck/instrumentor/proxy_wrapper/dumper.py": "6907ac0a198443dd9138a37dc9b5de1232d3d7fb26412f8bba3fb08c85cd222d", + "traincheck/instrumentor/proxy_wrapper/hash.py": "e917bec817df1981c682d8b8645307432c6badb220b9f7c424b4089d9bf97bd7", + "traincheck/instrumentor/proxy_wrapper/proxy.py": "fe241558f0591738391bc57d0666e3b11e04d8579eb220ce5de855abc309baf3", + "traincheck/instrumentor/proxy_wrapper/proxy_basics.py": "f012e97fa51cf77be1806807ef6130b06c9cec6fde8d047d57d45b6ec9fcc6e5", + "traincheck/instrumentor/proxy_wrapper/proxy_config.py": "afb2f8dd04ed10e628ff21339ec8ebef36357fd2a0cc084e99e99e37756cf16b", + "traincheck/instrumentor/proxy_wrapper/proxy_handler.py": "4bab74647d35c64c5e9c702295dc4cb010302bf620401da0fad8539692b95ac7", + "traincheck/instrumentor/proxy_wrapper/proxy_methods.py": "6ae18928aa12e89b33102368c497aff048e3d6bed330eca2118fac051b8e3a97", + "traincheck/instrumentor/proxy_wrapper/proxy_observer.py": "10e83570fbe1f72089e4a594a308f83e045f7d74bb9e49193012645ac906d97b", + "traincheck/instrumentor/proxy_wrapper/proxy_registry.py": "9dfecae65aee4185152d152a5c2fb0e1e1b63339efceab3087df59a5882410a8", + "traincheck/instrumentor/proxy_wrapper/subclass.py": "6152488b768edb35617451424c4e494b8924bf72ab0c21f198bca9dab15588bb", + "traincheck/instrumentor/proxy_wrapper/torch_proxy.py": "d6abd7f2eac207757fd90cbf55e179ae506b25996bed906c5a094564ff6faf2f", + "traincheck/instrumentor/proxy_wrapper/utils.py": "5f058609e89a96a3545561ed82d83f7dd6a153bb0376608654c199691984dc14", + "traincheck/instrumentor/replace_functions.py": "ca4f10b52083a4a5c251db194b7608406cbe36c85372dfdb9df17644f3d20b62", + "traincheck/instrumentor/source_file.py": "dad92c793efb41161d86250e89aa6a9636749564e567df8b9af92a477eb4858d", + "traincheck/instrumentor/torchscript_compat.py": "1b42072c80b29c2432f3ef83d6849365ecc0ae8c015dd2051a5b192ba209f88a", + "traincheck/instrumentor/tracer.py": "be6511d748415b17dc8056e2d120a2901003fcc242cfd2e0a16772432021bd9d", + "traincheck/instrumentor/types.py": "258679aa8d0b40247d900ba2e26ab03b25cb4626b046ab6d724973226c7b5469", + "traincheck/instrumentor/variable.py": "b01219cdd74f011055e0bc2a645ed283dbc47e66a834d5555b814fa815a9eab8", + "traincheck/invariant/DistinctArgumentRelation.py": "6e01540583668e30718ba90c341fd540cd9df53f20525a6cf502fcceee8ed7bc", + "traincheck/invariant/__init__.py": "773e9f767566ffd92658dc0e94107ac840672e446a41ba437a49cee9cc52197a", + "traincheck/invariant/base_cls.py": "8c419a0ea675146900be1b78a47de2da07f27442019b632ff224be870d48d5e6", + "traincheck/invariant/consistency_relation.py": "bc5b14810ea8a5e0e5f3e6c2c6be24d137a444d8609216d00eafd155b4aa3917", + "traincheck/invariant/consistency_transient_vars.py": "a702cd1b6787e8be1904bef9119773c0e037a83017959c771020dbbbbe1549e2", + "traincheck/invariant/contain_relation.py": "b32f819db4d5a11a11318bf7e38c64148f4c9360137ac5f09475b480c34c16a5", + "traincheck/invariant/cover_relation.py": "0eb0e1c30ad5b34a15ce32c7b0d15e894f7a3b9b68bfcaae5b901f5dc6a092c6", + "traincheck/invariant/lead_relation.py": "978a4db32b69dc1a90a84123aa93e52a3caaba97761321dbb844666cb47d4360", + "traincheck/invariant/precondition.py": "c14c7580643780954eab5a4d955b0b5d6a6768a57cd8e5e48328ad0d8f2f8703", + "traincheck/invariant/relation_pool.py": "8ac568cf1924162a791728ed5db8c41868a550f9e0cd5190fc9a19827f55d3e6", + "traincheck/invariant/symbolic_value.py": "28627e2c2a47a146087a96510f07f7deb8463c46371e418afa3fdeff33c7e6cb", + "traincheck/invariant/var_periodic_change_relation.py": "8086df3b9cfe34229b8bd46b18dce22a4b746902e38d1ba4bd02bc6f6c2f2ef4", + "traincheck/onlinechecker/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/onlinechecker/streamhandler_filesystem.py": "b5a298bd06a9408e857ff6721578cb4ae10f227762ba3d66aec685e574cfa511", + "traincheck/onlinechecker/utils.py": "65ba5d8a84b748adf4267800e704df1f4662c0f483534289af20096bf9a78a00", + "traincheck/parallel.py": "a431d169022264109c56b7057ebcdc8a587b698a8670a1257b765189d9e5bf76", + "traincheck/progress.py": "aab7d040324e283c8474aa82bc7550a777aec316131a26e1dc1b6a277333d1ae", + "traincheck/reporting/__init__.py": "299180153357c4e6f48cf75191e915d70cfca5ae6e7b3ac4d45144066b93ddca", + "traincheck/reporting/checker_report.py": "0d59b1753fc348701b49a0eb74e4b1d849a47d8234005ccbbb26e09e3cd0c808", + "traincheck/runner/__init__.py": "287ffa9cab82f01da0e82faea77589dc7d348710c862496458d2d60c364394ba", + "traincheck/runner/runner.py": "89027b488ded8640fddb387dd80d98f059cf40be5f94863966b259acea1da7c7", + "traincheck/script/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/script/num_invs_each_rel.py": "202f13d536929c52defa84cdce00dec2251ea09aabf3ae8ae57067c32d0b844d", + "traincheck/static_analyzer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/config.py": "840460167bd6d26db429076cd4dc227e6c17c4e5fb157172041c48574f44edf7", + "traincheck/static_analyzer/func_level/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/graph_generator/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/graph_generator/analyzer.py": "035684eb7ea4edf8ee0dbf4c6b12b91b428cd6dd57be5fe658253c21c8c420b1", + "traincheck/static_analyzer/graph_generator/anutils.py": "9e62b7ce5fced45a033b76de3d408bdff2fd8561ffa28bc58de82e4105b51d39", + "traincheck/static_analyzer/graph_generator/call_graph_parser.py": "69e2555a1595078a2d2bca174de32cd7d4b0f023c0a20b68a01ca00d1d3b8647", + "traincheck/static_analyzer/graph_generator/main.py": "a568488c50383a67a6ab8feecaf96e29b288861c1dce17481a1b5032d20748a4", + "traincheck/static_analyzer/graph_generator/node.py": "b6a0cea44a28d33f28fee79b7f7073e69914e95565f7a8d62f9e5a5fd6b240cd", + "traincheck/toolkit/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/toolkit/analyze_invariant.py": "706cf8c06062b6df3f1f86691f588c91696cbef6c27698b2f0b290ad8f53ff4b", + "traincheck/toolkit/analyze_trace.py": "48dfc81ef85d96678b5fe466b86ba80d892ecfc497387773fa7ab5995f22e822", + "traincheck/toolkit/detect_anomaly_from_trace_diff.py": "2ffc0723dd3709ccab09cb9b58d76c7bb0558dda09b66c47184e335b815cedd3", + "traincheck/trace/__init__.py": "ec84edd0ff32906c20956166a51b85b3847e22e48432c6a31698080cafbcbc2b", + "traincheck/trace/trace.py": "c4f6fae85de0a9d95591ab88d52ad8acf24f052f42978ec381a53042e1e49e67", + "traincheck/trace/trace_dict.py": "3aec874ef0ce2c34f6332dface604d5f79965976fd3d1c2b692f746f2af4e703", + "traincheck/trace/trace_pandas.py": "953d9cac029f83e015eb22e3bc9c8abc7b63919a0eef18b9794517392117ec24", + "traincheck/trace/trace_polars.py": "64136efcd60a851b0391e708108570b259c2288db6242564b636f69abd1b7b0e", + "traincheck/trace/types.py": "96c1b137f3d72fb05dcf806a4e7ea21fe453ae9cbc3e08352c55e95973fe96e0", + "traincheck/trace/utils.py": "6356156e4c5bbb6c111140302660f36ad1c126e5bc977bd55c81f097022102a3", + "traincheck/utils.py": "71034730f74a15978d37dc9fc9b748133389a69e0ea7f9c9109dd1bc573fd272" + }, + "source_archive_sha256": "b945a9d4c2b43def8481660bc3fc6ae45dbfbd34e3a5e8a0c848e5ca379276e6", + "source_commit": "3232887a8bdb6dbf6abcab233ead43a9697f0c18", + "source_tree": "cb74fdd3a8185deb0b3d31c0d20d2a7eac0b8097" + } + }, + "validation": { + "cases": [ + { + "label": "mnist-combined-r1", + "logical_hash_input_bytes": 67193392, + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "trace_bytes": { + "API": 363634, + "VAR": 55795 + }, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ], + "variant": "baseline", + "workload": "mnist" + }, + { + "label": "mnist-combined-r1", + "logical_hash_input_bytes": 67193392, + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "trace_bytes": { + "API": 363634, + "VAR": 55795 + }, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ], + "variant": "candidate", + "workload": "mnist" + }, + { + "label": "mnist-combined-r2", + "logical_hash_input_bytes": 67193392, + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "trace_bytes": { + "API": 363634, + "VAR": 55795 + }, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ], + "variant": "candidate", + "workload": "mnist" + }, + { + "label": "mnist-combined-r2", + "logical_hash_input_bytes": 67193392, + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "trace_bytes": { + "API": 363634, + "VAR": 55795 + }, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ], + "variant": "baseline", + "workload": "mnist" + }, + { + "label": "mnist-combined-r3", + "logical_hash_input_bytes": 67193392, + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "trace_bytes": { + "API": 363634, + "VAR": 55795 + }, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ], + "variant": "baseline", + "workload": "mnist" + }, + { + "label": "mnist-combined-r3", + "logical_hash_input_bytes": 67193392, + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "trace_bytes": { + "API": 363634, + "VAR": 55795 + }, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ], + "variant": "candidate", + "workload": "mnist" + }, + { + "label": "mnist-combined-r4", + "logical_hash_input_bytes": 67193392, + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "trace_bytes": { + "API": 363634, + "VAR": 55795 + }, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ], + "variant": "candidate", + "workload": "mnist" + }, + { + "label": "mnist-combined-r4", + "logical_hash_input_bytes": 67193392, + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "trace_bytes": { + "API": 363634, + "VAR": 55795 + }, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ], + "variant": "baseline", + "workload": "mnist" + }, + { + "label": "mnist-combined-r5", + "logical_hash_input_bytes": 67193392, + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "trace_bytes": { + "API": 363634, + "VAR": 55795 + }, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ], + "variant": "baseline", + "workload": "mnist" + }, + { + "label": "mnist-combined-r5", + "logical_hash_input_bytes": 67193392, + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "trace_bytes": { + "API": 363634, + "VAR": 55795 + }, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ], + "variant": "candidate", + "workload": "mnist" + }, + { + "label": "mnist-combined-r1-profile", + "logical_hash_input_bytes": 67193392, + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "trace_bytes": { + "API": 363634, + "VAR": 55795 + }, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ], + "variant": "baseline", + "workload": "mnist" + }, + { + "label": "mnist-combined-r1-profile", + "logical_hash_input_bytes": 67193392, + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "trace_bytes": { + "API": 363634, + "VAR": 55795 + }, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ], + "variant": "candidate", + "workload": "mnist" + }, + { + "label": "transformer-combined-r1", + "logical_hash_input_bytes": 214496, + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 1034378, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "variant": "baseline", + "workload": "transformer" + }, + { + "label": "transformer-combined-r1", + "logical_hash_input_bytes": 214496, + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 1034378, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "variant": "candidate", + "workload": "transformer" + }, + { + "label": "transformer-combined-r2", + "logical_hash_input_bytes": 214496, + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 1034378, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "variant": "candidate", + "workload": "transformer" + }, + { + "label": "transformer-combined-r2", + "logical_hash_input_bytes": 214496, + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 1034378, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "variant": "baseline", + "workload": "transformer" + }, + { + "label": "transformer-combined-r3", + "logical_hash_input_bytes": 214496, + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 1034378, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "variant": "baseline", + "workload": "transformer" + }, + { + "label": "transformer-combined-r3", + "logical_hash_input_bytes": 214496, + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 1034378, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "variant": "candidate", + "workload": "transformer" + }, + { + "label": "transformer-combined-r4", + "logical_hash_input_bytes": 214496, + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 1034378, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "variant": "candidate", + "workload": "transformer" + }, + { + "label": "transformer-combined-r4", + "logical_hash_input_bytes": 214496, + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 1034378, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "variant": "baseline", + "workload": "transformer" + }, + { + "label": "transformer-combined-r5", + "logical_hash_input_bytes": 214496, + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 1034378, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "variant": "baseline", + "workload": "transformer" + }, + { + "label": "transformer-combined-r5", + "logical_hash_input_bytes": 214496, + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 1034378, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "variant": "candidate", + "workload": "transformer" + }, + { + "label": "transformer-combined-r1-profile", + "logical_hash_input_bytes": 214496, + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 1034378, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "variant": "baseline", + "workload": "transformer" + }, + { + "label": "transformer-combined-r1-profile", + "logical_hash_input_bytes": 214496, + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 1034378, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "variant": "candidate", + "workload": "transformer" + } + ], + "exact_model_loss_rng_parity": true, + "source_and_raw_trace_hashes_verified": true + }, + "workloads": { + "mnist": { + "coverage": { + "logical_hash_input_bytes": 67193392, + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ] + }, + "median_reduction_percent_of_baseline_training": 8.67150236741045, + "median_reduction_seconds": 0.07534057646989822, + "paired_reduction_percent": [ + 13.675527655335843, + 16.865382859319496, + 4.564578664356592, + 12.20544322789504, + -1.9939121206708794 + ], + "paired_reductions_seconds": [ + 0.12899113073945045, + 0.14653143286705017, + 0.037951767444610596, + 0.11024190858006477, + -0.016614548861980438 + ], + "profiles": { + "baseline": { + "interpretation": "Separate cProfile run; cumulative rows overlap and are not an exclusive breakdown of ordinary timing.", + "rows": [ + { + "calls": 1503363, + "cumulative_seconds": 0.171535822, + "file": "~", + "function": "", + "line": 0, + "percent_of_profile_total": 10.72127954899098, + "self_seconds": 0.099762033 + }, + { + "calls": 211616, + "cumulative_seconds": 0.087001201, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "__enter__", + "line": 466, + "percent_of_profile_total": 5.43772248935242, + "self_seconds": 0.078316964 + }, + { + "calls": 260, + "cumulative_seconds": 0.60086757, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/tracer.py", + "function": "to_dict_args_kwargs", + "line": 160, + "percent_of_profile_total": 37.555241318008235, + "self_seconds": 0.000341501 + }, + { + "calls": 112, + "cumulative_seconds": 0.44745846, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/dumper.py", + "function": "dump_tensor", + "line": 302, + "percent_of_profile_total": 27.96691198542191, + "self_seconds": 0.009669434000000001 + }, + { + "calls": 211616, + "cumulative_seconds": 0.626706756, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/dumper.py", + "function": "safe_getattr", + "line": 328, + "percent_of_profile_total": 39.17023422849416, + "self_seconds": 0.18447759700000002 + }, + { + "calls": 112, + "cumulative_seconds": 0.38490406600000004, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/proxy_wrapper/hash.py", + "function": "hash_tensor_cpu", + "line": 47, + "percent_of_profile_total": 24.0571563596161, + "self_seconds": 0.175302747 + }, + { + "calls": 112, + "cumulative_seconds": 0.437784751, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/proxy_wrapper/hash.py", + "function": "tensor_hash", + "line": 95, + "percent_of_profile_total": 27.362288780453152, + "self_seconds": 0.015663134000000002 + }, + { + "calls": 468, + "cumulative_seconds": 1.537323336, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/dumper.py", + "function": "convert_var_to_dict", + "line": 343, + "percent_of_profile_total": 96.08531355301047, + "self_seconds": 0.22469802900000002 + }, + { + "calls": 64, + "cumulative_seconds": 0.6535382190000001, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/proxy_wrapper/subclass.py", + "function": "dump_trace", + "line": 183, + "percent_of_profile_total": 40.84724613293129, + "self_seconds": 0.00018410100000000002 + }, + { + "calls": 1780, + "cumulative_seconds": 0.00043699300000000005, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/utils.py", + "function": "safe_getattr", + "line": 21, + "percent_of_profile_total": 0.027312803001300897, + "self_seconds": 0.00023170500000000002 + }, + { + "calls": 584, + "cumulative_seconds": 0.004350003, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/dumper.py", + "function": "serialize", + "line": 85, + "percent_of_profile_total": 0.27188255874594763, + "self_seconds": 0.00040800200000000003 + }, + { + "calls": 520, + "cumulative_seconds": 0.006313733, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/dumper.py", + "function": "dump_trace_API", + "line": 241, + "percent_of_profile_total": 0.394619011361309, + "self_seconds": 0.00042752400000000004 + }, + { + "calls": 64, + "cumulative_seconds": 0.000849358, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/dumper.py", + "function": "dump_trace_VAR", + "line": 250, + "percent_of_profile_total": 0.05308631426951673, + "self_seconds": 6.1226e-05 + }, + { + "calls": 260, + "cumulative_seconds": 0.291809378, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/tracer.py", + "function": "to_dict_return_value", + "line": 184, + "percent_of_profile_total": 18.238580607117616, + "self_seconds": 0.0001231 + }, + { + "calls": 203335, + "cumulative_seconds": 0.085047091, + "file": "/home/yuxuan/experiments/codex-inference-runtime-20261025/venv/lib/python3.11/site-packages/torch/nn/parameter.py", + "function": "__instancecheck__", + "line": 21, + "percent_of_profile_total": 5.315587303038517, + "self_seconds": 0.060003436 + }, + { + "calls": 211616, + "cumulative_seconds": 0.066681273, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "__exit__", + "line": 487, + "percent_of_profile_total": 4.167692556459633, + "self_seconds": 0.058937562000000006 + }, + { + "calls": 211616, + "cumulative_seconds": 0.158621435, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "_add_filter", + "line": 181, + "percent_of_profile_total": 9.914108477569789, + "self_seconds": 0.08925611400000001 + }, + { + "calls": 211616, + "cumulative_seconds": 0.032625206000000004, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "__init__", + "line": 440, + "percent_of_profile_total": 2.039130659655555, + "self_seconds": 0.032625206000000004 + }, + { + "calls": 211616, + "cumulative_seconds": 0.227450228, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "simplefilter", + "line": 165, + "percent_of_profile_total": 14.216024673083945, + "self_seconds": 0.061110547 + } + ], + "sha256": "4b33291b26a436e50fa02aa5d32713c8ff6552284184829c3c94dfab103d2a94", + "total_seconds": 1.5999566209999987 + }, + "candidate": { + "interpretation": "Separate cProfile run; cumulative rows overlap and are not an exclusive breakdown of ordinary timing.", + "rows": [ + { + "calls": 464357, + "cumulative_seconds": 0.020462695000000003, + "file": "~", + "function": "", + "line": 0, + "percent_of_profile_total": 1.5475891033585303, + "self_seconds": 0.018717590000000003 + }, + { + "calls": 211616, + "cumulative_seconds": 0.08338677300000001, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "__enter__", + "line": 466, + "percent_of_profile_total": 6.306523224777151, + "self_seconds": 0.07591249800000001 + }, + { + "calls": 260, + "cumulative_seconds": 0.466124882, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/tracer.py", + "function": "to_dict_args_kwargs", + "line": 160, + "percent_of_profile_total": 35.25292187502578, + "self_seconds": 0.00034057300000000005 + }, + { + "calls": 112, + "cumulative_seconds": 0.419081605, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/dumper.py", + "function": "dump_tensor", + "line": 318, + "percent_of_profile_total": 31.695049225725334, + "self_seconds": 0.004339631 + }, + { + "calls": 211616, + "cumulative_seconds": 0.60530919, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/dumper.py", + "function": "safe_getattr", + "line": 344, + "percent_of_profile_total": 45.77940034813489, + "self_seconds": 0.177432474 + }, + { + "calls": 112, + "cumulative_seconds": 0.365371013, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/proxy_wrapper/hash.py", + "function": "hash_tensor_cpu", + "line": 47, + "percent_of_profile_total": 27.63292901555088, + "self_seconds": 0.168094142 + }, + { + "calls": 112, + "cumulative_seconds": 0.414737693, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/proxy_wrapper/hash.py", + "function": "tensor_hash", + "line": 95, + "percent_of_profile_total": 31.366520120583107, + "self_seconds": 0.013457708 + }, + { + "calls": 468, + "cumulative_seconds": 1.257303035, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/dumper.py", + "function": "convert_var_to_dict", + "line": 362, + "percent_of_profile_total": 95.08955084291725, + "self_seconds": 0.14695986900000002 + }, + { + "calls": 64, + "cumulative_seconds": 0.574950325, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/proxy_wrapper/subclass.py", + "function": "dump_trace", + "line": 183, + "percent_of_profile_total": 43.483366093393144, + "self_seconds": 0.000187755 + }, + { + "calls": 1780, + "cumulative_seconds": 0.00045410000000000003, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/utils.py", + "function": "safe_getattr", + "line": 21, + "percent_of_profile_total": 0.03434348270524037, + "self_seconds": 0.000239024 + }, + { + "calls": 584, + "cumulative_seconds": 0.004386208, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/dumper.py", + "function": "serialize", + "line": 101, + "percent_of_profile_total": 0.33172794228052616, + "self_seconds": 0.00041095600000000003 + }, + { + "calls": 64, + "cumulative_seconds": 0.000981751, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/dumper.py", + "function": "dump_trace_VAR", + "line": 266, + "percent_of_profile_total": 0.0742496112956451, + "self_seconds": 6.239800000000001e-05 + }, + { + "calls": 520, + "cumulative_seconds": 0.006941215000000001, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/dumper.py", + "function": "dump_trace_API", + "line": 257, + "percent_of_profile_total": 0.5249625573791126, + "self_seconds": 0.00043592500000000004 + }, + { + "calls": 260, + "cumulative_seconds": 0.22543626700000002, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/tracer.py", + "function": "to_dict_return_value", + "line": 184, + "percent_of_profile_total": 17.04969508224719, + "self_seconds": 0.000130289 + }, + { + "calls": 2817, + "cumulative_seconds": 0.0020264320000000003, + "file": "/home/yuxuan/experiments/codex-inference-runtime-20261025/venv/lib/python3.11/site-packages/torch/nn/parameter.py", + "function": "__instancecheck__", + "line": 21, + "percent_of_profile_total": 0.15325860459226084, + "self_seconds": 0.001462605 + }, + { + "calls": 211616, + "cumulative_seconds": 0.064519857, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "__exit__", + "line": 487, + "percent_of_profile_total": 4.879622534737021, + "self_seconds": 0.057657651000000004 + }, + { + "calls": 211616, + "cumulative_seconds": 0.15187645200000002, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "_add_filter", + "line": 181, + "percent_of_profile_total": 11.486382520579761, + "self_seconds": 0.08526881400000001 + }, + { + "calls": 211616, + "cumulative_seconds": 0.031807165000000005, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "__init__", + "line": 440, + "percent_of_profile_total": 2.4055688638630848, + "self_seconds": 0.031807165000000005 + }, + { + "calls": 211616, + "cumulative_seconds": 0.21994069500000002, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "simplefilter", + "line": 165, + "percent_of_profile_total": 16.634066185666256, + "self_seconds": 0.059943076000000005 + } + ], + "sha256": "7862ce8c96dc80a4a95a25f1214a5e32b733419ba3b32082b301ce91b1b4731e", + "total_seconds": 1.3222304910000007 + } + }, + "speedup_ratio_of_medians": 1.0949484836846382, + "variants": { + "baseline": { + "peak_RSS_bytes": { + "max": 592883712.0, + "median": 588767232.0, + "min": 584212480.0, + "values": [ + 584212480.0, + 588173312.0, + 592883712.0, + 588767232.0, + 590761984.0 + ] + }, + "post_metrics_exit_seconds": { + "max": 2.17674096301198, + "median": 2.077751439064741, + "min": 2.0082851946353912, + "values": [ + 2.1645433455705643, + 2.076347693800926, + 2.17674096301198, + 2.0082851946353912, + 2.077751439064741 + ] + }, + "process_seconds": { + "max": 7.2223930433392525, + "median": 7.0179259441792965, + "min": 6.817594639956951, + "values": [ + 7.2223930433392525, + 7.0179259441792965, + 7.1210783533751965, + 6.817594639956951, + 6.91872750595212 + ] + }, + "setup_seconds": { + "max": 2.778099913150072, + "median": 2.702235735952854, + "min": 2.592470161616802, + "values": [ + 2.778099913150072, + 2.702235735952854, + 2.7643737457692623, + 2.592470161616802, + 2.694968421012163 + ] + }, + "trace_bytes": { + "max": 419429.0, + "median": 419429.0, + "min": 419429.0, + "values": [ + 419429.0, + 419429.0, + 419429.0, + 419429.0, + 419429.0 + ] + }, + "training_seconds": { + "max": 0.9432259872555733, + "median": 0.8688295669853687, + "min": 0.8314407579600811, + "values": [ + 0.9432259872555733, + 0.8688295669853687, + 0.8314407579600811, + 0.9032192155718803, + 0.833263847976923 + ] + } + }, + "candidate": { + "peak_RSS_bytes": { + "max": 588763136.0, + "median": 588550144.0, + "min": 582574080.0, + "values": [ + 588550144.0, + 588550144.0, + 588763136.0, + 582574080.0, + 588165120.0 + ] + }, + "post_metrics_exit_seconds": { + "max": 2.1754636578261852, + "median": 2.1305285543203354, + "min": 2.0753588266670704, + "values": [ + 2.1305285543203354, + 2.1754636578261852, + 2.0753588266670704, + 2.13845232501626, + 2.092308398336172 + ] + }, + "process_seconds": { + "max": 7.326598588377237, + "median": 7.019078567624092, + "min": 6.816429901868105, + "values": [ + 7.124290060251951, + 7.019078567624092, + 7.326598588377237, + 6.818062514066696, + 6.816429901868105 + ] + }, + "setup_seconds": { + "max": 2.9761409126222134, + "median": 2.70905052497983, + "min": 2.5668062567710876, + "values": [ + 2.70905052497983, + 2.7149592004716396, + 2.9761409126222134, + 2.5683862902224064, + 2.5668062567710876 + ] + }, + "trace_bytes": { + "max": 419429.0, + "median": 419429.0, + "min": 419429.0, + "values": [ + 419429.0, + 419429.0, + 419429.0, + 419429.0, + 419429.0 + ] + }, + "training_seconds": { + "max": 0.8498783968389034, + "median": 0.7934889905154705, + "min": 0.7222981341183186, + "values": [ + 0.8142348565161228, + 0.7222981341183186, + 0.7934889905154705, + 0.7929773069918156, + 0.8498783968389034 + ] + } + } + }, + "workload_manifest": { + "adapter_sha256": "a5b5e597d4921a5b4baceeada541a6a1765f13c97ab3bd1c2ef79dea331f0934", + "api_disabled_ablation": "var mode keeps Instrumentor wrappers with funcs_to_instr=[]; uses production no-dump fast path", + "batch_size": 2, + "benchmark_git_head": null, + "benchmark_source": "/home/yuxuan/experiments/codex-trace-impact-20261013/benchmarks/instr-overhead/overhead-e2e/mnist/main.py", + "benchmark_source_sha256": "4478c5de92610a1e03d61c356801d51ffca749f99dac00f234d6b6ea2bcb2aa7", + "cpu_threads": 1, + "data": { + "count": 6, + "files": [ + { + "path": "/home/yuxuan/gitrepos/data/MNIST/raw/train-images-idx3-ubyte", + "sha256": "ba891046e6505d7aadcbbe25680a0738ad16aec93bde7f9b65e87a2fc25776db" + }, + { + "path": "/home/yuxuan/gitrepos/data/MNIST/raw/train-labels-idx1-ubyte", + "sha256": "65a50cbbf4e906d70832878ad85ccda5333a97f0f4c3dd2ef09a8a9eef7101c5" + } + ], + "kind": "cached_mnist_training_prefix", + "normalization": "(uint8 / 255 - 0.1307) / 0.3081", + "ordering": "first samples in IDX order; no shuffle" + }, + "deterministic_algorithms": true, + "device": "cpu", + "excluded_phases": [ + "model initialization", + "data preparation", + "metric extraction", + "checkpoint writing" + ], + "extracted_definitions_sha256": "f839142a57ce449054e05ffa79d746e5282e2599e101810e6ccf6e7a9dd33ce5", + "extracted_symbols": [ + "Net" + ], + "initial_state_hash": "877e9abeb783ae5ed703a9666a7276e921e0424b78bb2d50a41461221d90ddb0", + "mode": "combined", + "model_config": { + "architecture": "exact Net class" + }, + "model_tracker_style": "subclass", + "models_to_track": [ + "model" + ], + "observation_files": [ + { + "path": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/static_analyzer/func_level/nn_func_level.log", + "sha256": "0ea56ca02089d5b19a1c0697dea00d107fccd18e17d485ff97bc04a5378dfcba" + }, + { + "path": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/static_analyzer/func_level/optim_func_level.log", + "sha256": "300c47f9babd033ddf68487dde01a828746ff795b5f0111761b2f8fda08ad900" + } + ], + "plain_baseline": "Imports TrainCheck (including global Module.__setattr__ hook), but no API wrappers or proxied parameters; imports/setup excluded", + "precision": "float32", + "profiled": false, + "python": "3.11.4 (main, Jul 5 2023, 13:45:01) [GCC 11.2.0]", + "scan_proxy_in_args": true, + "seed": 42, + "setup_seconds": 2.778099913150072, + "source_commit": "88b6df2401e187ca6bd81bf0a83bc7c5d9a5cd54", + "source_tree": "5415a3b0e5eb08d119dffa91631d76866e6e1d11", + "steps": 3, + "tensor_dump_format": { + "dump_tensor_full": false, + "dump_tensor_hash": true, + "dump_tensor_stats": false + }, + "timing": "Cold three-step CPU training body including start_step and wrapper-disable assignment; excludes setup/loss extraction/state digest/drain", + "topology": "Static observers only in VAR-enabled modes, matching normal collect_trace activation; interaction is not assumed zero", + "torch_version": "2.16.0.dev20261012+cpu", + "tracked_parameter_names": [ + "conv1.weight", + "conv1.bias", + "conv2.weight", + "conv2.bias", + "fc1.weight", + "fc1.bias", + "fc2.weight", + "fc2.bias" + ], + "training_only_trace": true, + "update": "Exact benchmark forward, NLL loss, backward, Adadelta(lr=1.0). Epoch scheduler/evaluation omitted.", + "variant": "baseline_subclass_fix", + "workload_kind": "bounded_model_micro_workload", + "wrapped_functions": 21032 + } + }, + "transformer": { + "coverage": { + "logical_hash_input_bytes": 214496, + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ] + }, + "median_reduction_percent_of_baseline_training": 18.90125826253589, + "median_reduction_seconds": 0.18266938999295235, + "paired_reduction_percent": [ + 19.113870461709478, + 17.331180173553964, + 18.858957322850234, + 19.979830753413463, + 23.81235705577393 + ], + "paired_reductions_seconds": [ + 0.1831599548459053, + 0.16592683270573616, + 0.18226057663559914, + 0.19419122487306595, + 0.24496668577194214 + ], + "profiles": { + "baseline": { + "interpretation": "Separate cProfile run; cumulative rows overlap and are not an exclusive breakdown of ordinary timing.", + "rows": [ + { + "calls": 3622491, + "cumulative_seconds": 0.427562979, + "file": "~", + "function": "", + "line": 0, + "percent_of_profile_total": 15.08458680801047, + "self_seconds": 0.24889274400000003 + }, + { + "calls": 516009, + "cumulative_seconds": 0.216041698, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "__enter__", + "line": 466, + "percent_of_profile_total": 7.622034431636286, + "self_seconds": 0.19456483700000002 + }, + { + "calls": 670, + "cumulative_seconds": 1.5786948920000001, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/tracer.py", + "function": "to_dict_args_kwargs", + "line": 160, + "percent_of_profile_total": 55.69696468443942, + "self_seconds": 0.0008960670000000001 + }, + { + "calls": 64, + "cumulative_seconds": 0.005761861, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/proxy_wrapper/hash.py", + "function": "tensor_hash", + "line": 95, + "percent_of_profile_total": 0.20328067840080707, + "self_seconds": 0.000760977 + }, + { + "calls": 64, + "cumulative_seconds": 0.004656917, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/proxy_wrapper/hash.py", + "function": "hash_tensor_cpu", + "line": 47, + "percent_of_profile_total": 0.16429782790946382, + "self_seconds": 0.004139646 + }, + { + "calls": 516009, + "cumulative_seconds": 1.579442368, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/dumper.py", + "function": "safe_getattr", + "line": 328, + "percent_of_profile_total": 55.72333592601716, + "self_seconds": 0.46544991100000005 + }, + { + "calls": 1773, + "cumulative_seconds": 2.748502059, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/dumper.py", + "function": "convert_var_to_dict", + "line": 343, + "percent_of_profile_total": 96.96821272493993, + "self_seconds": 0.559197797 + }, + { + "calls": 64, + "cumulative_seconds": 0.005844259, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/dumper.py", + "function": "dump_tensor", + "line": 302, + "percent_of_profile_total": 0.20618771162130123, + "self_seconds": 7.9787e-05 + }, + { + "calls": 64, + "cumulative_seconds": 0.213352694, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/proxy_wrapper/subclass.py", + "function": "dump_trace", + "line": 183, + "percent_of_profile_total": 7.527165333380968, + "self_seconds": 0.00015941500000000002 + }, + { + "calls": 8292, + "cumulative_seconds": 0.002800037, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/utils.py", + "function": "safe_getattr", + "line": 21, + "percent_of_profile_total": 0.09878638532018745, + "self_seconds": 0.0011357350000000001 + }, + { + "calls": 1340, + "cumulative_seconds": 0.01722223, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/dumper.py", + "function": "dump_trace_API", + "line": 241, + "percent_of_profile_total": 0.6076069169274877, + "self_seconds": 0.00117887 + }, + { + "calls": 64, + "cumulative_seconds": 0.000883031, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/dumper.py", + "function": "dump_trace_VAR", + "line": 250, + "percent_of_profile_total": 0.03115367426061529, + "self_seconds": 4.7865e-05 + }, + { + "calls": 670, + "cumulative_seconds": 0.987921398, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/tracer.py", + "function": "to_dict_return_value", + "line": 184, + "percent_of_profile_total": 34.854247957754225, + "self_seconds": 0.00032125000000000003 + }, + { + "calls": 1404, + "cumulative_seconds": 0.012092690000000001, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/dumper.py", + "function": "serialize", + "line": 85, + "percent_of_profile_total": 0.42663476728971, + "self_seconds": 0.001008543 + }, + { + "calls": 495303, + "cumulative_seconds": 0.21179952300000002, + "file": "/home/yuxuan/experiments/codex-inference-runtime-20261025/venv/lib/python3.11/site-packages/torch/nn/parameter.py", + "function": "__instancecheck__", + "line": 21, + "percent_of_profile_total": 7.472368861450725, + "self_seconds": 0.150287937 + }, + { + "calls": 1, + "cumulative_seconds": 1.322e-06, + "file": "/home/yuxuan/experiments/codex-inference-runtime-20261025/venv/lib/python3.11/site-packages/torch/nn/parameter.py", + "function": "__instancecheck__", + "line": 240, + "percent_of_profile_total": 4.6640669888750686e-05, + "self_seconds": 1.002e-06 + }, + { + "calls": 516009, + "cumulative_seconds": 0.170823455, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "__exit__", + "line": 487, + "percent_of_profile_total": 6.0267173781473975, + "self_seconds": 0.15120122700000002 + }, + { + "calls": 516009, + "cumulative_seconds": 0.406155918, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "_add_filter", + "line": 181, + "percent_of_profile_total": 14.329337439334713, + "self_seconds": 0.22709152400000002 + }, + { + "calls": 516009, + "cumulative_seconds": 0.082309591, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "__init__", + "line": 440, + "percent_of_profile_total": 2.903914116875253, + "self_seconds": 0.082309591 + }, + { + "calls": 516009, + "cumulative_seconds": 0.579480675, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "simplefilter", + "line": 165, + "percent_of_profile_total": 20.444301716781712, + "self_seconds": 0.153575495 + } + ], + "sha256": "f7b2024e12967a7cc232ee63460a2fb4a2860979a48225bc638982993aa31cd9", + "total_seconds": 2.834436132999999 + }, + "candidate": { + "interpretation": "Separate cProfile run; cumulative rows overlap and are not an exclusive breakdown of ordinary timing.", + "rows": [ + { + "calls": 1154245, + "cumulative_seconds": 0.052964309, + "file": "~", + "function": "", + "line": 0, + "percent_of_profile_total": 2.4369284236685465, + "self_seconds": 0.047591463 + }, + { + "calls": 516009, + "cumulative_seconds": 0.203956469, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "__enter__", + "line": 466, + "percent_of_profile_total": 9.384193353625603, + "self_seconds": 0.18543056800000002 + }, + { + "calls": 64, + "cumulative_seconds": 0.005312453, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/proxy_wrapper/hash.py", + "function": "tensor_hash", + "line": 95, + "percent_of_profile_total": 0.24443003145954836, + "self_seconds": 0.000703901 + }, + { + "calls": 64, + "cumulative_seconds": 0.005392115, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/dumper.py", + "function": "dump_tensor", + "line": 318, + "percent_of_profile_total": 0.24809534109450057, + "self_seconds": 7.7172e-05 + }, + { + "calls": 64, + "cumulative_seconds": 0.004293179, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/proxy_wrapper/hash.py", + "function": "hash_tensor_cpu", + "line": 47, + "percent_of_profile_total": 0.19753245403422345, + "self_seconds": 0.003818175 + }, + { + "calls": 516009, + "cumulative_seconds": 1.4984058370000002, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/dumper.py", + "function": "safe_getattr", + "line": 344, + "percent_of_profile_total": 68.9428002237537, + "self_seconds": 0.437565478 + }, + { + "calls": 670, + "cumulative_seconds": 1.207365155, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/tracer.py", + "function": "to_dict_args_kwargs", + "line": 160, + "percent_of_profile_total": 55.5517955302028, + "self_seconds": 0.0008465870000000001 + }, + { + "calls": 1773, + "cumulative_seconds": 2.0914098190000003, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/dumper.py", + "function": "convert_var_to_dict", + "line": 362, + "percent_of_profile_total": 96.22736763091898, + "self_seconds": 0.363128973 + }, + { + "calls": 64, + "cumulative_seconds": 0.162592268, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/proxy_wrapper/subclass.py", + "function": "dump_trace", + "line": 183, + "percent_of_profile_total": 7.480994783825724, + "self_seconds": 0.00014322 + }, + { + "calls": 8292, + "cumulative_seconds": 0.002711686, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/utils.py", + "function": "safe_getattr", + "line": 21, + "percent_of_profile_total": 0.1247667498024767, + "self_seconds": 0.0011258210000000002 + }, + { + "calls": 64, + "cumulative_seconds": 0.000922073, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/dumper.py", + "function": "dump_trace_VAR", + "line": 266, + "percent_of_profile_total": 0.04242528496685056, + "self_seconds": 4.7424000000000005e-05 + }, + { + "calls": 1340, + "cumulative_seconds": 0.016164994000000002, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/dumper.py", + "function": "dump_trace_API", + "line": 257, + "percent_of_profile_total": 0.7437637550795105, + "self_seconds": 0.001122561 + }, + { + "calls": 670, + "cumulative_seconds": 0.752377733, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/tracer.py", + "function": "to_dict_return_value", + "line": 184, + "percent_of_profile_total": 34.61747575868505, + "self_seconds": 0.00031704700000000004 + }, + { + "calls": 1404, + "cumulative_seconds": 0.011245191, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/dumper.py", + "function": "serialize", + "line": 101, + "percent_of_profile_total": 0.517399850859599, + "self_seconds": 0.0009738310000000001 + }, + { + "calls": 8937, + "cumulative_seconds": 0.0062934160000000005, + "file": "/home/yuxuan/experiments/codex-inference-runtime-20261025/venv/lib/python3.11/site-packages/torch/nn/parameter.py", + "function": "__instancecheck__", + "line": 21, + "percent_of_profile_total": 0.2895648904316001, + "self_seconds": 0.0044612110000000005 + }, + { + "calls": 1, + "cumulative_seconds": 1.5100000000000002e-06, + "file": "/home/yuxuan/experiments/codex-inference-runtime-20261025/venv/lib/python3.11/site-packages/torch/nn/parameter.py", + "function": "__instancecheck__", + "line": 240, + "percent_of_profile_total": 6.947625654361893e-05, + "self_seconds": 1.2620000000000002e-06 + }, + { + "calls": 516009, + "cumulative_seconds": 0.161353457, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "__exit__", + "line": 487, + "percent_of_profile_total": 7.423996140881979, + "self_seconds": 0.144325115 + }, + { + "calls": 516009, + "cumulative_seconds": 0.381938691, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "_add_filter", + "line": 181, + "percent_of_profile_total": 17.573291708509934, + "self_seconds": 0.213641867 + }, + { + "calls": 516009, + "cumulative_seconds": 0.079526229, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "__init__", + "line": 440, + "percent_of_profile_total": 3.659062707252045, + "self_seconds": 0.079526229 + }, + { + "calls": 516009, + "cumulative_seconds": 0.552660111, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "simplefilter", + "line": 165, + "percent_of_profile_total": 25.428315002159547, + "self_seconds": 0.15112924600000002 + } + ], + "sha256": "a3fd25631d2d65a4819675afba8347fc50c95dbf22a56276f2519f32271cc158", + "total_seconds": 2.1734043759999997 + } + }, + "speedup_ratio_of_medians": 1.2330647536274206, + "variants": { + "baseline": { + "peak_RSS_bytes": { + "max": 540819456.0, + "median": 540479488.0, + "min": 540413952.0, + "values": [ + 540446720.0, + 540524544.0, + 540819456.0, + 540479488.0, + 540413952.0 + ] + }, + "post_metrics_exit_seconds": { + "max": 3.333291094750166, + "median": 2.0391327142715454, + "min": 1.9195968136191368, + "values": [ + 2.0391327142715454, + 1.9493112564086914, + 1.9195968136191368, + 2.38959226757288, + 3.333291094750166 + ] + }, + "process_seconds": { + "max": 8.535529389977455, + "median": 7.0182711854577065, + "min": 6.815501172095537, + "values": [ + 7.0182711854577065, + 6.815501172095537, + 6.817034367471933, + 7.320687089115381, + 8.535529389977455 + ] + }, + "setup_seconds": { + "max": 2.8161566741764545, + "median": 2.602743424475193, + "min": 2.5676101446151733, + "values": [ + 2.6592667177319527, + 2.5893340334296227, + 2.5676101446151733, + 2.602743424475193, + 2.8161566741764545 + ] + }, + "trace_bytes": { + "max": 1088456.0, + "median": 1088456.0, + "min": 1088456.0, + "values": [ + 1088456.0, + 1088456.0, + 1088456.0, + 1088456.0, + 1088456.0 + ] + }, + "training_seconds": { + "max": 1.0287376642227173, + "median": 0.9664403684437275, + "min": 0.9573891162872314, + "values": [ + 0.9582567550241947, + 0.9573891162872314, + 0.9664403684437275, + 0.9719362854957581, + 1.0287376642227173 + ] + } + }, + "candidate": { + "peak_RSS_bytes": { + "max": 540966912.0, + "median": 540696576.0, + "min": 540524544.0, + "values": [ + 540622848.0, + 540524544.0, + 540966912.0, + 540696576.0, + 540721152.0 + ] + }, + "post_metrics_exit_seconds": { + "max": 2.2381970062851906, + "median": 2.0395359471440315, + "min": 2.011737398803234, + "values": [ + 2.0347076691687107, + 2.011737398803234, + 2.0395359471440315, + 2.098541881889105, + 2.2381970062851906 + ] + }, + "process_seconds": { + "max": 7.020256042480469, + "median": 6.816576909273863, + "min": 6.71428145095706, + "values": [ + 6.91656219959259, + 6.71591405197978, + 6.71428145095706, + 6.816576909273863, + 7.020256042480469 + ] + }, + "setup_seconds": { + "max": 2.6585870534181595, + "median": 2.6094753071665764, + "min": 2.570909459143877, + "values": [ + 2.6569117195904255, + 2.5945258922874928, + 2.570909459143877, + 2.6094753071665764, + 2.6585870534181595 + ] + }, + "trace_bytes": { + "max": 1088456.0, + "median": 1088456.0, + "min": 1088456.0, + "values": [ + 1088456.0, + 1088456.0, + 1088456.0, + 1088456.0, + 1088456.0 + ] + }, + "training_seconds": { + "max": 0.7914622835814953, + "median": 0.7837709784507751, + "min": 0.7750968001782894, + "values": [ + 0.7750968001782894, + 0.7914622835814953, + 0.7841797918081284, + 0.7777450606226921, + 0.7837709784507751 + ] + } + } + }, + "workload_manifest": { + "adapter_sha256": "a5b5e597d4921a5b4baceeada541a6a1765f13c97ab3bd1c2ef79dea331f0934", + "api_disabled_ablation": "var mode keeps Instrumentor wrappers with funcs_to_instr=[]; uses production no-dump fast path", + "batch_size": 2, + "benchmark_git_head": null, + "benchmark_source": "/home/yuxuan/experiments/codex-trace-impact-20261013/benchmarks/false-positive/RNN/trainset/word_language_model-RNN_TANH-3epo/model.py", + "benchmark_source_sha256": "e4451a33e9b16953004bc639d736d14f42a2b21f2939ccabe4cff10d3c40b74c", + "cpu_threads": 1, + "data": { + "adaptation": "Whitespace tokenization plus ; vocabulary and tensors are bounded to the chosen prefix, unlike the full benchmark corpus.", + "kind": "wikitext2_training_prefix_with_prefix_only_vocabulary", + "path": "/home/yuxuan/experiments/codex-trace-impact-20261013/benchmarks/false-positive/RNN/trainset/word_language_model-RNN_TANH-3epo/data/wikitext-2/train.txt", + "prefix_sha256": "9b0b504c73ee103a9128f2072536e99b2696039399bd57669477f030db10eb35", + "sha256": "9e9fa1ad55b1c2c95b08e37dd8e653f638fac2c6de904b79e813611eefbc985f", + "token_count": 98, + "vocabulary_size": 60 + }, + "deterministic_algorithms": true, + "device": "cpu", + "excluded_phases": [ + "model initialization", + "data preparation", + "metric extraction", + "checkpoint writing" + ], + "extracted_definitions_sha256": "1f2db95454690be6a91aaa141e3a5a246308f9abca11444c9113e4adfb78b95d", + "extracted_symbols": [ + "PositionalEncoding", + "TransformerModel" + ], + "initial_state_hash": "e197d3c2a3f351146bc265c5b3eb54dcd1d921499ce24d1cc61bdc5e6dce886d", + "mode": "combined", + "model_config": { + "clip": 0.25, + "dropout_argument": 0.2, + "lr": 1.0, + "nhead": 2, + "nhid": 64, + "ninp": 32, + "nlayers": 2, + "ntoken": 60, + "sequence_length": 16 + }, + "model_tracker_style": "subclass", + "models_to_track": [ + "model" + ], + "observation_files": [ + { + "path": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/static_analyzer/func_level/nn_func_level.log", + "sha256": "0ea56ca02089d5b19a1c0697dea00d107fccd18e17d485ff97bc04a5378dfcba" + }, + { + "path": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/static_analyzer/func_level/optim_func_level.log", + "sha256": "300c47f9babd033ddf68487dde01a828746ff795b5f0111761b2f8fda08ad900" + } + ], + "plain_baseline": "Imports TrainCheck (including global Module.__setattr__ hook), but no API wrappers or proxied parameters; imports/setup excluded", + "precision": "float32", + "profiled": false, + "python": "3.11.4 (main, Jul 5 2023, 13:45:01) [GCC 11.2.0]", + "scan_proxy_in_args": true, + "seed": 42, + "setup_seconds": 2.6592667177319527, + "source_commit": "88b6df2401e187ca6bd81bf0a83bc7c5d9a5cd54", + "source_tree": "5415a3b0e5eb08d119dffa91631d76866e6e1d11", + "steps": 3, + "tensor_dump_format": { + "dump_tensor_full": false, + "dump_tensor_hash": true, + "dump_tensor_stats": false + }, + "timing": "Cold three-step CPU training body including start_step and wrapper-disable assignment; excludes setup/loss extraction/state digest/drain", + "topology": "Static observers only in VAR-enabled modes, matching normal collect_trace activation; interaction is not assumed zero", + "torch_version": "2.16.0.dev20261012+cpu", + "tracked_parameter_names": [ + "encoder.layers.0.self_attn.in_proj_weight", + "encoder.layers.0.self_attn.in_proj_bias", + "encoder.layers.0.self_attn.out_proj.weight", + "encoder.layers.0.self_attn.out_proj.bias", + "encoder.layers.0.linear1.weight", + "encoder.layers.0.linear1.bias", + "encoder.layers.0.linear2.weight", + "encoder.layers.0.linear2.bias", + "encoder.layers.0.norm1.weight", + "encoder.layers.0.norm1.bias", + "encoder.layers.0.norm2.weight", + "encoder.layers.0.norm2.bias", + "encoder.layers.1.self_attn.in_proj_weight", + "encoder.layers.1.self_attn.in_proj_bias", + "encoder.layers.1.self_attn.out_proj.weight", + "encoder.layers.1.self_attn.out_proj.bias", + "encoder.layers.1.linear1.weight", + "encoder.layers.1.linear1.bias", + "encoder.layers.1.linear2.weight", + "encoder.layers.1.linear2.bias", + "encoder.layers.1.norm1.weight", + "encoder.layers.1.norm1.bias", + "encoder.layers.1.norm2.weight", + "encoder.layers.1.norm2.bias", + "encoder.norm.weight", + "encoder.norm.bias", + "decoder.weight", + "decoder.bias", + "input_emb.weight" + ], + "training_only_trace": true, + "update": "Adapted language-model forward/NLL/backward/clipped manual SGD. Model classes copied exactly; the benchmark main.py training path is RNN-specific and is not executed. Transformer dimensions/vocabulary reduced.", + "variant": "baseline_subclass_fix", + "workload_kind": "bounded_model_micro_workload", + "wrapped_functions": 21032 + } + } + } +} diff --git a/docs/measurements/metadata-classification-20261027.md b/docs/measurements/metadata-classification-20261027.md new file mode 100644 index 00000000..d7d122c7 --- /dev/null +++ b/docs/measurements/metadata-classification-20261027.md @@ -0,0 +1,227 @@ +# Transformer tracker correctness and metadata fast-path measurements + +The fixed subclass tracker completes the previously failing Transformer workload. +In five alternating pairs with **both API and model tracing active**, the metadata +classification candidate reduced median three-step CPU training time by **8.67% +for MNIST** and **18.90% for the small Transformer**. These are reductions from the +baseline's training-body time with both tracers enabled under ordinary collection +and its normal API exclusions (`use_full_instr=False`). They are not whole-process +speedups or comparisons against the earlier PR #153 measurements. + +## Exact comparison and scope + +| Variant | Commit | Tree | +|---|---|---| +| Baseline: subclass assignment fix | `88b6df2401e187ca6bd81bf0a83bc7c5d9a5cd54` | `5415a3b0e5eb08d119dffa91631d76866e6e1d11` | +| Candidate: same fix plus callable metadata fast path | `3232887a8bdb6dbf6abcab233ead43a9697f0c18` | `cb74fdd3a8185deb0b3d31c0d20d2a7eac0b8097` | + +All 77 executed production Python files per variant match the respective exact +Git archive byte for byte. The only differing production file is +`traincheck/instrumentor/dumper.py`. Hash algorithms, hash backend, tensor dump +format, sampling, observer depth, and tracing coverage remain unchanged. + +The performance change recognizes exact built-in callable types before testing +them against framework tensor/module classes. It retains attribute discovery and +warning handling, and retains the generic path for arbitrary callable objects and +subclasses. This study does not change or optimize hashing. + +## Workload and timing contract + +- labmach, Intel Core i9-13900K, Python 3.11.4, PyTorch + `2.16.0.dev20261012+cpu`; one PyTorch/native thread, CPU only, deterministic + algorithms, batch 2, float32, three cold training steps. +- MNIST: exact benchmark `Net`, 1,199,882 parameters, first six cached MNIST + training images, NLL loss and Adadelta with learning rate 1. +- Transformer: exact benchmark model definitions with bounded dimensions + (embedding 32, two layers, two heads, feedforward 64, sequence length 16, + vocabulary 60 from the WikiText-2 prefix). Original gradient clipping and + **manual SGD parameter updates remain present**. +- Initialization seeds 42; training seeds 43 for Python, NumPy and PyTorch. +- Explicit subclass parameter tracking before optimizer construction; normal + depth-3 static observers; parameter hash format enabled, CPU backend; + `scan_proxy_in_args=True`; API descriptors disabled; sampling interval 1, + warmup 0; `use_full_instr=False`, retaining normal API exclusions. This does + not exercise `--use-full-instr`, default Proxy, or VarSampler mode. +- Ordinary timing includes `start_step`, the training body, and disabling + instrumentation on exit. It excludes imports/setup, loss extraction, RNG/state + verification and final background-writer drain. **RNG snapshots are outside + the timer and after `DISABLE_WRAPPER=True`**, so they add no API trace records. + Separate cProfile runs cover the training body, not setup or diagnostics. +- Every case uses a fresh process. Five pairs per workload alternate + baseline/candidate, candidate/baseline, baseline/candidate, candidate/baseline, + baseline/candidate. The shared host was not reserved or pinned to a CPU. +- Limits: 90 seconds, 4 GiB sampled main-process RSS, 256 MiB trace per case. + No case hit a limit. Native threads were restricted to one; other users' work + and shared environments were not changed. + +## Correctness smoke + +The fixed baseline passed plain, API-only, VAR-only and combined Transformer +three-step runs. Initial/final model state, every loss value, and Python/NumPy/ +PyTorch CPU RNG state before training and after each step matched exactly. + +| Mode | Single smoke time (ms) | API records | VAR records | Tracked parameters | +|---|---:|---:|---:|---:| +| Plain | 15.418 | 0 | 0 | 0 | +| API | 877.959 | 1,338 | 0 | 0 | +| VAR | 88.366 | 0 | 64 | 29 | +| Combined | 940.872 | 1,340 | 64 | 29 | + +These smoke times establish execution, not a repeated ablation estimate. +VAR-only keeps Instrumentor's no-dump wrappers. All modes import TrainCheck's +global module-assignment hook; plain installs no API wrappers or parameter +proxies. Modes with VAR tracking also install observers and argument scanning, +so four-mode differences are not a physically exclusive cost partition. + +## Ordinary paired measurements + +| Workload | Baseline median (range), ms | Candidate median (range), ms | Reduction from baseline median | Ratio of medians | +|---|---:|---:|---:|---:| +| MNIST | 868.830 (831.441–943.226) | 793.489 (722.298–849.878) | 75.341 ms / **8.67%** | 1.095× | +| Transformer | 966.440 (957.389–1,028.738) | 783.771 (775.097–791.462) | 182.669 ms / **18.90%** | 1.233× | + +All individual ordinary runs, in repetition order: + +| Workload | Baseline times (ms) | Candidate times (ms) | Paired reductions (ms) | +|---|---|---|---| +| MNIST | 943.226, 868.830, 831.441, 903.219, 833.264 | 814.235, 722.298, 793.489, 792.977, 849.878 | 128.991, 146.531, 37.952, 110.242, **−16.615** | +| Transformer | 958.257, 957.389, 966.440, 971.936, 1,028.738 | 775.097, 791.462, 784.180, 777.745, 783.771 | 183.160, 165.927, 182.261, 194.191, 244.967 | + +MNIST improves in four of five pairs; its paired reductions range from −1.99% +to 16.87%. Transformer improves in all five pairs (17.33%–23.81%). These are +bounded cold-step diagnostics on a shared host, not confidence intervals or +steady-state/GPU/distributed scaling claims. No run was discarded as an outlier. + +## Profiles: what changed and what remains + +Each variant has one separate cProfile run per workload. Total profiled time is +1.599957 → 1.322230 seconds for MNIST and 2.834436 → 2.173404 seconds for +Transformer. Percentages below use **that variant's profiled total**. Cumulative +rows overlap; do not add them or apply them to the ordinary timing denominator. + +| Profile component | MNIST baseline → candidate, ms (% of respective total) | Transformer baseline → candidate, ms (% of respective total) | +|---|---|---| +| All `convert_var_to_dict`, cumulative | 1,537.323 (96.09%) → 1,257.303 (95.09%) | 2,748.502 (96.97%) → 2,091.410 (96.23%) | +| Built-in `isinstance`, cumulative | 171.536 (10.72%) → 20.463 (1.55%) | 427.563 (15.08%) → 52.964 (2.44%) | +| Dumper `safe_getattr`, cumulative | 626.707 (39.17%) → 605.309 (45.78%) | 1,579.442 (55.72%) → 1,498.406 (68.94%) | +| Warning `simplefilter`, cumulative, nested in `safe_getattr` | 227.450 (14.22%) → 219.941 (16.63%) | 579.481 (20.44%) → 552.660 (25.43%) | +| API argument conversion, cumulative | 600.868 (37.55%) → 466.125 (35.25%) | 1,578.695 (55.70%) → 1,207.365 (55.55%) | +| API return conversion, cumulative | 291.809 (18.24%) → 225.436 (17.05%) | 987.921 (34.85%) → 752.378 (34.62%) | +| `ProxyParameter.dump_trace`, cumulative | 653.538 (40.85%) → 574.950 (43.48%) | 213.353 (7.53%) → 162.592 (7.48%) | +| `tensor_hash`, cumulative, nested in snapshots | 437.785 (27.36%) → 414.738 (31.37%) | 5.762 (0.20%) → 5.312 (0.24%) | +| Final `serialize`, cumulative | 4.350 (0.27%) → 4.386 (0.33%) | 12.093 (0.43%) → 11.245 (0.52%) | + +The classifier evidence is concrete: `isinstance` calls fall from 1,503,363 to +464,357 for MNIST and 3,622,491 to 1,154,245 for Transformer. Parameter +`__instancecheck__` calls fall from 203,335 to 2,817 and 495,303 to 8,937, +respectively (Transformer also has one other instance-check entry in both +variants). The number of attribute lookups/warning scopes is unchanged: +211,616 for MNIST and 516,009 for Transformer. Thus repeated per-attribute +lookup/warning setup remains an actionable instrumentation bottleneck; this +candidate does not remove it. Final JSON encoding is a small part of these +profiles. Hash call counts remain 112 and 64; hash timing variation is not +attributed to a hash optimization. + +## Activation, data integrity and resource accounting + +All **28 successful cases** (four smoke, twenty ordinary, four profile) passed +exact loss/model/RNG checks. All saved raw trace SHA-256s, sizes, counts and API +pre/post pair multiplicities were independently reread and validated locally. +Every tracked parameter has at least two distinct data hashes across the three +steps. The five ordinary runs and separate profiles have matching coverage +signatures across variants: + +| Workload | API records / complete pairs | VAR records / parameters | Data+gradient hash fields | Raw API+VAR bytes | +|---|---:|---:|---:|---:| +| MNIST | 520 / 260 | 64 / 8 | 112 | 419,429 | +| Transformer | 1,340 / 670 | 64 / 29 | 64 | 1,088,456 | + +Transformer snapshots have `grad=None` and capture changed parameter values at +the current observers' assignment/zero-grad observation points. They do **not** +establish pre/post snapshots for each manual SGD update, or gradient snapshot +coverage. The final state check independently verifies that all three training +steps and their updates matched. We did not replace manual SGD or add synthetic +observers to improve coverage. MNIST's Adadelta path has data and gradient hashes. + +Logical hashed input volume, counting repeated snapshots, is 67,193,392 bytes +for MNIST and 214,496 bytes for Transformer. This is not physical memory traffic +or a time partition. No cross-process raw trace equality is claimed: identities +and timestamps remain intact in the retained raw records. + +| Ordinary median | MNIST baseline → candidate | Transformer baseline → candidate | +|---|---:|---:| +| Setup | 2.702 → 2.709 s | 2.603 → 2.609 s | +| Whole process | 7.018 → 7.019 s | 7.018 → 6.817 s | +| Post-metrics process exit | 2.078 → 2.131 s | 2.039 → 2.040 s | +| Sampled main-process peak RSS | 561.492 → 561.285 MiB | 515.441 → 515.648 MiB | + +Post-metrics exit includes writer drain, Python shutdown and up to one polling +interval; it is **not a pure I/O drain timer**. The small training-body savings +do not materially reduce whole-process MNIST time in this short workload. Full +per-run distributions for setup/process/exit/RSS/bytes are in [`data/metadata-classification-20261027.json`](data/metadata-classification-20261027.json). + +## Reproduction and artifacts + +Remote root: +`/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed`. +Local root: +`/Users/matt/gitrepos/traincheck/outputs/instrumenter-performance-20261027`. +Use a fresh copy of the experiment directory for reruns because run directories +are deliberately created exclusively. `baseline/source.tar` and +`candidate/source.tar` retain the exact source archives; the only experimental +adapter change in `source/tools/fresh_validation/benchmark_workload.py` is the +typed registration callback before optimizer construction, identical in both +variants. Retained harnesses are `component_runner.py` and `run_experiment.py`; +`analyze_experiment.py` rereads and validates the evidence. + +```bash +/home/yuxuan/experiments/codex-inference-runtime-20261025/venv/bin/python run_experiment.py --root "$EXPERIMENT_ROOT" --phase smoke +/home/yuxuan/experiments/codex-inference-runtime-20261025/venv/bin/python run_experiment.py --root "$EXPERIMENT_ROOT" --phase paired +python analyze_experiment.py --root "$LOCAL_EVIDENCE_ROOT" +``` + +The runner sets `PYTHONPATH` to each isolated source tree, restricts native +threads, disables bytecode writes and CUDA visibility, and records exact child +commands. It aborts on execution, coverage, loss, model-state or RNG divergence. +No installation or shared source/environment edit was needed. + +| Artifact | SHA-256 | +|---|---| +| Baseline source archive | `ac1a830f490bef1af6b1b15ba49d116ff28a8de2e57f51dd80afc714f4e8d909` | +| Candidate source archive | `b945a9d4c2b43def8481660bc3fc6ae45dbfbd34e3a5e8a0c848e5ca379276e6` | +| Raw evidence archive, 24,826,924 bytes | `1de069eb0dd5d8dd4ce60e758a4d3dae5716fa69bda1f3e30570b983f4e69773` | +| Executed runner | `d17315872c4e412d3a2be075fa733eebb9c7d4f8189dff82d53dc6bae92c0818` | +| Executed driver | `3a89d821238902362b110abbc9d66bb2b87c1b1cef09144bf7407a5400cd72f4` | + +The raw archive retains every log, metrics/manifest file, trace, profile and +environment inventory. Large/raw data remains in outputs and the isolated +remote directory, not in commits. + +Scoped mypy passes for all three new harness/analysis modules with +`--check-untyped-defs --disallow-untyped-defs --disallow-any-generics` and +`--follow-imports=skip --ignore-missing-imports`. The latter two options mean +dependency typing is not fully checked. Production test/check results belong +to the focused implementation PRs; this report records the actual workload +validation only. + +## Asynchronous work: what this measurement supports + +The subclass API/VAR file writers already run in background threads. Producers +still inspect live values and build immutable serialized records before enqueueing +so later application mutations cannot change earlier observations. Final JSON +encoding contributes 0.27–0.52% of the measured profiles; moving just that encoding +to a worker has little headroom here. The larger synchronous work is metadata +inspection and conversion, including per-attribute warning scopes. + +Moving arbitrary object inspection to a worker after a shallow copy would change +which values, getters and exceptions are observed. A future asynchronous tensor +snapshot path needs owned copies, producer-stream ordering for CUDA, bounded byte +and item queues, per-destination ordering, and explicit flush/error behavior. +It would need separate contract tests and an explicit decision about delayed +serialization errors. This PR implements the measured synchronous classification +fast path only; it does not add asynchronous snapshot conversion. + +The separate CLI default validation also reproduced a pre-existing startup +recursion under `--use-full-instr` in `torch._ops.__getattr__` on unchanged main +and the subclass candidate. Those broader-flag runs were stopped at 45 seconds; +they are not included in the successful ordinary-collection measurements above. From fe3e399a32f7b5865bb072e4552a1f9c4bcd2531 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Tue, 27 Oct 2026 03:18:06 +0800 Subject: [PATCH 3/5] Anchor all paired measurements to plain training --- .../metadata-classification-20261027.json | 742 ++++++++++++++++++ .../metadata-classification-20261027.md | 20 +- 2 files changed, 759 insertions(+), 3 deletions(-) diff --git a/docs/measurements/data/metadata-classification-20261027.json b/docs/measurements/data/metadata-classification-20261027.json index 379d3193..2e63662e 100644 --- a/docs/measurements/data/metadata-classification-20261027.json +++ b/docs/measurements/data/metadata-classification-20261027.json @@ -13,6 +13,748 @@ "Post-metrics exit includes Python shutdown and polling delay; it is not a pure writer timer", "Full raw traces retained; no normalized cross-process trace equivalence claim" ], + "plain_reference_validation": { + "compared_fields": [ + "initial_state_hash", + "final_state_hash", + "initial_shapes", + "final_shapes", + "rng_states", + "every step ID and loss field (only seconds excluded)" + ], + "mnist_reference_metrics": { + "error": null, + "final_shapes": { + "model.conv1.bias": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.conv1.weight": { + "dtype": "torch.float32", + "shape": [ + 32, + 1, + 3, + 3 + ] + }, + "model.conv2.bias": { + "dtype": "torch.float32", + "shape": [ + 64 + ] + }, + "model.conv2.weight": { + "dtype": "torch.float32", + "shape": [ + 64, + 32, + 3, + 3 + ] + }, + "model.fc1.bias": { + "dtype": "torch.float32", + "shape": [ + 128 + ] + }, + "model.fc1.weight": { + "dtype": "torch.float32", + "shape": [ + 128, + 9216 + ] + }, + "model.fc2.bias": { + "dtype": "torch.float32", + "shape": [ + 10 + ] + }, + "model.fc2.weight": { + "dtype": "torch.float32", + "shape": [ + 10, + 128 + ] + } + }, + "final_state_hash": "f20e5ffbd73ae70379e3a9360e4fe2cf78c4cae0168b259f8070a8d77fc2f097", + "initial_shapes": { + "model.conv1.bias": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.conv1.weight": { + "dtype": "torch.float32", + "shape": [ + 32, + 1, + 3, + 3 + ] + }, + "model.conv2.bias": { + "dtype": "torch.float32", + "shape": [ + 64 + ] + }, + "model.conv2.weight": { + "dtype": "torch.float32", + "shape": [ + 64, + 32, + 3, + 3 + ] + }, + "model.fc1.bias": { + "dtype": "torch.float32", + "shape": [ + 128 + ] + }, + "model.fc1.weight": { + "dtype": "torch.float32", + "shape": [ + 128, + 9216 + ] + }, + "model.fc2.bias": { + "dtype": "torch.float32", + "shape": [ + 10 + ] + }, + "model.fc2.weight": { + "dtype": "torch.float32", + "shape": [ + 10, + 128 + ] + } + }, + "initial_state_hash": "877e9abeb783ae5ed703a9666a7276e921e0424b78bb2d50a41461221d90ddb0", + "ready_exit_monotonic": 24076201.16190604, + "rng_states": [ + { + "numpy": "9b2dca039a39b1c5eb2bf7ea735d053e4c30f5de50f7ed22807eae95fbe0fdec", + "python": "a4514b792ce994268ee4bc77c0c5a3fc24b14400e9ab8d182b1a78f069da0024", + "torch_cpu": "5d898ff0719d3cff95ebfa4424bcc53134ef38bca55e960f0e4a730f5120a3bf" + }, + { + "numpy": "9b2dca039a39b1c5eb2bf7ea735d053e4c30f5de50f7ed22807eae95fbe0fdec", + "python": "a4514b792ce994268ee4bc77c0c5a3fc24b14400e9ab8d182b1a78f069da0024", + "torch_cpu": "a32e42c3152f9ec4a0f34befd06fec03a9e9a7de13a6ef9e7fabce37f904f03b" + }, + { + "numpy": "9b2dca039a39b1c5eb2bf7ea735d053e4c30f5de50f7ed22807eae95fbe0fdec", + "python": "a4514b792ce994268ee4bc77c0c5a3fc24b14400e9ab8d182b1a78f069da0024", + "torch_cpu": "2612dc551398f429437ad911407a676f3284fd23225a01f495cc4349ba731085" + }, + { + "numpy": "9b2dca039a39b1c5eb2bf7ea735d053e4c30f5de50f7ed22807eae95fbe0fdec", + "python": "a4514b792ce994268ee4bc77c0c5a3fc24b14400e9ab8d182b1a78f069da0024", + "torch_cpu": "6c23a2d8afae7c8a12280b615104d87dae13d27388d21e7161a47f5ac88d2743" + } + ], + "setup_seconds": 0.08462173491716385, + "steps": [ + { + "loss": 2.2311902046203613, + "seconds": 0.055096585303545, + "step": 1 + }, + { + "loss": 3.296325445175171, + "seconds": 0.030034486204385757, + "step": 2 + }, + { + "loss": 2.6148438453674316, + "seconds": 0.01990584284067154, + "step": 3 + } + ], + "success": true, + "training_seconds": 0.1050369143486023 + }, + "performance_claim": "None; supplemental run establishes a plain correctness reference only", + "production_file_hashes_match_measured_baseline": true, + "reference_for_mnist": "New supplemental plain 88b6df2 run, same data/seeds/configuration", + "reference_for_transformer": "Existing plain 88b6df2 Transformer smoke run", + "remote_supplement": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-plain-parity", + "source_commit": "88b6df2401e187ca6bd81bf0a83bc7c5d9a5cd54", + "source_tree": "5415a3b0e5eb08d119dffa91631d76866e6e1d11", + "success": true, + "supplement_archive_bytes": 12709, + "supplement_archive_sha256": "6093c207f75f30331248220838613369a50d6fa3a4f6f534e0075ce7f8f4d4c1", + "transformer_reference_metrics": { + "error": null, + "final_shapes": { + "model.decoder.bias": { + "dtype": "torch.float32", + "shape": [ + 60 + ] + }, + "model.decoder.weight": { + "dtype": "torch.float32", + "shape": [ + 60, + 32 + ] + }, + "model.encoder.layers.0.linear1.bias": { + "dtype": "torch.float32", + "shape": [ + 64 + ] + }, + "model.encoder.layers.0.linear1.weight": { + "dtype": "torch.float32", + "shape": [ + 64, + 32 + ] + }, + "model.encoder.layers.0.linear2.bias": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.encoder.layers.0.linear2.weight": { + "dtype": "torch.float32", + "shape": [ + 32, + 64 + ] + }, + "model.encoder.layers.0.norm1.bias": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.encoder.layers.0.norm1.weight": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.encoder.layers.0.norm2.bias": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.encoder.layers.0.norm2.weight": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.encoder.layers.0.self_attn.in_proj_bias": { + "dtype": "torch.float32", + "shape": [ + 96 + ] + }, + "model.encoder.layers.0.self_attn.in_proj_weight": { + "dtype": "torch.float32", + "shape": [ + 96, + 32 + ] + }, + "model.encoder.layers.0.self_attn.out_proj.bias": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.encoder.layers.0.self_attn.out_proj.weight": { + "dtype": "torch.float32", + "shape": [ + 32, + 32 + ] + }, + "model.encoder.layers.1.linear1.bias": { + "dtype": "torch.float32", + "shape": [ + 64 + ] + }, + "model.encoder.layers.1.linear1.weight": { + "dtype": "torch.float32", + "shape": [ + 64, + 32 + ] + }, + "model.encoder.layers.1.linear2.bias": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.encoder.layers.1.linear2.weight": { + "dtype": "torch.float32", + "shape": [ + 32, + 64 + ] + }, + "model.encoder.layers.1.norm1.bias": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.encoder.layers.1.norm1.weight": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.encoder.layers.1.norm2.bias": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.encoder.layers.1.norm2.weight": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.encoder.layers.1.self_attn.in_proj_bias": { + "dtype": "torch.float32", + "shape": [ + 96 + ] + }, + "model.encoder.layers.1.self_attn.in_proj_weight": { + "dtype": "torch.float32", + "shape": [ + 96, + 32 + ] + }, + "model.encoder.layers.1.self_attn.out_proj.bias": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.encoder.layers.1.self_attn.out_proj.weight": { + "dtype": "torch.float32", + "shape": [ + 32, + 32 + ] + }, + "model.encoder.norm.bias": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.encoder.norm.weight": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.input_emb.weight": { + "dtype": "torch.float32", + "shape": [ + 60, + 32 + ] + }, + "model.pos_encoder.pe": { + "dtype": "torch.float32", + "shape": [ + 5000, + 1, + 32 + ] + } + }, + "final_state_hash": "8f17d60efae8fc6f1fcdc3995f560af9a16a85a39c48074810f243f7946cdbc1", + "initial_shapes": { + "model.decoder.bias": { + "dtype": "torch.float32", + "shape": [ + 60 + ] + }, + "model.decoder.weight": { + "dtype": "torch.float32", + "shape": [ + 60, + 32 + ] + }, + "model.encoder.layers.0.linear1.bias": { + "dtype": "torch.float32", + "shape": [ + 64 + ] + }, + "model.encoder.layers.0.linear1.weight": { + "dtype": "torch.float32", + "shape": [ + 64, + 32 + ] + }, + "model.encoder.layers.0.linear2.bias": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.encoder.layers.0.linear2.weight": { + "dtype": "torch.float32", + "shape": [ + 32, + 64 + ] + }, + "model.encoder.layers.0.norm1.bias": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.encoder.layers.0.norm1.weight": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.encoder.layers.0.norm2.bias": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.encoder.layers.0.norm2.weight": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.encoder.layers.0.self_attn.in_proj_bias": { + "dtype": "torch.float32", + "shape": [ + 96 + ] + }, + "model.encoder.layers.0.self_attn.in_proj_weight": { + "dtype": "torch.float32", + "shape": [ + 96, + 32 + ] + }, + "model.encoder.layers.0.self_attn.out_proj.bias": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.encoder.layers.0.self_attn.out_proj.weight": { + "dtype": "torch.float32", + "shape": [ + 32, + 32 + ] + }, + "model.encoder.layers.1.linear1.bias": { + "dtype": "torch.float32", + "shape": [ + 64 + ] + }, + "model.encoder.layers.1.linear1.weight": { + "dtype": "torch.float32", + "shape": [ + 64, + 32 + ] + }, + "model.encoder.layers.1.linear2.bias": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.encoder.layers.1.linear2.weight": { + "dtype": "torch.float32", + "shape": [ + 32, + 64 + ] + }, + "model.encoder.layers.1.norm1.bias": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.encoder.layers.1.norm1.weight": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.encoder.layers.1.norm2.bias": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.encoder.layers.1.norm2.weight": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.encoder.layers.1.self_attn.in_proj_bias": { + "dtype": "torch.float32", + "shape": [ + 96 + ] + }, + "model.encoder.layers.1.self_attn.in_proj_weight": { + "dtype": "torch.float32", + "shape": [ + 96, + 32 + ] + }, + "model.encoder.layers.1.self_attn.out_proj.bias": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.encoder.layers.1.self_attn.out_proj.weight": { + "dtype": "torch.float32", + "shape": [ + 32, + 32 + ] + }, + "model.encoder.norm.bias": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.encoder.norm.weight": { + "dtype": "torch.float32", + "shape": [ + 32 + ] + }, + "model.input_emb.weight": { + "dtype": "torch.float32", + "shape": [ + 60, + 32 + ] + }, + "model.pos_encoder.pe": { + "dtype": "torch.float32", + "shape": [ + 5000, + 1, + 32 + ] + } + }, + "initial_state_hash": "e197d3c2a3f351146bc265c5b3eb54dcd1d921499ce24d1cc61bdc5e6dce886d", + "ready_exit_monotonic": 24075261.033744246, + "rng_states": [ + { + "numpy": "9b2dca039a39b1c5eb2bf7ea735d053e4c30f5de50f7ed22807eae95fbe0fdec", + "python": "a4514b792ce994268ee4bc77c0c5a3fc24b14400e9ab8d182b1a78f069da0024", + "torch_cpu": "5d898ff0719d3cff95ebfa4424bcc53134ef38bca55e960f0e4a730f5120a3bf" + }, + { + "numpy": "9b2dca039a39b1c5eb2bf7ea735d053e4c30f5de50f7ed22807eae95fbe0fdec", + "python": "a4514b792ce994268ee4bc77c0c5a3fc24b14400e9ab8d182b1a78f069da0024", + "torch_cpu": "08de71b1d3a21783308f7dad34328ed9413914a6e169c136ee0f9c1362f1a20a" + }, + { + "numpy": "9b2dca039a39b1c5eb2bf7ea735d053e4c30f5de50f7ed22807eae95fbe0fdec", + "python": "a4514b792ce994268ee4bc77c0c5a3fc24b14400e9ab8d182b1a78f069da0024", + "torch_cpu": "bc44ff0d3d6a8177eeaa8e61a2274ef645260c845c149a86c775fe86e21f027a" + }, + { + "numpy": "9b2dca039a39b1c5eb2bf7ea735d053e4c30f5de50f7ed22807eae95fbe0fdec", + "python": "a4514b792ce994268ee4bc77c0c5a3fc24b14400e9ab8d182b1a78f069da0024", + "torch_cpu": "1f563177295a4734ad2f8c17890af790ae8bf9168ec014d213446ca87db77688" + } + ], + "setup_seconds": 0.030553825199604034, + "steps": [ + { + "loss": 4.222344875335693, + "seconds": 0.012831639498472214, + "step": 1 + }, + { + "loss": 4.122839450836182, + "seconds": 0.001412227749824524, + "step": 2 + }, + { + "loss": 4.136002063751221, + "seconds": 0.001173999160528183, + "step": 3 + } + ], + "success": true, + "training_seconds": 0.01541786640882492 + }, + "validated_paired_and_profiled_cases": [ + { + "label": "mnist-combined-r1", + "variant": "baseline", + "workload": "mnist" + }, + { + "label": "mnist-combined-r1", + "variant": "candidate", + "workload": "mnist" + }, + { + "label": "mnist-combined-r2", + "variant": "candidate", + "workload": "mnist" + }, + { + "label": "mnist-combined-r2", + "variant": "baseline", + "workload": "mnist" + }, + { + "label": "mnist-combined-r3", + "variant": "baseline", + "workload": "mnist" + }, + { + "label": "mnist-combined-r3", + "variant": "candidate", + "workload": "mnist" + }, + { + "label": "mnist-combined-r4", + "variant": "candidate", + "workload": "mnist" + }, + { + "label": "mnist-combined-r4", + "variant": "baseline", + "workload": "mnist" + }, + { + "label": "mnist-combined-r5", + "variant": "baseline", + "workload": "mnist" + }, + { + "label": "mnist-combined-r5", + "variant": "candidate", + "workload": "mnist" + }, + { + "label": "mnist-combined-r1-profile", + "variant": "baseline", + "workload": "mnist" + }, + { + "label": "mnist-combined-r1-profile", + "variant": "candidate", + "workload": "mnist" + }, + { + "label": "transformer-combined-r1", + "variant": "baseline", + "workload": "transformer" + }, + { + "label": "transformer-combined-r1", + "variant": "candidate", + "workload": "transformer" + }, + { + "label": "transformer-combined-r2", + "variant": "candidate", + "workload": "transformer" + }, + { + "label": "transformer-combined-r2", + "variant": "baseline", + "workload": "transformer" + }, + { + "label": "transformer-combined-r3", + "variant": "baseline", + "workload": "transformer" + }, + { + "label": "transformer-combined-r3", + "variant": "candidate", + "workload": "transformer" + }, + { + "label": "transformer-combined-r4", + "variant": "candidate", + "workload": "transformer" + }, + { + "label": "transformer-combined-r4", + "variant": "baseline", + "workload": "transformer" + }, + { + "label": "transformer-combined-r5", + "variant": "baseline", + "workload": "transformer" + }, + { + "label": "transformer-combined-r5", + "variant": "candidate", + "workload": "transformer" + }, + { + "label": "transformer-combined-r1-profile", + "variant": "baseline", + "workload": "transformer" + }, + { + "label": "transformer-combined-r1-profile", + "variant": "candidate", + "workload": "transformer" + } + ] + }, "production_changed_files": [ "traincheck/instrumentor/dumper.py" ], diff --git a/docs/measurements/metadata-classification-20261027.md b/docs/measurements/metadata-classification-20261027.md index d7d122c7..a4f96cba 100644 --- a/docs/measurements/metadata-classification-20261027.md +++ b/docs/measurements/metadata-classification-20261027.md @@ -124,9 +124,16 @@ attributed to a hash optimization. ## Activation, data integrity and resource accounting -All **28 successful cases** (four smoke, twenty ordinary, four profile) passed -exact loss/model/RNG checks. All saved raw trace SHA-256s, sizes, counts and API -pre/post pair multiplicities were independently reread and validated locally. +All **28 original cases** (four smoke, twenty ordinary, four profile) completed +successfully. The paired driver initially compared each workload against its +first baseline combined run. A supplementary check explicitly anchors all 24 +ordinary/profile cases to plain training: the existing Transformer plain smoke +and a new isolated MNIST plain run using the exact baseline source. Every initial +and final state digest/shape, loss, step ID, and per-step Python/NumPy/PyTorch RNG +snapshot matches. Timing fields alone are excluded from this behavioral comparison. +The supplemental run supplies a correctness reference, not another speedup estimate. +All saved raw trace SHA-256s, sizes, counts and API pre/post pair multiplicities +were independently reread and validated locally. Every tracked parameter has at least two distinct data hashes across the three steps. The five ordinary runs and separate profiles have matching coverage signatures across variants: @@ -225,3 +232,10 @@ The separate CLI default validation also reproduced a pre-existing startup recursion under `--use-full-instr` in `torch._ops.__getattr__` on unchanged main and the subclass candidate. Those broader-flag runs were stopped at 45 seconds; they are not included in the successful ordinary-collection measurements above. + +Plain-reference validation is included in the measurement JSON under +`plain_reference_validation`. Its raw supplementary archive is retained as +`plain-parity.tar.gz` (12,709 bytes), SHA-256 +`6093c207f75f30331248220838613369a50d6fa3a4f6f534e0075ce7f8f4d4c1`, +with its checker alongside the other local evidence. The original 28-case +archive is unchanged. From bcf9bb82c3c11812336d1199dc699aa4ee6473e2 Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Tue, 27 Oct 2026 03:24:18 +0800 Subject: [PATCH 4/5] Check metadata lookup side effects across PyTorch versions --- tests/test_metadata_classification.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_metadata_classification.py b/tests/test_metadata_classification.py index 3c0fc7cc..c417ad14 100644 --- a/tests/test_metadata_classification.py +++ b/tests/test_metadata_classification.py @@ -130,12 +130,18 @@ def test_tensor_and_parameter_subclasses_keep_identity_and_values(self) -> None: self.assertEqual(before["value"], [3.0, 4.0]) def test_dynamic_class_keeps_parameter_fallback(self) -> None: + # Parameter.__instancecheck__ makes a different number of __class__ reads + # across supported PyTorch versions. Preserve the framework's exact + # lookup behavior, including side effects, without pinning that count. + native = ChangingClass() + self.assertFalse(isinstance(native, torch.Tensor)) + self.assertTrue(isinstance(native, torch.nn.Parameter)) value = ChangingClass() record = dumper.convert_var_to_dict( AttributeHolder(value), False, {"value": None} ) self.assertEqual(record, {"_TRAINCHECK_value_ID": id(value)}) - self.assertEqual(value.reads, 2) + self.assertEqual(value.reads, native.reads) def test_dynamic_class_exception_still_propagates(self) -> None: error = ValueError("dynamic class failed") From 8484805dc6e1d67e8a168aad9357f9545412b7eb Mon Sep 17 00:00:00 2001 From: Yuxuan Jiang Date: Tue, 27 Oct 2026 03:50:04 +0800 Subject: [PATCH 5/5] Refresh tracing measurements against the final reviewed correctness stack --- .../metadata-classification-20261027.json | 1990 ++++++----------- .../metadata-classification-20261027.md | 413 ++-- 2 files changed, 852 insertions(+), 1551 deletions(-) diff --git a/docs/measurements/data/metadata-classification-20261027.json b/docs/measurements/data/metadata-classification-20261027.json index 2e63662e..87996b13 100644 --- a/docs/measurements/data/metadata-classification-20261027.json +++ b/docs/measurements/data/metadata-classification-20261027.json @@ -1,10 +1,10 @@ { "artifacts": { - "driver_sha256": "3a89d821238902362b110abbc9d66bb2b87c1b1cef09144bf7407a5400cd72f4", - "evidence_archive_bytes": 24826924, - "evidence_archive_sha256": "1de069eb0dd5d8dd4ce60e758a4d3dae5716fa69bda1f3e30570b983f4e69773", - "remote": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed", - "runner_sha256": "d17315872c4e412d3a2be075fa733eebb9c7d4f8189dff82d53dc6bae92c0818" + "driver_sha256": "5a0d4c4a02ea397bce646a482c2657bb80a1cd28bb5fa6e7cd0127071c6fda67", + "evidence_archive_bytes": 25722591, + "evidence_archive_sha256": "875be9098fab644afed9798cd6005f66af6567719e6be1765d84e9bd74725584", + "remote": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched", + "runner_sha256": "21984b309b688e95de62d37b59e6e5f7c9a1685ef5293e54dfaacd60e9305875" }, "limitations": [ "CPU only, one native thread, three cold steps and five pairs on a shared host", @@ -13,748 +13,6 @@ "Post-metrics exit includes Python shutdown and polling delay; it is not a pure writer timer", "Full raw traces retained; no normalized cross-process trace equivalence claim" ], - "plain_reference_validation": { - "compared_fields": [ - "initial_state_hash", - "final_state_hash", - "initial_shapes", - "final_shapes", - "rng_states", - "every step ID and loss field (only seconds excluded)" - ], - "mnist_reference_metrics": { - "error": null, - "final_shapes": { - "model.conv1.bias": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.conv1.weight": { - "dtype": "torch.float32", - "shape": [ - 32, - 1, - 3, - 3 - ] - }, - "model.conv2.bias": { - "dtype": "torch.float32", - "shape": [ - 64 - ] - }, - "model.conv2.weight": { - "dtype": "torch.float32", - "shape": [ - 64, - 32, - 3, - 3 - ] - }, - "model.fc1.bias": { - "dtype": "torch.float32", - "shape": [ - 128 - ] - }, - "model.fc1.weight": { - "dtype": "torch.float32", - "shape": [ - 128, - 9216 - ] - }, - "model.fc2.bias": { - "dtype": "torch.float32", - "shape": [ - 10 - ] - }, - "model.fc2.weight": { - "dtype": "torch.float32", - "shape": [ - 10, - 128 - ] - } - }, - "final_state_hash": "f20e5ffbd73ae70379e3a9360e4fe2cf78c4cae0168b259f8070a8d77fc2f097", - "initial_shapes": { - "model.conv1.bias": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.conv1.weight": { - "dtype": "torch.float32", - "shape": [ - 32, - 1, - 3, - 3 - ] - }, - "model.conv2.bias": { - "dtype": "torch.float32", - "shape": [ - 64 - ] - }, - "model.conv2.weight": { - "dtype": "torch.float32", - "shape": [ - 64, - 32, - 3, - 3 - ] - }, - "model.fc1.bias": { - "dtype": "torch.float32", - "shape": [ - 128 - ] - }, - "model.fc1.weight": { - "dtype": "torch.float32", - "shape": [ - 128, - 9216 - ] - }, - "model.fc2.bias": { - "dtype": "torch.float32", - "shape": [ - 10 - ] - }, - "model.fc2.weight": { - "dtype": "torch.float32", - "shape": [ - 10, - 128 - ] - } - }, - "initial_state_hash": "877e9abeb783ae5ed703a9666a7276e921e0424b78bb2d50a41461221d90ddb0", - "ready_exit_monotonic": 24076201.16190604, - "rng_states": [ - { - "numpy": "9b2dca039a39b1c5eb2bf7ea735d053e4c30f5de50f7ed22807eae95fbe0fdec", - "python": "a4514b792ce994268ee4bc77c0c5a3fc24b14400e9ab8d182b1a78f069da0024", - "torch_cpu": "5d898ff0719d3cff95ebfa4424bcc53134ef38bca55e960f0e4a730f5120a3bf" - }, - { - "numpy": "9b2dca039a39b1c5eb2bf7ea735d053e4c30f5de50f7ed22807eae95fbe0fdec", - "python": "a4514b792ce994268ee4bc77c0c5a3fc24b14400e9ab8d182b1a78f069da0024", - "torch_cpu": "a32e42c3152f9ec4a0f34befd06fec03a9e9a7de13a6ef9e7fabce37f904f03b" - }, - { - "numpy": "9b2dca039a39b1c5eb2bf7ea735d053e4c30f5de50f7ed22807eae95fbe0fdec", - "python": "a4514b792ce994268ee4bc77c0c5a3fc24b14400e9ab8d182b1a78f069da0024", - "torch_cpu": "2612dc551398f429437ad911407a676f3284fd23225a01f495cc4349ba731085" - }, - { - "numpy": "9b2dca039a39b1c5eb2bf7ea735d053e4c30f5de50f7ed22807eae95fbe0fdec", - "python": "a4514b792ce994268ee4bc77c0c5a3fc24b14400e9ab8d182b1a78f069da0024", - "torch_cpu": "6c23a2d8afae7c8a12280b615104d87dae13d27388d21e7161a47f5ac88d2743" - } - ], - "setup_seconds": 0.08462173491716385, - "steps": [ - { - "loss": 2.2311902046203613, - "seconds": 0.055096585303545, - "step": 1 - }, - { - "loss": 3.296325445175171, - "seconds": 0.030034486204385757, - "step": 2 - }, - { - "loss": 2.6148438453674316, - "seconds": 0.01990584284067154, - "step": 3 - } - ], - "success": true, - "training_seconds": 0.1050369143486023 - }, - "performance_claim": "None; supplemental run establishes a plain correctness reference only", - "production_file_hashes_match_measured_baseline": true, - "reference_for_mnist": "New supplemental plain 88b6df2 run, same data/seeds/configuration", - "reference_for_transformer": "Existing plain 88b6df2 Transformer smoke run", - "remote_supplement": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-plain-parity", - "source_commit": "88b6df2401e187ca6bd81bf0a83bc7c5d9a5cd54", - "source_tree": "5415a3b0e5eb08d119dffa91631d76866e6e1d11", - "success": true, - "supplement_archive_bytes": 12709, - "supplement_archive_sha256": "6093c207f75f30331248220838613369a50d6fa3a4f6f534e0075ce7f8f4d4c1", - "transformer_reference_metrics": { - "error": null, - "final_shapes": { - "model.decoder.bias": { - "dtype": "torch.float32", - "shape": [ - 60 - ] - }, - "model.decoder.weight": { - "dtype": "torch.float32", - "shape": [ - 60, - 32 - ] - }, - "model.encoder.layers.0.linear1.bias": { - "dtype": "torch.float32", - "shape": [ - 64 - ] - }, - "model.encoder.layers.0.linear1.weight": { - "dtype": "torch.float32", - "shape": [ - 64, - 32 - ] - }, - "model.encoder.layers.0.linear2.bias": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.encoder.layers.0.linear2.weight": { - "dtype": "torch.float32", - "shape": [ - 32, - 64 - ] - }, - "model.encoder.layers.0.norm1.bias": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.encoder.layers.0.norm1.weight": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.encoder.layers.0.norm2.bias": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.encoder.layers.0.norm2.weight": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.encoder.layers.0.self_attn.in_proj_bias": { - "dtype": "torch.float32", - "shape": [ - 96 - ] - }, - "model.encoder.layers.0.self_attn.in_proj_weight": { - "dtype": "torch.float32", - "shape": [ - 96, - 32 - ] - }, - "model.encoder.layers.0.self_attn.out_proj.bias": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.encoder.layers.0.self_attn.out_proj.weight": { - "dtype": "torch.float32", - "shape": [ - 32, - 32 - ] - }, - "model.encoder.layers.1.linear1.bias": { - "dtype": "torch.float32", - "shape": [ - 64 - ] - }, - "model.encoder.layers.1.linear1.weight": { - "dtype": "torch.float32", - "shape": [ - 64, - 32 - ] - }, - "model.encoder.layers.1.linear2.bias": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.encoder.layers.1.linear2.weight": { - "dtype": "torch.float32", - "shape": [ - 32, - 64 - ] - }, - "model.encoder.layers.1.norm1.bias": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.encoder.layers.1.norm1.weight": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.encoder.layers.1.norm2.bias": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.encoder.layers.1.norm2.weight": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.encoder.layers.1.self_attn.in_proj_bias": { - "dtype": "torch.float32", - "shape": [ - 96 - ] - }, - "model.encoder.layers.1.self_attn.in_proj_weight": { - "dtype": "torch.float32", - "shape": [ - 96, - 32 - ] - }, - "model.encoder.layers.1.self_attn.out_proj.bias": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.encoder.layers.1.self_attn.out_proj.weight": { - "dtype": "torch.float32", - "shape": [ - 32, - 32 - ] - }, - "model.encoder.norm.bias": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.encoder.norm.weight": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.input_emb.weight": { - "dtype": "torch.float32", - "shape": [ - 60, - 32 - ] - }, - "model.pos_encoder.pe": { - "dtype": "torch.float32", - "shape": [ - 5000, - 1, - 32 - ] - } - }, - "final_state_hash": "8f17d60efae8fc6f1fcdc3995f560af9a16a85a39c48074810f243f7946cdbc1", - "initial_shapes": { - "model.decoder.bias": { - "dtype": "torch.float32", - "shape": [ - 60 - ] - }, - "model.decoder.weight": { - "dtype": "torch.float32", - "shape": [ - 60, - 32 - ] - }, - "model.encoder.layers.0.linear1.bias": { - "dtype": "torch.float32", - "shape": [ - 64 - ] - }, - "model.encoder.layers.0.linear1.weight": { - "dtype": "torch.float32", - "shape": [ - 64, - 32 - ] - }, - "model.encoder.layers.0.linear2.bias": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.encoder.layers.0.linear2.weight": { - "dtype": "torch.float32", - "shape": [ - 32, - 64 - ] - }, - "model.encoder.layers.0.norm1.bias": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.encoder.layers.0.norm1.weight": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.encoder.layers.0.norm2.bias": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.encoder.layers.0.norm2.weight": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.encoder.layers.0.self_attn.in_proj_bias": { - "dtype": "torch.float32", - "shape": [ - 96 - ] - }, - "model.encoder.layers.0.self_attn.in_proj_weight": { - "dtype": "torch.float32", - "shape": [ - 96, - 32 - ] - }, - "model.encoder.layers.0.self_attn.out_proj.bias": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.encoder.layers.0.self_attn.out_proj.weight": { - "dtype": "torch.float32", - "shape": [ - 32, - 32 - ] - }, - "model.encoder.layers.1.linear1.bias": { - "dtype": "torch.float32", - "shape": [ - 64 - ] - }, - "model.encoder.layers.1.linear1.weight": { - "dtype": "torch.float32", - "shape": [ - 64, - 32 - ] - }, - "model.encoder.layers.1.linear2.bias": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.encoder.layers.1.linear2.weight": { - "dtype": "torch.float32", - "shape": [ - 32, - 64 - ] - }, - "model.encoder.layers.1.norm1.bias": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.encoder.layers.1.norm1.weight": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.encoder.layers.1.norm2.bias": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.encoder.layers.1.norm2.weight": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.encoder.layers.1.self_attn.in_proj_bias": { - "dtype": "torch.float32", - "shape": [ - 96 - ] - }, - "model.encoder.layers.1.self_attn.in_proj_weight": { - "dtype": "torch.float32", - "shape": [ - 96, - 32 - ] - }, - "model.encoder.layers.1.self_attn.out_proj.bias": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.encoder.layers.1.self_attn.out_proj.weight": { - "dtype": "torch.float32", - "shape": [ - 32, - 32 - ] - }, - "model.encoder.norm.bias": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.encoder.norm.weight": { - "dtype": "torch.float32", - "shape": [ - 32 - ] - }, - "model.input_emb.weight": { - "dtype": "torch.float32", - "shape": [ - 60, - 32 - ] - }, - "model.pos_encoder.pe": { - "dtype": "torch.float32", - "shape": [ - 5000, - 1, - 32 - ] - } - }, - "initial_state_hash": "e197d3c2a3f351146bc265c5b3eb54dcd1d921499ce24d1cc61bdc5e6dce886d", - "ready_exit_monotonic": 24075261.033744246, - "rng_states": [ - { - "numpy": "9b2dca039a39b1c5eb2bf7ea735d053e4c30f5de50f7ed22807eae95fbe0fdec", - "python": "a4514b792ce994268ee4bc77c0c5a3fc24b14400e9ab8d182b1a78f069da0024", - "torch_cpu": "5d898ff0719d3cff95ebfa4424bcc53134ef38bca55e960f0e4a730f5120a3bf" - }, - { - "numpy": "9b2dca039a39b1c5eb2bf7ea735d053e4c30f5de50f7ed22807eae95fbe0fdec", - "python": "a4514b792ce994268ee4bc77c0c5a3fc24b14400e9ab8d182b1a78f069da0024", - "torch_cpu": "08de71b1d3a21783308f7dad34328ed9413914a6e169c136ee0f9c1362f1a20a" - }, - { - "numpy": "9b2dca039a39b1c5eb2bf7ea735d053e4c30f5de50f7ed22807eae95fbe0fdec", - "python": "a4514b792ce994268ee4bc77c0c5a3fc24b14400e9ab8d182b1a78f069da0024", - "torch_cpu": "bc44ff0d3d6a8177eeaa8e61a2274ef645260c845c149a86c775fe86e21f027a" - }, - { - "numpy": "9b2dca039a39b1c5eb2bf7ea735d053e4c30f5de50f7ed22807eae95fbe0fdec", - "python": "a4514b792ce994268ee4bc77c0c5a3fc24b14400e9ab8d182b1a78f069da0024", - "torch_cpu": "1f563177295a4734ad2f8c17890af790ae8bf9168ec014d213446ca87db77688" - } - ], - "setup_seconds": 0.030553825199604034, - "steps": [ - { - "loss": 4.222344875335693, - "seconds": 0.012831639498472214, - "step": 1 - }, - { - "loss": 4.122839450836182, - "seconds": 0.001412227749824524, - "step": 2 - }, - { - "loss": 4.136002063751221, - "seconds": 0.001173999160528183, - "step": 3 - } - ], - "success": true, - "training_seconds": 0.01541786640882492 - }, - "validated_paired_and_profiled_cases": [ - { - "label": "mnist-combined-r1", - "variant": "baseline", - "workload": "mnist" - }, - { - "label": "mnist-combined-r1", - "variant": "candidate", - "workload": "mnist" - }, - { - "label": "mnist-combined-r2", - "variant": "candidate", - "workload": "mnist" - }, - { - "label": "mnist-combined-r2", - "variant": "baseline", - "workload": "mnist" - }, - { - "label": "mnist-combined-r3", - "variant": "baseline", - "workload": "mnist" - }, - { - "label": "mnist-combined-r3", - "variant": "candidate", - "workload": "mnist" - }, - { - "label": "mnist-combined-r4", - "variant": "candidate", - "workload": "mnist" - }, - { - "label": "mnist-combined-r4", - "variant": "baseline", - "workload": "mnist" - }, - { - "label": "mnist-combined-r5", - "variant": "baseline", - "workload": "mnist" - }, - { - "label": "mnist-combined-r5", - "variant": "candidate", - "workload": "mnist" - }, - { - "label": "mnist-combined-r1-profile", - "variant": "baseline", - "workload": "mnist" - }, - { - "label": "mnist-combined-r1-profile", - "variant": "candidate", - "workload": "mnist" - }, - { - "label": "transformer-combined-r1", - "variant": "baseline", - "workload": "transformer" - }, - { - "label": "transformer-combined-r1", - "variant": "candidate", - "workload": "transformer" - }, - { - "label": "transformer-combined-r2", - "variant": "candidate", - "workload": "transformer" - }, - { - "label": "transformer-combined-r2", - "variant": "baseline", - "workload": "transformer" - }, - { - "label": "transformer-combined-r3", - "variant": "baseline", - "workload": "transformer" - }, - { - "label": "transformer-combined-r3", - "variant": "candidate", - "workload": "transformer" - }, - { - "label": "transformer-combined-r4", - "variant": "candidate", - "workload": "transformer" - }, - { - "label": "transformer-combined-r4", - "variant": "baseline", - "workload": "transformer" - }, - { - "label": "transformer-combined-r5", - "variant": "baseline", - "workload": "transformer" - }, - { - "label": "transformer-combined-r5", - "variant": "candidate", - "workload": "transformer" - }, - { - "label": "transformer-combined-r1-profile", - "variant": "baseline", - "workload": "transformer" - }, - { - "label": "transformer-combined-r1-profile", - "variant": "candidate", - "workload": "transformer" - } - ] - }, "production_changed_files": [ "traincheck/instrumentor/dumper.py" ], @@ -772,7 +30,8 @@ "VAR": 0 }, "tracked_variables": [], - "training_seconds": 0.01541786640882492 + "training_seconds": 0.007946331053972244, + "workload": "transformer" }, { "logical_hash_input_bytes": 0, @@ -787,7 +46,8 @@ "VAR": 0 }, "tracked_variables": [], - "training_seconds": 0.8779591619968414 + "training_seconds": 0.7302097044885159, + "workload": "transformer" }, { "logical_hash_input_bytes": 214496, @@ -832,7 +92,8 @@ "model.encoder.norm.weight", "model.input_emb.weight" ], - "training_seconds": 0.08836576342582703 + "training_seconds": 0.07497242465615273, + "workload": "transformer" }, { "logical_hash_input_bytes": 214496, @@ -877,18 +138,60 @@ "model.encoder.norm.weight", "model.input_emb.weight" ], - "training_seconds": 0.9408722668886185 + "training_seconds": 0.7909529991447926, + "workload": "transformer" + }, + { + "logical_hash_input_bytes": 0, + "mode": "plain", + "records": { + "API": 0, + "VAR": 0 + }, + "tensor_hash_fields": 0, + "trace_bytes": { + "API": 0, + "VAR": 0 + }, + "tracked_variables": [], + "training_seconds": 0.03079724684357643, + "workload": "mnist" + }, + { + "logical_hash_input_bytes": 67193392, + "mode": "combined", + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "trace_bytes": { + "API": 363634, + "VAR": 55795 + }, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ], + "training_seconds": 0.6652194261550903, + "workload": "mnist" } ], "sources": { "baseline": { "adapter_sha256": "a5b5e597d4921a5b4baceeada541a6a1765f13c97ab3bd1c2ef79dea331f0934", - "production_files_verified": 77, + "production_files_verified": 78, "production_hashes": { "traincheck/__init__.py": "47fdcb090c4c8c5822769d642dc23a520e8d8ff3e4724583450723e05408164b", "traincheck/checker.py": "44dc20fc66ca82bb6b71fa02033aae3b7d489ae7afbc1611c44e857f399716b2", "traincheck/checker_online.py": "2824c9ccd2f684c86884c68f2214a1a4be1d69c6098ce30b817d03f1fe1077d6", - "traincheck/collect_trace.py": "c1db96c798af9689f8ee7f7b0b929d90694d49a234cb217beaf435d4b8d5cfb0", + "traincheck/collect_trace.py": "2e0a0b9a4e05fef450d09788e1747db2d3f1c4aa86f2828b63229cbbb7445d00", "traincheck/config/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "traincheck/config/config.py": "4c940a4d92f5de5fe8d3885e570078e479d9842d9626651fd60483e78f0160a9", "traincheck/developer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", @@ -897,6 +200,7 @@ "traincheck/inference_sampling.py": "d261b3416aba3ead6fa1e9b15091511eb3525137d531e4f0147146e76308976f", "traincheck/instrumentor/__init__.py": "228b4f6a11cde8118cde9aba2f98b60a839f4331e7fa2b06c37cee99e79fcd50", "traincheck/instrumentor/caches.py": "320eb1ffa1ce5f5b97fe4675a16d47e9591265779fdc29b94b72cdf5df93f168", + "traincheck/instrumentor/constructor_snapshot.py": "efdbdf75433ce2c3fcb0985107f59fa8e0a8ced6808b824a990030cfbf0845db", "traincheck/instrumentor/control.py": "3612a39b61e85fb77b40008b7fcd2f7569b246e8985ca64a109ea53733cd29ef", "traincheck/instrumentor/dumper.py": "cd4a56d4eb95049dcab85bacc1c64d7e55bfe45087fe5096ed247e4d0d7e8373", "traincheck/instrumentor/proxy_wrapper/__init__.py": "7c71aaf70b978ee04e236497af6f803e4436049bc0acdf18c3edb48bfcece63d", @@ -908,14 +212,14 @@ "traincheck/instrumentor/proxy_wrapper/proxy_handler.py": "4bab74647d35c64c5e9c702295dc4cb010302bf620401da0fad8539692b95ac7", "traincheck/instrumentor/proxy_wrapper/proxy_methods.py": "6ae18928aa12e89b33102368c497aff048e3d6bed330eca2118fac051b8e3a97", "traincheck/instrumentor/proxy_wrapper/proxy_observer.py": "10e83570fbe1f72089e4a594a308f83e045f7d74bb9e49193012645ac906d97b", - "traincheck/instrumentor/proxy_wrapper/proxy_registry.py": "9dfecae65aee4185152d152a5c2fb0e1e1b63339efceab3087df59a5882410a8", - "traincheck/instrumentor/proxy_wrapper/subclass.py": "6152488b768edb35617451424c4e494b8924bf72ab0c21f198bca9dab15588bb", + "traincheck/instrumentor/proxy_wrapper/proxy_registry.py": "1f7b69ddfad7e8140b05a75fc5e4b4d8acf6e818e1cf030fde714f8e79f5d07b", + "traincheck/instrumentor/proxy_wrapper/subclass.py": "f370473a0e9676701d03771babc25c2c01619cac3bb3517ad6a22560283bccfb", "traincheck/instrumentor/proxy_wrapper/torch_proxy.py": "d6abd7f2eac207757fd90cbf55e179ae506b25996bed906c5a094564ff6faf2f", "traincheck/instrumentor/proxy_wrapper/utils.py": "5f058609e89a96a3545561ed82d83f7dd6a153bb0376608654c199691984dc14", "traincheck/instrumentor/replace_functions.py": "ca4f10b52083a4a5c251db194b7608406cbe36c85372dfdb9df17644f3d20b62", "traincheck/instrumentor/source_file.py": "dad92c793efb41161d86250e89aa6a9636749564e567df8b9af92a477eb4858d", "traincheck/instrumentor/torchscript_compat.py": "1b42072c80b29c2432f3ef83d6849365ecc0ae8c015dd2051a5b192ba209f88a", - "traincheck/instrumentor/tracer.py": "be6511d748415b17dc8056e2d120a2901003fcc242cfd2e0a16772432021bd9d", + "traincheck/instrumentor/tracer.py": "e5a74e766eea7b26141a0b14ee26d2c6f852a2819869bb9151becfe921f262c1", "traincheck/instrumentor/types.py": "258679aa8d0b40247d900ba2e26ab03b25cb4626b046ab6d724973226c7b5469", "traincheck/instrumentor/variable.py": "b01219cdd74f011055e0bc2a645ed283dbc47e66a834d5555b814fa815a9eab8", "traincheck/invariant/DistinctArgumentRelation.py": "6e01540583668e30718ba90c341fd540cd9df53f20525a6cf502fcceee8ed7bc", @@ -963,18 +267,18 @@ "traincheck/trace/utils.py": "6356156e4c5bbb6c111140302660f36ad1c126e5bc977bd55c81f097022102a3", "traincheck/utils.py": "71034730f74a15978d37dc9fc9b748133389a69e0ea7f9c9109dd1bc573fd272" }, - "source_archive_sha256": "ac1a830f490bef1af6b1b15ba49d116ff28a8de2e57f51dd80afc714f4e8d909", - "source_commit": "88b6df2401e187ca6bd81bf0a83bc7c5d9a5cd54", - "source_tree": "5415a3b0e5eb08d119dffa91631d76866e6e1d11" + "source_archive_sha256": "efc665f1b6914964f3ac93ac76a3e00791b5cf07c310818c65e7711496260537", + "source_commit": "7a40649d2ce55c1a92380e26cd0f74b61c05824b", + "source_tree": "c84687d977b748a5080f034ace78c78b45cec172" }, "candidate": { "adapter_sha256": "a5b5e597d4921a5b4baceeada541a6a1765f13c97ab3bd1c2ef79dea331f0934", - "production_files_verified": 77, + "production_files_verified": 78, "production_hashes": { "traincheck/__init__.py": "47fdcb090c4c8c5822769d642dc23a520e8d8ff3e4724583450723e05408164b", "traincheck/checker.py": "44dc20fc66ca82bb6b71fa02033aae3b7d489ae7afbc1611c44e857f399716b2", "traincheck/checker_online.py": "2824c9ccd2f684c86884c68f2214a1a4be1d69c6098ce30b817d03f1fe1077d6", - "traincheck/collect_trace.py": "c1db96c798af9689f8ee7f7b0b929d90694d49a234cb217beaf435d4b8d5cfb0", + "traincheck/collect_trace.py": "2e0a0b9a4e05fef450d09788e1747db2d3f1c4aa86f2828b63229cbbb7445d00", "traincheck/config/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "traincheck/config/config.py": "4c940a4d92f5de5fe8d3885e570078e479d9842d9626651fd60483e78f0160a9", "traincheck/developer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", @@ -983,6 +287,7 @@ "traincheck/inference_sampling.py": "d261b3416aba3ead6fa1e9b15091511eb3525137d531e4f0147146e76308976f", "traincheck/instrumentor/__init__.py": "228b4f6a11cde8118cde9aba2f98b60a839f4331e7fa2b06c37cee99e79fcd50", "traincheck/instrumentor/caches.py": "320eb1ffa1ce5f5b97fe4675a16d47e9591265779fdc29b94b72cdf5df93f168", + "traincheck/instrumentor/constructor_snapshot.py": "efdbdf75433ce2c3fcb0985107f59fa8e0a8ced6808b824a990030cfbf0845db", "traincheck/instrumentor/control.py": "3612a39b61e85fb77b40008b7fcd2f7569b246e8985ca64a109ea53733cd29ef", "traincheck/instrumentor/dumper.py": "faf87dcfb0aa08dd97bb544aa80120925ec1eed05d03947a1de743fdf5da53e2", "traincheck/instrumentor/proxy_wrapper/__init__.py": "7c71aaf70b978ee04e236497af6f803e4436049bc0acdf18c3edb48bfcece63d", @@ -994,14 +299,14 @@ "traincheck/instrumentor/proxy_wrapper/proxy_handler.py": "4bab74647d35c64c5e9c702295dc4cb010302bf620401da0fad8539692b95ac7", "traincheck/instrumentor/proxy_wrapper/proxy_methods.py": "6ae18928aa12e89b33102368c497aff048e3d6bed330eca2118fac051b8e3a97", "traincheck/instrumentor/proxy_wrapper/proxy_observer.py": "10e83570fbe1f72089e4a594a308f83e045f7d74bb9e49193012645ac906d97b", - "traincheck/instrumentor/proxy_wrapper/proxy_registry.py": "9dfecae65aee4185152d152a5c2fb0e1e1b63339efceab3087df59a5882410a8", - "traincheck/instrumentor/proxy_wrapper/subclass.py": "6152488b768edb35617451424c4e494b8924bf72ab0c21f198bca9dab15588bb", + "traincheck/instrumentor/proxy_wrapper/proxy_registry.py": "1f7b69ddfad7e8140b05a75fc5e4b4d8acf6e818e1cf030fde714f8e79f5d07b", + "traincheck/instrumentor/proxy_wrapper/subclass.py": "f370473a0e9676701d03771babc25c2c01619cac3bb3517ad6a22560283bccfb", "traincheck/instrumentor/proxy_wrapper/torch_proxy.py": "d6abd7f2eac207757fd90cbf55e179ae506b25996bed906c5a094564ff6faf2f", "traincheck/instrumentor/proxy_wrapper/utils.py": "5f058609e89a96a3545561ed82d83f7dd6a153bb0376608654c199691984dc14", "traincheck/instrumentor/replace_functions.py": "ca4f10b52083a4a5c251db194b7608406cbe36c85372dfdb9df17644f3d20b62", "traincheck/instrumentor/source_file.py": "dad92c793efb41161d86250e89aa6a9636749564e567df8b9af92a477eb4858d", "traincheck/instrumentor/torchscript_compat.py": "1b42072c80b29c2432f3ef83d6849365ecc0ae8c015dd2051a5b192ba209f88a", - "traincheck/instrumentor/tracer.py": "be6511d748415b17dc8056e2d120a2901003fcc242cfd2e0a16772432021bd9d", + "traincheck/instrumentor/tracer.py": "e5a74e766eea7b26141a0b14ee26d2c6f852a2819869bb9151becfe921f262c1", "traincheck/instrumentor/types.py": "258679aa8d0b40247d900ba2e26ab03b25cb4626b046ab6d724973226c7b5469", "traincheck/instrumentor/variable.py": "b01219cdd74f011055e0bc2a645ed283dbc47e66a834d5555b814fa815a9eab8", "traincheck/invariant/DistinctArgumentRelation.py": "6e01540583668e30718ba90c341fd540cd9df53f20525a6cf502fcceee8ed7bc", @@ -1049,9 +354,9 @@ "traincheck/trace/utils.py": "6356156e4c5bbb6c111140302660f36ad1c126e5bc977bd55c81f097022102a3", "traincheck/utils.py": "71034730f74a15978d37dc9fc9b748133389a69e0ea7f9c9109dd1bc573fd272" }, - "source_archive_sha256": "b945a9d4c2b43def8481660bc3fc6ae45dbfbd34e3a5e8a0c848e5ca379276e6", - "source_commit": "3232887a8bdb6dbf6abcab233ead43a9697f0c18", - "source_tree": "cb74fdd3a8185deb0b3d31c0d20d2a7eac0b8097" + "source_archive_sha256": "793087ee08bbfb10ed60712d7d22da2153809236c26eb113d78592695754fe4e", + "source_commit": "035f7f1976c5d84bd01f5d96bb095fb07c9cc059", + "source_tree": "89fdd92ebc6a57e5878d27cfb1b18dc69ee28a0b" } }, "validation": { @@ -1910,6 +1215,7 @@ } ], "exact_model_loss_rng_parity": true, + "reference": "Each workload's candidate plain correctness case; every paired and profiled run checked against it", "source_and_raw_trace_hashes_verified": true }, "workloads": { @@ -1932,429 +1238,429 @@ "model.fc2.weight" ] }, - "median_reduction_percent_of_baseline_training": 8.67150236741045, - "median_reduction_seconds": 0.07534057646989822, + "median_reduction_percent_of_baseline_training": 9.96277605477705, + "median_reduction_seconds": 0.0665290355682373, "paired_reduction_percent": [ - 13.675527655335843, - 16.865382859319496, - 4.564578664356592, - 12.20544322789504, - -1.9939121206708794 + 7.064396033143514, + 13.47503614924583, + 10.059949814656402, + 9.825393357719033, + 7.328966441316239 ], "paired_reductions_seconds": [ - 0.12899113073945045, - 0.14653143286705017, - 0.037951767444610596, - 0.11024190858006477, - -0.016614548861980438 + 0.04763709753751755, + 0.08998306840658188, + 0.06571457535028458, + 0.06551166623830795, + 0.04950897395610809 ], "profiles": { "baseline": { "interpretation": "Separate cProfile run; cumulative rows overlap and are not an exclusive breakdown of ordinary timing.", "rows": [ { - "calls": 1503363, - "cumulative_seconds": 0.171535822, + "calls": 1503819, + "cumulative_seconds": 0.16993218200000001, "file": "~", "function": "", "line": 0, - "percent_of_profile_total": 10.72127954899098, - "self_seconds": 0.099762033 + "percent_of_profile_total": 12.385472890531892, + "self_seconds": 0.09892717200000001 }, { "calls": 211616, - "cumulative_seconds": 0.087001201, + "cumulative_seconds": 0.089078695, "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", "function": "__enter__", "line": 466, - "percent_of_profile_total": 5.43772248935242, - "self_seconds": 0.078316964 - }, - { - "calls": 260, - "cumulative_seconds": 0.60086757, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/tracer.py", - "function": "to_dict_args_kwargs", - "line": 160, - "percent_of_profile_total": 37.555241318008235, - "self_seconds": 0.000341501 + "percent_of_profile_total": 6.492482760248784, + "self_seconds": 0.08047354400000001 }, { "calls": 112, - "cumulative_seconds": 0.44745846, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/dumper.py", + "cumulative_seconds": 0.221655714, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/dumper.py", "function": "dump_tensor", "line": 302, - "percent_of_profile_total": 27.96691198542191, - "self_seconds": 0.009669434000000001 - }, - { - "calls": 211616, - "cumulative_seconds": 0.626706756, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/dumper.py", - "function": "safe_getattr", - "line": 328, - "percent_of_profile_total": 39.17023422849416, - "self_seconds": 0.18447759700000002 - }, - { - "calls": 112, - "cumulative_seconds": 0.38490406600000004, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/proxy_wrapper/hash.py", - "function": "hash_tensor_cpu", - "line": 47, - "percent_of_profile_total": 24.0571563596161, - "self_seconds": 0.175302747 + "percent_of_profile_total": 16.155332112304016, + "self_seconds": 0.0005632370000000001 }, { "calls": 112, - "cumulative_seconds": 0.437784751, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/proxy_wrapper/hash.py", + "cumulative_seconds": 0.22108786000000002, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/proxy_wrapper/hash.py", "function": "tensor_hash", "line": 95, - "percent_of_profile_total": 27.362288780453152, - "self_seconds": 0.015663134000000002 + "percent_of_profile_total": 16.11394418778022, + "self_seconds": 0.010103038 }, { "calls": 468, - "cumulative_seconds": 1.537323336, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/dumper.py", + "cumulative_seconds": 1.3077217060000002, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/dumper.py", "function": "convert_var_to_dict", "line": 343, - "percent_of_profile_total": 96.08531355301047, - "self_seconds": 0.22469802900000002 + "percent_of_profile_total": 95.31303339601158, + "self_seconds": 0.223835801 + }, + { + "calls": 260, + "cumulative_seconds": 0.599447363, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/tracer.py", + "function": "to_dict_args_kwargs", + "line": 166, + "percent_of_profile_total": 43.69060042868943, + "self_seconds": 0.000508692 + }, + { + "calls": 211616, + "cumulative_seconds": 0.6243659410000001, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/dumper.py", + "function": "safe_getattr", + "line": 328, + "percent_of_profile_total": 45.50678597198814, + "self_seconds": 0.18085563400000002 + }, + { + "calls": 112, + "cumulative_seconds": 0.184003886, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/proxy_wrapper/hash.py", + "function": "hash_tensor_cpu", + "line": 47, + "percent_of_profile_total": 13.411086204998654, + "self_seconds": 0.158479586 }, { "calls": 64, - "cumulative_seconds": 0.6535382190000001, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/proxy_wrapper/subclass.py", + "cumulative_seconds": 0.42377681500000003, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/proxy_wrapper/subclass.py", "function": "dump_trace", - "line": 183, - "percent_of_profile_total": 40.84724613293129, - "self_seconds": 0.00018410100000000002 + "line": 218, + "percent_of_profile_total": 30.88688788694803, + "self_seconds": 0.000185841 }, { "calls": 1780, - "cumulative_seconds": 0.00043699300000000005, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/utils.py", + "cumulative_seconds": 0.00044063900000000005, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/utils.py", "function": "safe_getattr", "line": 21, - "percent_of_profile_total": 0.027312803001300897, - "self_seconds": 0.00023170500000000002 + "percent_of_profile_total": 0.03211588484758633, + "self_seconds": 0.000238251 }, { - "calls": 584, - "cumulative_seconds": 0.004350003, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/dumper.py", - "function": "serialize", - "line": 85, - "percent_of_profile_total": 0.27188255874594763, - "self_seconds": 0.00040800200000000003 + "calls": 260, + "cumulative_seconds": 0.293872481, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/tracer.py", + "function": "to_dict_return_value", + "line": 210, + "percent_of_profile_total": 21.418836643307788, + "self_seconds": 0.000125961 }, { "calls": 520, - "cumulative_seconds": 0.006313733, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/dumper.py", + "cumulative_seconds": 0.006746617000000001, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/dumper.py", "function": "dump_trace_API", "line": 241, - "percent_of_profile_total": 0.394619011361309, - "self_seconds": 0.00042752400000000004 + "percent_of_profile_total": 0.4917258224595833, + "self_seconds": 0.00042564800000000004 }, { "calls": 64, - "cumulative_seconds": 0.000849358, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/dumper.py", + "cumulative_seconds": 0.0010291760000000001, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/dumper.py", "function": "dump_trace_VAR", "line": 250, - "percent_of_profile_total": 0.05308631426951673, - "self_seconds": 6.1226e-05 + "percent_of_profile_total": 0.07501128566445436, + "self_seconds": 5.3818000000000005e-05 }, { - "calls": 260, - "cumulative_seconds": 0.291809378, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/tracer.py", - "function": "to_dict_return_value", - "line": 184, - "percent_of_profile_total": 18.238580607117616, - "self_seconds": 0.0001231 + "calls": 584, + "cumulative_seconds": 0.004231365, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/dumper.py", + "function": "serialize", + "line": 85, + "percent_of_profile_total": 0.3084021865702017, + "self_seconds": 0.00040759300000000004 }, { "calls": 203335, - "cumulative_seconds": 0.085047091, + "cumulative_seconds": 0.084410277, "file": "/home/yuxuan/experiments/codex-inference-runtime-20261025/venv/lib/python3.11/site-packages/torch/nn/parameter.py", "function": "__instancecheck__", "line": 21, - "percent_of_profile_total": 5.315587303038517, - "self_seconds": 0.060003436 + "percent_of_profile_total": 6.152226053719406, + "self_seconds": 0.059368754 }, { "calls": 211616, - "cumulative_seconds": 0.066681273, + "cumulative_seconds": 0.067194185, "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", "function": "__exit__", "line": 487, - "percent_of_profile_total": 4.167692556459633, - "self_seconds": 0.058937562000000006 + "percent_of_profile_total": 4.897434652600911, + "self_seconds": 0.059458518 }, { "calls": 211616, - "cumulative_seconds": 0.158621435, + "cumulative_seconds": 0.158761849, "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", "function": "_add_filter", "line": 181, - "percent_of_profile_total": 9.914108477569789, - "self_seconds": 0.08925611400000001 + "percent_of_profile_total": 11.571325417572863, + "self_seconds": 0.089265255 }, { "calls": 211616, - "cumulative_seconds": 0.032625206000000004, + "cumulative_seconds": 0.033080641, "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", "function": "__init__", "line": 440, - "percent_of_profile_total": 2.039130659655555, - "self_seconds": 0.032625206000000004 + "percent_of_profile_total": 2.4110758626457103, + "self_seconds": 0.033080641 }, { "calls": 211616, - "cumulative_seconds": 0.227450228, + "cumulative_seconds": 0.22810855700000002, "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", "function": "simplefilter", "line": 165, - "percent_of_profile_total": 14.216024673083945, - "self_seconds": 0.061110547 + "percent_of_profile_total": 16.62564627588816, + "self_seconds": 0.06152440000000001 } ], - "sha256": "4b33291b26a436e50fa02aa5d32713c8ff6552284184829c3c94dfab103d2a94", - "total_seconds": 1.5999566209999987 + "sha256": "5e59942c772a2e5dc341de52946b26bfe79a96e2df96cada624a7f902b7496c1", + "total_seconds": 1.3720282100000003 }, "candidate": { "interpretation": "Separate cProfile run; cumulative rows overlap and are not an exclusive breakdown of ordinary timing.", "rows": [ { - "calls": 464357, - "cumulative_seconds": 0.020462695000000003, + "calls": 464813, + "cumulative_seconds": 0.020077905, "file": "~", "function": "", "line": 0, - "percent_of_profile_total": 1.5475891033585303, - "self_seconds": 0.018717590000000003 + "percent_of_profile_total": 1.7884348567809505, + "self_seconds": 0.018422498000000002 }, { "calls": 211616, - "cumulative_seconds": 0.08338677300000001, + "cumulative_seconds": 0.082044563, "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", "function": "__enter__", "line": 466, - "percent_of_profile_total": 6.306523224777151, - "self_seconds": 0.07591249800000001 + "percent_of_profile_total": 7.3081009337657825, + "self_seconds": 0.074606007 }, { - "calls": 260, - "cumulative_seconds": 0.466124882, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/tracer.py", - "function": "to_dict_args_kwargs", - "line": 160, - "percent_of_profile_total": 35.25292187502578, - "self_seconds": 0.00034057300000000005 + "calls": 112, + "cumulative_seconds": 0.195671804, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/proxy_wrapper/hash.py", + "function": "hash_tensor_cpu", + "line": 47, + "percent_of_profile_total": 17.429421782940512, + "self_seconds": 0.172274382 }, { "calls": 112, - "cumulative_seconds": 0.419081605, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/dumper.py", + "cumulative_seconds": 0.231207348, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/dumper.py", "function": "dump_tensor", "line": 318, - "percent_of_profile_total": 31.695049225725334, - "self_seconds": 0.004339631 + "percent_of_profile_total": 20.594742345233897, + "self_seconds": 0.0005136580000000001 }, { - "calls": 211616, - "cumulative_seconds": 0.60530919, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/dumper.py", - "function": "safe_getattr", - "line": 344, - "percent_of_profile_total": 45.77940034813489, - "self_seconds": 0.177432474 - }, - { - "calls": 112, - "cumulative_seconds": 0.365371013, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/proxy_wrapper/hash.py", - "function": "hash_tensor_cpu", - "line": 47, - "percent_of_profile_total": 27.63292901555088, - "self_seconds": 0.168094142 + "calls": 260, + "cumulative_seconds": 0.46156553600000005, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/tracer.py", + "function": "to_dict_args_kwargs", + "line": 166, + "percent_of_profile_total": 41.11384595510252, + "self_seconds": 0.000501468 }, { "calls": 112, - "cumulative_seconds": 0.414737693, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/proxy_wrapper/hash.py", + "cumulative_seconds": 0.23068943700000002, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/proxy_wrapper/hash.py", "function": "tensor_hash", "line": 95, - "percent_of_profile_total": 31.366520120583107, - "self_seconds": 0.013457708 + "percent_of_profile_total": 20.548609539788796, + "self_seconds": 0.009633974 }, { "calls": 468, - "cumulative_seconds": 1.257303035, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/dumper.py", + "cumulative_seconds": 1.06152485, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/dumper.py", "function": "convert_var_to_dict", "line": 362, - "percent_of_profile_total": 95.08955084291725, - "self_seconds": 0.14695986900000002 + "percent_of_profile_total": 94.55508645345071, + "self_seconds": 0.146122746 + }, + { + "calls": 211616, + "cumulative_seconds": 0.5983217670000001, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/dumper.py", + "function": "safe_getattr", + "line": 344, + "percent_of_profile_total": 53.29537636887765, + "self_seconds": 0.17498206900000002 }, { "calls": 64, - "cumulative_seconds": 0.574950325, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/proxy_wrapper/subclass.py", + "cumulative_seconds": 0.38794965600000003, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/proxy_wrapper/subclass.py", "function": "dump_trace", - "line": 183, - "percent_of_profile_total": 43.483366093393144, - "self_seconds": 0.000187755 + "line": 218, + "percent_of_profile_total": 34.55652805741332, + "self_seconds": 0.000167806 }, { "calls": 1780, - "cumulative_seconds": 0.00045410000000000003, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/utils.py", + "cumulative_seconds": 0.00044975900000000004, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/utils.py", "function": "safe_getattr", "line": 21, - "percent_of_profile_total": 0.03434348270524037, - "self_seconds": 0.000239024 + "percent_of_profile_total": 0.04006218142535009, + "self_seconds": 0.00022905 }, { "calls": 584, - "cumulative_seconds": 0.004386208, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/dumper.py", + "cumulative_seconds": 0.00550764, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/dumper.py", "function": "serialize", "line": 101, - "percent_of_profile_total": 0.33172794228052616, - "self_seconds": 0.00041095600000000003 + "percent_of_profile_total": 0.49059179005982123, + "self_seconds": 0.0016963430000000001 + }, + { + "calls": 260, + "cumulative_seconds": 0.22334777700000003, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/tracer.py", + "function": "to_dict_return_value", + "line": 210, + "percent_of_profile_total": 19.894652832122613, + "self_seconds": 0.00012765900000000002 }, { "calls": 64, - "cumulative_seconds": 0.000981751, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/dumper.py", + "cumulative_seconds": 0.0022816850000000003, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/dumper.py", "function": "dump_trace_VAR", "line": 266, - "percent_of_profile_total": 0.0742496112956451, - "self_seconds": 6.239800000000001e-05 + "percent_of_profile_total": 0.20324057645427865, + "self_seconds": 5.284e-05 }, { "calls": 520, - "cumulative_seconds": 0.006941215000000001, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/dumper.py", + "cumulative_seconds": 0.006337504000000001, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/dumper.py", "function": "dump_trace_API", "line": 257, - "percent_of_profile_total": 0.5249625573791126, - "self_seconds": 0.00043592500000000004 - }, - { - "calls": 260, - "cumulative_seconds": 0.22543626700000002, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/tracer.py", - "function": "to_dict_return_value", - "line": 184, - "percent_of_profile_total": 17.04969508224719, - "self_seconds": 0.000130289 + "percent_of_profile_total": 0.5645117385797324, + "self_seconds": 0.000423195 }, { "calls": 2817, - "cumulative_seconds": 0.0020264320000000003, + "cumulative_seconds": 0.0019264170000000001, "file": "/home/yuxuan/experiments/codex-inference-runtime-20261025/venv/lib/python3.11/site-packages/torch/nn/parameter.py", "function": "__instancecheck__", "line": 21, - "percent_of_profile_total": 0.15325860459226084, - "self_seconds": 0.001462605 + "percent_of_profile_total": 0.1715951595296117, + "self_seconds": 0.001401609 }, { "calls": 211616, - "cumulative_seconds": 0.064519857, + "cumulative_seconds": 0.063982831, "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", "function": "__exit__", "line": 487, - "percent_of_profile_total": 4.879622534737021, - "self_seconds": 0.057657651000000004 + "percent_of_profile_total": 5.699256231958701, + "self_seconds": 0.057104195 }, { "calls": 211616, - "cumulative_seconds": 0.15187645200000002, + "cumulative_seconds": 0.15220593500000001, "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", "function": "_add_filter", "line": 181, - "percent_of_profile_total": 11.486382520579761, - "self_seconds": 0.08526881400000001 + "percent_of_profile_total": 13.557709310953294, + "self_seconds": 0.085092887 }, { "calls": 211616, - "cumulative_seconds": 0.031807165000000005, + "cumulative_seconds": 0.031969863, "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", "function": "__init__", "line": 440, - "percent_of_profile_total": 2.4055688638630848, - "self_seconds": 0.031807165000000005 + "percent_of_profile_total": 2.847708331905725, + "self_seconds": 0.031969863 }, { "calls": 211616, - "cumulative_seconds": 0.21994069500000002, + "cumulative_seconds": 0.219395593, "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", "function": "simplefilter", "line": 165, - "percent_of_profile_total": 16.634066185666256, - "self_seconds": 0.059943076000000005 + "percent_of_profile_total": 19.54261293423426, + "self_seconds": 0.05925521500000001 } ], - "sha256": "7862ce8c96dc80a4a95a25f1214a5e32b733419ba3b32082b301ce91b1b4731e", - "total_seconds": 1.3222304910000007 + "sha256": "43ec809c22a641250f2ee269119905729cf082c8c5aa51f7489291d6e8f71c1c", + "total_seconds": 1.122652297 } }, - "speedup_ratio_of_medians": 1.0949484836846382, + "speedup_ratio_of_medians": 1.1106517462248529, "variants": { "baseline": { "peak_RSS_bytes": { - "max": 592883712.0, - "median": 588767232.0, - "min": 584212480.0, + "max": 588607488.0, + "median": 583864320.0, + "min": 582459392.0, "values": [ - 584212480.0, - 588173312.0, - 592883712.0, - 588767232.0, - 590761984.0 + 582459392.0, + 583032832.0, + 588607488.0, + 583864320.0, + 588369920.0 ] }, "post_metrics_exit_seconds": { - "max": 2.17674096301198, - "median": 2.077751439064741, - "min": 2.0082851946353912, + "max": 2.2820894457399845, + "median": 2.2715826109051704, + "min": 2.241224192082882, "values": [ - 2.1645433455705643, - 2.076347693800926, - 2.17674096301198, - 2.0082851946353912, - 2.077751439064741 + 2.241224192082882, + 2.2820894457399845, + 2.275044210255146, + 2.243912100791931, + 2.2715826109051704 ] }, "process_seconds": { - "max": 7.2223930433392525, - "median": 7.0179259441792965, - "min": 6.817594639956951, + "max": 6.9157307259738445, + "median": 6.815988626331091, + "min": 6.813528638333082, "values": [ - 7.2223930433392525, - 7.0179259441792965, - 7.1210783533751965, - 6.817594639956951, - 6.91872750595212 + 6.813528638333082, + 6.813662972301245, + 6.819815680384636, + 6.815988626331091, + 6.9157307259738445 ] }, "setup_seconds": { - "max": 2.778099913150072, - "median": 2.702235735952854, - "min": 2.592470161616802, + "max": 2.5969117917120457, + "median": 2.5764283053576946, + "min": 2.5584622099995613, "values": [ - 2.778099913150072, - 2.702235735952854, - 2.7643737457692623, - 2.592470161616802, - 2.694968421012163 + 2.568533480167389, + 2.5584622099995613, + 2.5764283053576946, + 2.5915246196091175, + 2.5969117917120457 ] }, "trace_bytes": { @@ -2370,65 +1676,65 @@ ] }, "training_seconds": { - "max": 0.9432259872555733, - "median": 0.8688295669853687, - "min": 0.8314407579600811, + "max": 0.6755246371030807, + "median": 0.6677760817110538, + "min": 0.6532296538352966, "values": [ - 0.9432259872555733, - 0.8688295669853687, - 0.8314407579600811, - 0.9032192155718803, - 0.833263847976923 + 0.6743265427649021, + 0.6677760817110538, + 0.6532296538352966, + 0.6667587123811245, + 0.6755246371030807 ] } }, "candidate": { "peak_RSS_bytes": { - "max": 588763136.0, - "median": 588550144.0, - "min": 582574080.0, + "max": 594067456.0, + "median": 588800000.0, + "min": 584409088.0, "values": [ - 588550144.0, - 588550144.0, - 588763136.0, - 582574080.0, - 588165120.0 + 587386880.0, + 594067456.0, + 588951552.0, + 588800000.0, + 584409088.0 ] }, "post_metrics_exit_seconds": { - "max": 2.1754636578261852, - "median": 2.1305285543203354, - "min": 2.0753588266670704, + "max": 2.4097439981997013, + "median": 2.23392204195261, + "min": 2.1729552559554577, "values": [ - 2.1305285543203354, - 2.1754636578261852, - 2.0753588266670704, - 2.13845232501626, - 2.092308398336172 + 2.4097439981997013, + 2.23392204195261, + 2.2292166277766228, + 2.1729552559554577, + 2.3070859499275684 ] }, "process_seconds": { - "max": 7.326598588377237, - "median": 7.019078567624092, - "min": 6.816429901868105, + "max": 6.918491534888744, + "median": 6.9149907901883125, + "min": 6.7165322713553905, "values": [ - 7.124290060251951, - 7.019078567624092, - 7.326598588377237, - 6.818062514066696, - 6.816429901868105 + 6.9149907901883125, + 6.916295755654573, + 6.7165322713553905, + 6.7182507291436195, + 6.918491534888744 ] }, "setup_seconds": { - "max": 2.9761409126222134, - "median": 2.70905052497983, - "min": 2.5668062567710876, + "max": 2.7161961905658245, + "median": 2.6359554640948772, + "min": 2.568043265491724, "values": [ - 2.70905052497983, - 2.7149592004716396, - 2.9761409126222134, - 2.5683862902224064, - 2.5668062567710876 + 2.568043265491724, + 2.7161961905658245, + 2.5984866246581078, + 2.6359554640948772, + 2.673058681190014 ] }, "trace_bytes": { @@ -2444,15 +1750,15 @@ ] }, "training_seconds": { - "max": 0.8498783968389034, - "median": 0.7934889905154705, - "min": 0.7222981341183186, + "max": 0.6266894452273846, + "median": 0.6012470461428165, + "min": 0.577793013304472, "values": [ - 0.8142348565161228, - 0.7222981341183186, - 0.7934889905154705, - 0.7929773069918156, - 0.8498783968389034 + 0.6266894452273846, + 0.577793013304472, + 0.587515078485012, + 0.6012470461428165, + 0.6260156631469727 ] } } @@ -2465,6 +1771,7 @@ "benchmark_source": "/home/yuxuan/experiments/codex-trace-impact-20261013/benchmarks/instr-overhead/overhead-e2e/mnist/main.py", "benchmark_source_sha256": "4478c5de92610a1e03d61c356801d51ffca749f99dac00f234d6b6ea2bcb2aa7", "cpu_threads": 1, + "cuda_available": false, "data": { "count": 6, "files": [ @@ -2504,11 +1811,11 @@ ], "observation_files": [ { - "path": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/static_analyzer/func_level/nn_func_level.log", + "path": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/static_analyzer/func_level/nn_func_level.log", "sha256": "0ea56ca02089d5b19a1c0697dea00d107fccd18e17d485ff97bc04a5378dfcba" }, { - "path": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/static_analyzer/func_level/optim_func_level.log", + "path": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/static_analyzer/func_level/optim_func_level.log", "sha256": "300c47f9babd033ddf68487dde01a828746ff795b5f0111761b2f8fda08ad900" } ], @@ -2518,9 +1825,9 @@ "python": "3.11.4 (main, Jul 5 2023, 13:45:01) [GCC 11.2.0]", "scan_proxy_in_args": true, "seed": 42, - "setup_seconds": 2.778099913150072, - "source_commit": "88b6df2401e187ca6bd81bf0a83bc7c5d9a5cd54", - "source_tree": "5415a3b0e5eb08d119dffa91631d76866e6e1d11", + "setup_seconds": 2.568533480167389, + "source_commit": "7a40649d2ce55c1a92380e26cd0f74b61c05824b", + "source_tree": "c84687d977b748a5080f034ace78c78b45cec172", "steps": 3, "tensor_dump_format": { "dump_tensor_full": false, @@ -2542,7 +1849,8 @@ ], "training_only_trace": true, "update": "Exact benchmark forward, NLL loss, backward, Adadelta(lr=1.0). Epoch scheduler/evaluation omitted.", - "variant": "baseline_subclass_fix", + "use_full_instr": false, + "variant": "final_correctness_without_metadata_fastpath", "workload_kind": "bounded_model_micro_workload", "wrapped_functions": 21032 } @@ -2587,447 +1895,447 @@ "model.input_emb.weight" ] }, - "median_reduction_percent_of_baseline_training": 18.90125826253589, - "median_reduction_seconds": 0.18266938999295235, + "median_reduction_percent_of_baseline_training": 18.43201701823739, + "median_reduction_seconds": 0.17684413865208626, "paired_reduction_percent": [ - 19.113870461709478, - 17.331180173553964, - 18.858957322850234, - 19.979830753413463, - 23.81235705577393 + 15.601390824394477, + 20.54804037418375, + 17.7208926939699, + 15.512118752865149, + 25.14732482059326 ], "paired_reductions_seconds": [ - 0.1831599548459053, - 0.16592683270573616, - 0.18226057663559914, - 0.19419122487306595, - 0.24496668577194214 + 0.14744044840335846, + 0.19879167526960373, + 0.16855186223983765, + 0.14882946759462357, + 0.257242564111948 ], "profiles": { "baseline": { "interpretation": "Separate cProfile run; cumulative rows overlap and are not an exclusive breakdown of ordinary timing.", "rows": [ { - "calls": 3622491, - "cumulative_seconds": 0.427562979, + "calls": 3622987, + "cumulative_seconds": 0.41540509400000003, "file": "~", "function": "", "line": 0, - "percent_of_profile_total": 15.08458680801047, - "self_seconds": 0.24889274400000003 + "percent_of_profile_total": 15.272859297011918, + "self_seconds": 0.24020582000000001 }, { "calls": 516009, - "cumulative_seconds": 0.216041698, + "cumulative_seconds": 0.211889453, "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", "function": "__enter__", "line": 466, - "percent_of_profile_total": 7.622034431636286, - "self_seconds": 0.19456483700000002 + "percent_of_profile_total": 7.790366196592234, + "self_seconds": 0.190857086 }, { - "calls": 670, - "cumulative_seconds": 1.5786948920000001, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/tracer.py", - "function": "to_dict_args_kwargs", - "line": 160, - "percent_of_profile_total": 55.69696468443942, - "self_seconds": 0.0008960670000000001 + "calls": 64, + "cumulative_seconds": 0.005668298, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/dumper.py", + "function": "dump_tensor", + "line": 302, + "percent_of_profile_total": 0.20840167599758433, + "self_seconds": 7.4091e-05 }, { "calls": 64, - "cumulative_seconds": 0.005761861, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/proxy_wrapper/hash.py", + "cumulative_seconds": 0.005591704, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/proxy_wrapper/hash.py", "function": "tensor_hash", "line": 95, - "percent_of_profile_total": 0.20328067840080707, - "self_seconds": 0.000760977 + "percent_of_profile_total": 0.20558560705213386, + "self_seconds": 0.00060932 }, { - "calls": 64, - "cumulative_seconds": 0.004656917, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/proxy_wrapper/hash.py", - "function": "hash_tensor_cpu", - "line": 47, - "percent_of_profile_total": 0.16429782790946382, - "self_seconds": 0.004139646 + "calls": 1773, + "cumulative_seconds": 2.641869315, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/dumper.py", + "function": "convert_var_to_dict", + "line": 343, + "percent_of_profile_total": 97.1314481018094, + "self_seconds": 0.538952239 + }, + { + "calls": 670, + "cumulative_seconds": 1.513930065, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/tracer.py", + "function": "to_dict_args_kwargs", + "line": 166, + "percent_of_profile_total": 55.6614283315965, + "self_seconds": 0.0012697680000000001 }, { "calls": 516009, - "cumulative_seconds": 1.579442368, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/dumper.py", + "cumulative_seconds": 1.5134438030000001, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/dumper.py", "function": "safe_getattr", "line": 328, - "percent_of_profile_total": 55.72333592601716, - "self_seconds": 0.46544991100000005 - }, - { - "calls": 1773, - "cumulative_seconds": 2.748502059, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/dumper.py", - "function": "convert_var_to_dict", - "line": 343, - "percent_of_profile_total": 96.96821272493993, - "self_seconds": 0.559197797 + "percent_of_profile_total": 55.64355033439629, + "self_seconds": 0.44812079600000004 }, { "calls": 64, - "cumulative_seconds": 0.005844259, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/dumper.py", - "function": "dump_tensor", - "line": 302, - "percent_of_profile_total": 0.20618771162130123, - "self_seconds": 7.9787e-05 + "cumulative_seconds": 0.0046741460000000006, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/proxy_wrapper/hash.py", + "function": "hash_tensor_cpu", + "line": 47, + "percent_of_profile_total": 0.17185050261249582, + "self_seconds": 0.004206617 }, { "calls": 64, - "cumulative_seconds": 0.213352694, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/proxy_wrapper/subclass.py", + "cumulative_seconds": 0.20849510200000002, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/proxy_wrapper/subclass.py", "function": "dump_trace", - "line": 183, - "percent_of_profile_total": 7.527165333380968, - "self_seconds": 0.00015941500000000002 + "line": 218, + "percent_of_profile_total": 7.66556886989486, + "self_seconds": 0.000155234 }, { "calls": 8292, - "cumulative_seconds": 0.002800037, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/utils.py", + "cumulative_seconds": 0.002601659, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/utils.py", "function": "safe_getattr", "line": 21, - "percent_of_profile_total": 0.09878638532018745, - "self_seconds": 0.0011357350000000001 + "percent_of_profile_total": 0.09565306834153729, + "self_seconds": 0.001095569 + }, + { + "calls": 1404, + "cumulative_seconds": 0.01060878, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/dumper.py", + "function": "serialize", + "line": 85, + "percent_of_profile_total": 0.39004433646390013, + "self_seconds": 0.000932733 + }, + { + "calls": 670, + "cumulative_seconds": 0.950235758, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/tracer.py", + "function": "to_dict_return_value", + "line": 210, + "percent_of_profile_total": 34.936540838190744, + "self_seconds": 0.000306486 }, { "calls": 1340, - "cumulative_seconds": 0.01722223, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/dumper.py", + "cumulative_seconds": 0.015419796000000001, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/dumper.py", "function": "dump_trace_API", "line": 241, - "percent_of_profile_total": 0.6076069169274877, - "self_seconds": 0.00117887 + "percent_of_profile_total": 0.566927026409135, + "self_seconds": 0.001075404 }, { "calls": 64, - "cumulative_seconds": 0.000883031, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/dumper.py", + "cumulative_seconds": 0.001013873, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/dumper.py", "function": "dump_trace_VAR", "line": 250, - "percent_of_profile_total": 0.03115367426061529, - "self_seconds": 4.7865e-05 - }, - { - "calls": 670, - "cumulative_seconds": 0.987921398, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/tracer.py", - "function": "to_dict_return_value", - "line": 184, - "percent_of_profile_total": 34.854247957754225, - "self_seconds": 0.00032125000000000003 - }, - { - "calls": 1404, - "cumulative_seconds": 0.012092690000000001, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/instrumentor/dumper.py", - "function": "serialize", - "line": 85, - "percent_of_profile_total": 0.42663476728971, - "self_seconds": 0.001008543 + "percent_of_profile_total": 0.03727623926065616, + "self_seconds": 4.5441e-05 }, { "calls": 495303, - "cumulative_seconds": 0.21179952300000002, + "cumulative_seconds": 0.207537147, "file": "/home/yuxuan/experiments/codex-inference-runtime-20261025/venv/lib/python3.11/site-packages/torch/nn/parameter.py", "function": "__instancecheck__", "line": 21, - "percent_of_profile_total": 7.472368861450725, - "self_seconds": 0.150287937 + "percent_of_profile_total": 7.630348522000259, + "self_seconds": 0.146889277 }, { "calls": 1, - "cumulative_seconds": 1.322e-06, + "cumulative_seconds": 1.39e-06, "file": "/home/yuxuan/experiments/codex-inference-runtime-20261025/venv/lib/python3.11/site-packages/torch/nn/parameter.py", "function": "__instancecheck__", "line": 240, - "percent_of_profile_total": 4.6640669888750686e-05, - "self_seconds": 1.002e-06 + "percent_of_profile_total": 5.110499300436254e-05, + "self_seconds": 1.044e-06 }, { "calls": 516009, - "cumulative_seconds": 0.170823455, + "cumulative_seconds": 0.162974535, "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", "function": "__exit__", "line": 487, - "percent_of_profile_total": 6.0267173781473975, - "self_seconds": 0.15120122700000002 + "percent_of_profile_total": 5.991951418031825, + "self_seconds": 0.144182988 }, { "calls": 516009, - "cumulative_seconds": 0.406155918, + "cumulative_seconds": 0.38795989000000003, "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", "function": "_add_filter", "line": 181, - "percent_of_profile_total": 14.329337439334713, - "self_seconds": 0.22709152400000002 + "percent_of_profile_total": 14.263803931239758, + "self_seconds": 0.216126051 }, { "calls": 516009, - "cumulative_seconds": 0.082309591, + "cumulative_seconds": 0.07956756400000001, "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", "function": "__init__", "line": 440, - "percent_of_profile_total": 2.903914116875253, - "self_seconds": 0.082309591 + "percent_of_profile_total": 2.9253955407152294, + "self_seconds": 0.07956756400000001 }, { "calls": 516009, - "cumulative_seconds": 0.579480675, + "cumulative_seconds": 0.553407844, "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", "function": "simplefilter", "line": 165, - "percent_of_profile_total": 20.444301716781712, - "self_seconds": 0.153575495 + "percent_of_profile_total": 20.346693522431192, + "self_seconds": 0.146483431 } ], - "sha256": "f7b2024e12967a7cc232ee63460a2fb4a2860979a48225bc638982993aa31cd9", - "total_seconds": 2.834436132999999 + "sha256": "2428566e23bf2a60b055d3755c2ada906bb809a40ae26a05455788e5727459df", + "total_seconds": 2.7198907939999986 }, "candidate": { "interpretation": "Separate cProfile run; cumulative rows overlap and are not an exclusive breakdown of ordinary timing.", "rows": [ { - "calls": 1154245, - "cumulative_seconds": 0.052964309, + "calls": 1154741, + "cumulative_seconds": 0.053038279, "file": "~", "function": "", "line": 0, - "percent_of_profile_total": 2.4369284236685465, - "self_seconds": 0.047591463 + "percent_of_profile_total": 2.483533581874309, + "self_seconds": 0.04782687 }, { "calls": 516009, - "cumulative_seconds": 0.203956469, + "cumulative_seconds": 0.20110108200000001, "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", "function": "__enter__", "line": 466, - "percent_of_profile_total": 9.384193353625603, - "self_seconds": 0.18543056800000002 + "percent_of_profile_total": 9.416619466447228, + "self_seconds": 0.182952516 }, { "calls": 64, - "cumulative_seconds": 0.005312453, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/proxy_wrapper/hash.py", - "function": "tensor_hash", - "line": 95, - "percent_of_profile_total": 0.24443003145954836, - "self_seconds": 0.000703901 + "cumulative_seconds": 0.00432889, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/proxy_wrapper/hash.py", + "function": "hash_tensor_cpu", + "line": 47, + "percent_of_profile_total": 0.20270159382886233, + "self_seconds": 0.0039002060000000002 }, { "calls": 64, - "cumulative_seconds": 0.005392115, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/dumper.py", + "cumulative_seconds": 0.005282504, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/dumper.py", "function": "dump_tensor", "line": 318, - "percent_of_profile_total": 0.24809534109450057, - "self_seconds": 7.7172e-05 - }, - { - "calls": 64, - "cumulative_seconds": 0.004293179, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/proxy_wrapper/hash.py", - "function": "hash_tensor_cpu", - "line": 47, - "percent_of_profile_total": 0.19753245403422345, - "self_seconds": 0.003818175 - }, - { - "calls": 516009, - "cumulative_seconds": 1.4984058370000002, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/dumper.py", - "function": "safe_getattr", - "line": 344, - "percent_of_profile_total": 68.9428002237537, - "self_seconds": 0.437565478 + "percent_of_profile_total": 0.2473548600697501, + "self_seconds": 7.26e-05 }, { "calls": 670, - "cumulative_seconds": 1.207365155, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/tracer.py", + "cumulative_seconds": 1.184774024, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/tracer.py", "function": "to_dict_args_kwargs", - "line": 160, - "percent_of_profile_total": 55.5517955302028, - "self_seconds": 0.0008465870000000001 + "line": 166, + "percent_of_profile_total": 55.47740482937538, + "self_seconds": 0.0012466880000000001 + }, + { + "calls": 64, + "cumulative_seconds": 0.005207354, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/proxy_wrapper/hash.py", + "function": "tensor_hash", + "line": 95, + "percent_of_profile_total": 0.24383593841171788, + "self_seconds": 0.000586207 }, { "calls": 1773, - "cumulative_seconds": 2.0914098190000003, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/dumper.py", + "cumulative_seconds": 2.056601617, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/dumper.py", "function": "convert_var_to_dict", "line": 362, - "percent_of_profile_total": 96.22736763091898, - "self_seconds": 0.363128973 + "percent_of_profile_total": 96.30099763147493, + "self_seconds": 0.35871008800000004 + }, + { + "calls": 516009, + "cumulative_seconds": 1.4721962940000002, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/dumper.py", + "function": "safe_getattr", + "line": 344, + "percent_of_profile_total": 68.93604023727663, + "self_seconds": 0.433983473 }, { "calls": 64, - "cumulative_seconds": 0.162592268, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/proxy_wrapper/subclass.py", + "cumulative_seconds": 0.161195886, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/proxy_wrapper/subclass.py", "function": "dump_trace", - "line": 183, - "percent_of_profile_total": 7.480994783825724, - "self_seconds": 0.00014322 + "line": 218, + "percent_of_profile_total": 7.548046499415692, + "self_seconds": 0.000153255 }, { "calls": 8292, - "cumulative_seconds": 0.002711686, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/utils.py", + "cumulative_seconds": 0.002652792, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/utils.py", "function": "safe_getattr", "line": 21, - "percent_of_profile_total": 0.1247667498024767, - "self_seconds": 0.0011258210000000002 + "percent_of_profile_total": 0.12421779405262211, + "self_seconds": 0.001105293 }, { "calls": 64, - "cumulative_seconds": 0.000922073, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/dumper.py", + "cumulative_seconds": 0.000806478, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/dumper.py", "function": "dump_trace_VAR", "line": 266, - "percent_of_profile_total": 0.04242528496685056, - "self_seconds": 4.7424000000000005e-05 + "percent_of_profile_total": 0.037763578189307936, + "self_seconds": 4.4666e-05 }, { "calls": 1340, - "cumulative_seconds": 0.016164994000000002, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/dumper.py", + "cumulative_seconds": 0.015288563000000002, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/dumper.py", "function": "dump_trace_API", "line": 257, - "percent_of_profile_total": 0.7437637550795105, - "self_seconds": 0.001122561 - }, - { - "calls": 670, - "cumulative_seconds": 0.752377733, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/tracer.py", - "function": "to_dict_return_value", - "line": 184, - "percent_of_profile_total": 34.61747575868505, - "self_seconds": 0.00031704700000000004 + "percent_of_profile_total": 0.7158916228993976, + "self_seconds": 0.001078509 }, { "calls": 1404, - "cumulative_seconds": 0.011245191, - "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/candidate/source/traincheck/instrumentor/dumper.py", + "cumulative_seconds": 0.010575724, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/dumper.py", "function": "serialize", "line": 101, - "percent_of_profile_total": 0.517399850859599, - "self_seconds": 0.0009738310000000001 + "percent_of_profile_total": 0.4952115001060667, + "self_seconds": 0.000939728 + }, + { + "calls": 670, + "cumulative_seconds": 0.742331382, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/tracer.py", + "function": "to_dict_return_value", + "line": 210, + "percent_of_profile_total": 34.759893247595116, + "self_seconds": 0.00031331 }, { "calls": 8937, - "cumulative_seconds": 0.0062934160000000005, + "cumulative_seconds": 0.006075963, "file": "/home/yuxuan/experiments/codex-inference-runtime-20261025/venv/lib/python3.11/site-packages/torch/nn/parameter.py", "function": "__instancecheck__", "line": 21, - "percent_of_profile_total": 0.2895648904316001, - "self_seconds": 0.0044612110000000005 + "percent_of_profile_total": 0.28450881961546626, + "self_seconds": 0.004308247 }, { "calls": 1, - "cumulative_seconds": 1.5100000000000002e-06, + "cumulative_seconds": 1.334e-06, "file": "/home/yuxuan/experiments/codex-inference-runtime-20261025/venv/lib/python3.11/site-packages/torch/nn/parameter.py", "function": "__instancecheck__", "line": 240, - "percent_of_profile_total": 6.947625654361893e-05, - "self_seconds": 1.2620000000000002e-06 + "percent_of_profile_total": 6.246495664424423e-05, + "self_seconds": 1.068e-06 }, { "calls": 516009, - "cumulative_seconds": 0.161353457, + "cumulative_seconds": 0.15780037300000002, "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", "function": "__exit__", "line": 487, - "percent_of_profile_total": 7.423996140881979, - "self_seconds": 0.144325115 + "percent_of_profile_total": 7.389050568133857, + "self_seconds": 0.141191683 }, { "calls": 516009, - "cumulative_seconds": 0.381938691, + "cumulative_seconds": 0.37356121800000003, "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", "function": "_add_filter", "line": 181, - "percent_of_profile_total": 17.573291708509934, - "self_seconds": 0.213641867 + "percent_of_profile_total": 17.492117905802896, + "self_seconds": 0.20996710500000001 }, { "calls": 516009, - "cumulative_seconds": 0.079526229, + "cumulative_seconds": 0.07808938900000001, "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", "function": "__init__", "line": 440, - "percent_of_profile_total": 3.659062707252045, - "self_seconds": 0.079526229 + "percent_of_profile_total": 3.6565594439734044, + "self_seconds": 0.07808938900000001 }, { "calls": 516009, - "cumulative_seconds": 0.552660111, + "cumulative_seconds": 0.540450281, "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", "function": "simplefilter", "line": 165, - "percent_of_profile_total": 25.428315002159547, - "self_seconds": 0.15112924600000002 + "percent_of_profile_total": 25.306749153699105, + "self_seconds": 0.146992578 } ], - "sha256": "a3fd25631d2d65a4819675afba8347fc50c95dbf22a56276f2519f32271cc158", - "total_seconds": 2.1734043759999997 + "sha256": "99d75dabca1f2ee57a623d97c35581f222b6fd148a2c13b4f4f8575a2df1280f", + "total_seconds": 2.1355974160000004 } }, - "speedup_ratio_of_medians": 1.2330647536274206, + "speedup_ratio_of_medians": 1.225971224792435, "variants": { "baseline": { "peak_RSS_bytes": { - "max": 540819456.0, - "median": 540479488.0, - "min": 540413952.0, + "max": 540844032.0, + "median": 540766208.0, + "min": 539996160.0, "values": [ - 540446720.0, - 540524544.0, - 540819456.0, - 540479488.0, - 540413952.0 + 540766208.0, + 540528640.0, + 540844032.0, + 540782592.0, + 539996160.0 ] }, "post_metrics_exit_seconds": { - "max": 3.333291094750166, - "median": 2.0391327142715454, - "min": 1.9195968136191368, + "max": 2.292447991669178, + "median": 1.9001667201519012, + "min": 1.8594940602779388, "values": [ - 2.0391327142715454, - 1.9493112564086914, - 1.9195968136191368, - 2.38959226757288, - 3.333291094750166 + 1.9001667201519012, + 1.9286032617092133, + 1.8995989076793194, + 1.8594940602779388, + 2.292447991669178 ] }, "process_seconds": { - "max": 8.535529389977455, - "median": 7.0182711854577065, - "min": 6.815501172095537, + "max": 7.21809833869338, + "median": 6.719264965504408, + "min": 6.713514465838671, "values": [ - 7.0182711854577065, - 6.815501172095537, - 6.817034367471933, - 7.320687089115381, - 8.535529389977455 + 6.716304361820221, + 6.719264965504408, + 6.713514465838671, + 7.1309390515089035, + 7.21809833869338 ] }, "setup_seconds": { - "max": 2.8161566741764545, - "median": 2.602743424475193, - "min": 2.5676101446151733, + "max": 2.5845856182277203, + "median": 2.550613649189472, + "min": 2.526260081678629, "values": [ - 2.6592667177319527, - 2.5893340334296227, - 2.5676101446151733, - 2.602743424475193, - 2.8161566741764545 + 2.561651110649109, + 2.526260081678629, + 2.550613649189472, + 2.5845856182277203, + 2.535349767655134 ] }, "trace_bytes": { @@ -3043,65 +2351,65 @@ ] }, "training_seconds": { - "max": 1.0287376642227173, - "median": 0.9664403684437275, - "min": 0.9573891162872314, + "max": 1.022942066192627, + "median": 0.9594399705529213, + "min": 0.9450468234717846, "values": [ - 0.9582567550241947, - 0.9573891162872314, - 0.9664403684437275, - 0.9719362854957581, - 1.0287376642227173 + 0.9450468234717846, + 0.9674483388662338, + 0.9511476941406727, + 0.9594399705529213, + 1.022942066192627 ] } }, "candidate": { "peak_RSS_bytes": { - "max": 540966912.0, - "median": 540696576.0, - "min": 540524544.0, + "max": 540979200.0, + "median": 540778496.0, + "min": 540712960.0, "values": [ - 540622848.0, - 540524544.0, 540966912.0, - 540696576.0, - 540721152.0 + 540979200.0, + 540778496.0, + 540745728.0, + 540712960.0 ] }, "post_metrics_exit_seconds": { - "max": 2.2381970062851906, - "median": 2.0395359471440315, - "min": 2.011737398803234, + "max": 2.2137519530951977, + "median": 2.1560278981924057, + "min": 2.0495513416826725, "values": [ - 2.0347076691687107, - 2.011737398803234, - 2.0395359471440315, - 2.098541881889105, - 2.2381970062851906 + 2.1560278981924057, + 2.0495513416826725, + 2.2079070173203945, + 2.2137519530951977, + 2.0717223808169365 ] }, "process_seconds": { - "max": 7.020256042480469, - "median": 6.816576909273863, - "min": 6.71428145095706, + "max": 6.919477663934231, + "median": 6.816005241125822, + "min": 6.717189069837332, "values": [ - 6.91656219959259, - 6.71591405197978, - 6.71428145095706, - 6.816576909273863, - 7.020256042480469 + 6.816975295543671, + 6.815951332449913, + 6.816005241125822, + 6.919477663934231, + 6.717189069837332 ] }, "setup_seconds": { - "max": 2.6585870534181595, - "median": 2.6094753071665764, - "min": 2.570909459143877, + "max": 2.55265936255455, + "median": 2.5275897532701492, + "min": 2.5160773061215878, "values": [ - 2.6569117195904255, - 2.5945258922874928, - 2.570909459143877, - 2.6094753071665764, - 2.6585870534181595 + 2.5471372343599796, + 2.5160773061215878, + 2.5275897532701492, + 2.5250496082007885, + 2.55265936255455 ] }, "trace_bytes": { @@ -3117,15 +2425,15 @@ ] }, "training_seconds": { - "max": 0.7914622835814953, - "median": 0.7837709784507751, - "min": 0.7750968001782894, + "max": 0.8106105029582977, + "median": 0.782595831900835, + "min": 0.7656995020806789, "values": [ - 0.7750968001782894, - 0.7914622835814953, - 0.7841797918081284, - 0.7777450606226921, - 0.7837709784507751 + 0.7976063750684261, + 0.7686566635966301, + 0.782595831900835, + 0.8106105029582977, + 0.7656995020806789 ] } } @@ -3138,6 +2446,7 @@ "benchmark_source": "/home/yuxuan/experiments/codex-trace-impact-20261013/benchmarks/false-positive/RNN/trainset/word_language_model-RNN_TANH-3epo/model.py", "benchmark_source_sha256": "e4451a33e9b16953004bc639d736d14f42a2b21f2939ccabe4cff10d3c40b74c", "cpu_threads": 1, + "cuda_available": false, "data": { "adaptation": "Whitespace tokenization plus ; vocabulary and tensors are bounded to the chosen prefix, unlike the full benchmark corpus.", "kind": "wikitext2_training_prefix_with_prefix_only_vocabulary", @@ -3179,11 +2488,11 @@ ], "observation_files": [ { - "path": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/static_analyzer/func_level/nn_func_level.log", + "path": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/static_analyzer/func_level/nn_func_level.log", "sha256": "0ea56ca02089d5b19a1c0697dea00d107fccd18e17d485ff97bc04a5378dfcba" }, { - "path": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed/baseline/source/traincheck/static_analyzer/func_level/optim_func_level.log", + "path": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/static_analyzer/func_level/optim_func_level.log", "sha256": "300c47f9babd033ddf68487dde01a828746ff795b5f0111761b2f8fda08ad900" } ], @@ -3193,9 +2502,9 @@ "python": "3.11.4 (main, Jul 5 2023, 13:45:01) [GCC 11.2.0]", "scan_proxy_in_args": true, "seed": 42, - "setup_seconds": 2.6592667177319527, - "source_commit": "88b6df2401e187ca6bd81bf0a83bc7c5d9a5cd54", - "source_tree": "5415a3b0e5eb08d119dffa91631d76866e6e1d11", + "setup_seconds": 2.561651110649109, + "source_commit": "7a40649d2ce55c1a92380e26cd0f74b61c05824b", + "source_tree": "c84687d977b748a5080f034ace78c78b45cec172", "steps": 3, "tensor_dump_format": { "dump_tensor_full": false, @@ -3238,7 +2547,8 @@ ], "training_only_trace": true, "update": "Adapted language-model forward/NLL/backward/clipped manual SGD. Model classes copied exactly; the benchmark main.py training path is RNN-specific and is not executed. Transformer dimensions/vocabulary reduced.", - "variant": "baseline_subclass_fix", + "use_full_instr": false, + "variant": "final_correctness_without_metadata_fastpath", "workload_kind": "bounded_model_micro_workload", "wrapped_functions": 21032 } diff --git a/docs/measurements/metadata-classification-20261027.md b/docs/measurements/metadata-classification-20261027.md index a4f96cba..0cccc6ce 100644 --- a/docs/measurements/metadata-classification-20261027.md +++ b/docs/measurements/metadata-classification-20261027.md @@ -1,241 +1,232 @@ -# Transformer tracker correctness and metadata fast-path measurements +# Final matched metadata-classification measurements -The fixed subclass tracker completes the previously failing Transformer workload. -In five alternating pairs with **both API and model tracing active**, the metadata -classification candidate reduced median three-step CPU training time by **8.67% -for MNIST** and **18.90% for the small Transformer**. These are reductions from the -baseline's training-body time with both tracers enabled under ordinary collection -and its normal API exclusions (`use_full_instr=False`). They are not whole-process -speedups or comparisons against the earlier PR #153 measurements. +With the reviewed correctness changes on both sides, the metadata fast path +reduces median three-step CPU training time by **9.96% on MNIST** and **18.43% +on the small Transformer**. Both API and subclass model tracing are enabled +under ordinary collection (`use_full_instr=False`, retaining normal exclusions). +Every one of the ten matched pairs improves. Hashing is unchanged. -## Exact comparison and scope +These final numbers supersede the initial study for the current implementation. +The initial evidence remains in the parent output directory and in repository +checkpoint `bcf9bb82c3c11812336d1199dc699aa4ee6473e2`. Do not compare timings +across the studies to attribute additional improvements. + +## Exact sources and experiment | Variant | Commit | Tree | |---|---|---| -| Baseline: subclass assignment fix | `88b6df2401e187ca6bd81bf0a83bc7c5d9a5cd54` | `5415a3b0e5eb08d119dffa91631d76866e6e1d11` | -| Candidate: same fix plus callable metadata fast path | `3232887a8bdb6dbf6abcab233ead43a9697f0c18` | `cb74fdd3a8185deb0b3d31c0d20d2a7eac0b8097` | - -All 77 executed production Python files per variant match the respective exact -Git archive byte for byte. The only differing production file is -`traincheck/instrumentor/dumper.py`. Hash algorithms, hash backend, tensor dump -format, sampling, observer depth, and tracing coverage remain unchanged. - -The performance change recognizes exact built-in callable types before testing -them against framework tensor/module classes. It retains attribute discovery and -warning handling, and retains the generic path for arbitrary callable objects and -subclasses. This study does not change or optimize hashing. - -## Workload and timing contract - -- labmach, Intel Core i9-13900K, Python 3.11.4, PyTorch - `2.16.0.dev20261012+cpu`; one PyTorch/native thread, CPU only, deterministic - algorithms, batch 2, float32, three cold training steps. -- MNIST: exact benchmark `Net`, 1,199,882 parameters, first six cached MNIST - training images, NLL loss and Adadelta with learning rate 1. -- Transformer: exact benchmark model definitions with bounded dimensions - (embedding 32, two layers, two heads, feedforward 64, sequence length 16, - vocabulary 60 from the WikiText-2 prefix). Original gradient clipping and - **manual SGD parameter updates remain present**. -- Initialization seeds 42; training seeds 43 for Python, NumPy and PyTorch. -- Explicit subclass parameter tracking before optimizer construction; normal - depth-3 static observers; parameter hash format enabled, CPU backend; - `scan_proxy_in_args=True`; API descriptors disabled; sampling interval 1, - warmup 0; `use_full_instr=False`, retaining normal API exclusions. This does - not exercise `--use-full-instr`, default Proxy, or VarSampler mode. -- Ordinary timing includes `start_step`, the training body, and disabling - instrumentation on exit. It excludes imports/setup, loss extraction, RNG/state - verification and final background-writer drain. **RNG snapshots are outside - the timer and after `DISABLE_WRAPPER=True`**, so they add no API trace records. - Separate cProfile runs cover the training body, not setup or diagnostics. -- Every case uses a fresh process. Five pairs per workload alternate - baseline/candidate, candidate/baseline, baseline/candidate, candidate/baseline, - baseline/candidate. The shared host was not reserved or pinned to a CPU. -- Limits: 90 seconds, 4 GiB sampled main-process RSS, 256 MiB trace per case. - No case hit a limit. Native threads were restricted to one; other users' work - and shared environments were not changed. - -## Correctness smoke - -The fixed baseline passed plain, API-only, VAR-only and combined Transformer -three-step runs. Initial/final model state, every loss value, and Python/NumPy/ -PyTorch CPU RNG state before training and after each step matched exactly. - -| Mode | Single smoke time (ms) | API records | VAR records | Tracked parameters | +| Baseline: all current correctness/default changes, metadata fast path removed | `7a40649d2ce55c1a92380e26cd0f74b61c05824b` | `c84687d977b748a5080f034ace78c78b45cec172` | +| Candidate: all current changes including metadata fast path | `035f7f1976c5d84bd01f5d96bb095fb07c9cc059` | `89fdd92ebc6a57e5878d27cfb1b18dc69ee28a0b` | + +All **78 production Python files per variant** match the respective exact Git +archive byte for byte. The only production difference is +`traincheck/instrumentor/dumper.py`. Both sides include the parameter identity, +alias/ownership lifecycle, default-tracker and constructor-snapshot fixes. + +The optimization recognizes exact built-in callable types before repeatedly +checking them against framework tensor/module classes. Attribute enumeration, +getter evaluation, warning scope/restoration, supported metadata fields and the +generic fallback remain present. No hash algorithm, backend, dump format, +sampling or observation-depth change is part of the comparison. + +Conditions: labmach Intel Core i9-13900K; Python 3.11.4; PyTorch +`2.16.0.dev20261012+cpu`; one PyTorch/native thread; CPU only; float32; batch 2; +three **cold** steps. Initialization seeds are 42, training seeds 43 for Python, +NumPy and PyTorch. No affinity or exclusive host reservation was used. + +- MNIST uses the exact benchmark `Net` (1,199,882 parameters), the first six + cached MNIST training images, NLL loss and Adadelta with learning rate 1. +- Transformer uses exact benchmark model definitions with bounded dimensions: + embedding 32, two layers, two heads, feedforward 64, sequence length 16 and + vocabulary 60 from the WikiText-2 prefix. Gradient clipping and the original + manual SGD updates remain present. +- Model registration occurs before optimizer construction. Both-tracer runs use + explicit subclass tracking, depth-3 static observers, argument scanning, + hash-format tensor dumps on CPU, sampling interval 1, warmup 0, and disabled + API descriptors. This does not measure `--use-full-instr`, Proxy/VarSampler + modes, CUDA, MPS or distributed training. + +The timer includes `start_step`, the training body and disabling instrumentation +on exit. It excludes imports/setup, loss extraction, model/RNG verification and +final writer shutdown. RNG capture runs **after the timer with wrappers disabled** +and adds no API trace records. Every case uses a fresh process and has a 90-second, +4-GiB sampled main-process RSS, 256-MiB trace limit; no case hit a limit. + +## Ordinary measurements + +Five pairs per workload alternate baseline/candidate then candidate/baseline, +ending baseline/candidate. Profiles are separate from these measurements. +Percent reduction uses the baseline median **training-body time** as denominator, +not added overhead or whole-process time. + +| Workload | Baseline median (range), ms | Candidate median (range), ms | Reduction | Ratio of medians | |---|---:|---:|---:|---:| -| Plain | 15.418 | 0 | 0 | 0 | -| API | 877.959 | 1,338 | 0 | 0 | -| VAR | 88.366 | 0 | 64 | 29 | -| Combined | 940.872 | 1,340 | 64 | 29 | - -These smoke times establish execution, not a repeated ablation estimate. -VAR-only keeps Instrumentor's no-dump wrappers. All modes import TrainCheck's -global module-assignment hook; plain installs no API wrappers or parameter -proxies. Modes with VAR tracking also install observers and argument scanning, -so four-mode differences are not a physically exclusive cost partition. +| MNIST | 667.776 (653.230–675.525) | 601.247 (577.793–626.689) | **66.529 ms / 9.96%** | 1.111× | +| Transformer | 959.440 (945.047–1,022.942) | 782.596 (765.700–810.611) | **176.844 ms / 18.43%** | 1.226× | -## Ordinary paired measurements - -| Workload | Baseline median (range), ms | Candidate median (range), ms | Reduction from baseline median | Ratio of medians | -|---|---:|---:|---:|---:| -| MNIST | 868.830 (831.441–943.226) | 793.489 (722.298–849.878) | 75.341 ms / **8.67%** | 1.095× | -| Transformer | 966.440 (957.389–1,028.738) | 783.771 (775.097–791.462) | 182.669 ms / **18.90%** | 1.233× | - -All individual ordinary runs, in repetition order: - -| Workload | Baseline times (ms) | Candidate times (ms) | Paired reductions (ms) | +| Workload | Baseline runs, ms | Candidate runs, ms | Paired reductions, ms | |---|---|---|---| -| MNIST | 943.226, 868.830, 831.441, 903.219, 833.264 | 814.235, 722.298, 793.489, 792.977, 849.878 | 128.991, 146.531, 37.952, 110.242, **−16.615** | -| Transformer | 958.257, 957.389, 966.440, 971.936, 1,028.738 | 775.097, 791.462, 784.180, 777.745, 783.771 | 183.160, 165.927, 182.261, 194.191, 244.967 | +| MNIST | 674.327, 667.776, 653.230, 666.759, 675.525 | 626.689, 577.793, 587.515, 601.247, 626.016 | 47.637, 89.983, 65.715, 65.512, 49.509 | +| Transformer | 945.047, 967.448, 951.148, 959.440, 1,022.942 | 797.606, 768.657, 782.596, 810.611, 765.700 | 147.440, 198.792, 168.552, 148.829, 257.243 | -MNIST improves in four of five pairs; its paired reductions range from −1.99% -to 16.87%. Transformer improves in all five pairs (17.33%–23.81%). These are -bounded cold-step diagnostics on a shared host, not confidence intervals or -steady-state/GPU/distributed scaling claims. No run was discarded as an outlier. +Paired reductions range from 7.06%–13.48% for MNIST and 15.51%–25.15% for +Transformer. No outliers were discarded. Five short pairs on a shared machine +do not establish a confidence interval, steady-state scaling or GPU performance. -## Profiles: what changed and what remains +## Profiles and remaining work -Each variant has one separate cProfile run per workload. Total profiled time is -1.599957 → 1.322230 seconds for MNIST and 2.834436 → 2.173404 seconds for -Transformer. Percentages below use **that variant's profiled total**. Cumulative -rows overlap; do not add them or apply them to the ordinary timing denominator. +One separate profile per variant/workload has total time **1.372028 → 1.122652 s** +for MNIST and **2.719891 → 2.135597 s** for Transformer. Every percentage below +uses that variant's **profiled total**, not its ordinary median. Cumulative rows +overlap; they cannot be added into an exclusive cost partition. -| Profile component | MNIST baseline → candidate, ms (% of respective total) | Transformer baseline → candidate, ms (% of respective total) | +| Cumulative component | MNIST baseline → candidate, ms (% of profile total) | Transformer baseline → candidate, ms (% of profile total) | |---|---|---| -| All `convert_var_to_dict`, cumulative | 1,537.323 (96.09%) → 1,257.303 (95.09%) | 2,748.502 (96.97%) → 2,091.410 (96.23%) | -| Built-in `isinstance`, cumulative | 171.536 (10.72%) → 20.463 (1.55%) | 427.563 (15.08%) → 52.964 (2.44%) | -| Dumper `safe_getattr`, cumulative | 626.707 (39.17%) → 605.309 (45.78%) | 1,579.442 (55.72%) → 1,498.406 (68.94%) | -| Warning `simplefilter`, cumulative, nested in `safe_getattr` | 227.450 (14.22%) → 219.941 (16.63%) | 579.481 (20.44%) → 552.660 (25.43%) | -| API argument conversion, cumulative | 600.868 (37.55%) → 466.125 (35.25%) | 1,578.695 (55.70%) → 1,207.365 (55.55%) | -| API return conversion, cumulative | 291.809 (18.24%) → 225.436 (17.05%) | 987.921 (34.85%) → 752.378 (34.62%) | -| `ProxyParameter.dump_trace`, cumulative | 653.538 (40.85%) → 574.950 (43.48%) | 213.353 (7.53%) → 162.592 (7.48%) | -| `tensor_hash`, cumulative, nested in snapshots | 437.785 (27.36%) → 414.738 (31.37%) | 5.762 (0.20%) → 5.312 (0.24%) | -| Final `serialize`, cumulative | 4.350 (0.27%) → 4.386 (0.33%) | 12.093 (0.43%) → 11.245 (0.52%) | - -The classifier evidence is concrete: `isinstance` calls fall from 1,503,363 to -464,357 for MNIST and 3,622,491 to 1,154,245 for Transformer. Parameter -`__instancecheck__` calls fall from 203,335 to 2,817 and 495,303 to 8,937, -respectively (Transformer also has one other instance-check entry in both -variants). The number of attribute lookups/warning scopes is unchanged: -211,616 for MNIST and 516,009 for Transformer. Thus repeated per-attribute -lookup/warning setup remains an actionable instrumentation bottleneck; this -candidate does not remove it. Final JSON encoding is a small part of these -profiles. Hash call counts remain 112 and 64; hash timing variation is not -attributed to a hash optimization. - -## Activation, data integrity and resource accounting - -All **28 original cases** (four smoke, twenty ordinary, four profile) completed -successfully. The paired driver initially compared each workload against its -first baseline combined run. A supplementary check explicitly anchors all 24 -ordinary/profile cases to plain training: the existing Transformer plain smoke -and a new isolated MNIST plain run using the exact baseline source. Every initial -and final state digest/shape, loss, step ID, and per-step Python/NumPy/PyTorch RNG -snapshot matches. Timing fields alone are excluded from this behavioral comparison. -The supplemental run supplies a correctness reference, not another speedup estimate. -All saved raw trace SHA-256s, sizes, counts and API pre/post pair multiplicities -were independently reread and validated locally. -Every tracked parameter has at least two distinct data hashes across the three -steps. The five ordinary runs and separate profiles have matching coverage -signatures across variants: - -| Workload | API records / complete pairs | VAR records / parameters | Data+gradient hash fields | Raw API+VAR bytes | +| Metadata conversion, `convert_var_to_dict` | 1,307.722 (95.31%) → 1,061.525 (94.56%) | 2,641.869 (97.13%) → 2,056.602 (96.30%) | +| Built-in `isinstance` | 169.932 (12.39%) → 20.078 (1.79%) | 415.405 (15.27%) → 53.038 (2.48%) | +| Attribute lookup, dumper `safe_getattr` | 624.366 (45.51%) → 598.322 (53.30%) | 1,513.444 (55.64%) → 1,472.196 (68.94%) | +| Warning `simplefilter`, nested in attribute lookup | 228.109 (16.63%) → 219.396 (19.54%) | 553.408 (20.35%) → 540.450 (25.31%) | +| API argument conversion | 599.447 (43.69%) → 461.566 (41.11%) | 1,513.930 (55.66%) → 1,184.774 (55.48%) | +| API return conversion | 293.872 (21.42%) → 223.348 (19.90%) | 950.236 (34.94%) → 742.331 (34.76%) | +| `ProxyParameter.dump_trace` | 423.777 (30.89%) → 387.950 (34.56%) | 208.495 (7.67%) → 161.196 (7.55%) | +| `tensor_hash`, nested in snapshots | 221.088 (16.11%) → 230.689 (20.55%) | 5.592 (0.21%) → 5.207 (0.24%) | +| Final JSON `serialize` | 4.231 (0.31%) → 5.508 (0.49%) | 10.609 (0.39%) → 10.576 (0.50%) | + +`isinstance` calls fall from **1,503,819 to 464,813** for MNIST and from +**3,622,987 to 1,154,741** for Transformer. Parameter `__instancecheck__` calls +fall from 203,335 to 2,817 and 495,303 to 8,937 respectively; Transformer also +has one separate instance-check entry in each profile. + +Attribute lookups and warning scopes stay at **211,616 for MNIST** and **516,009 +for Transformer**. Their repeated setup remains a concrete next instrumentation +bottleneck, subject to preserving getter side effects and warning restoration. +Final JSON encoding is small in these profiles. Hashing was not optimized: +both variants use the existing CPU implementation, with unchanged call counts. +The small profile hash-time differences are not a hash-speedup claim. +Metadata conversion and JSON encoding still run synchronously in the producer; +the background threads handle buffered file writing. The small JSON row does +not imply that all preparation of a trace record is cheap or asynchronous. + +## Correctness, coverage and resources + +All **30 cases** passed: six candidate correctness cases (Transformer plain, +API, VAR and combined; MNIST plain and combined), twenty ordinary paired cases +and four profiles. The six correctness cases complete before measurement starts. +Every paired/profiled run is checked directly against the corresponding plain +candidate run: exact initial/final model-state digests and shapes, every loss +and step ID, and Python/NumPy/PyTorch RNG state before training and after each +step. Only elapsed seconds are excluded from loss-row comparison. + +All retained raw trace checksums, sizes, counts and API pre/post multiplicities +were reread locally. Every tracked parameter has changing data hashes across +the three steps. Counts and coverage signatures agree across ordinary/profiled +baseline and candidate runs: + +| Workload | API records / complete pairs | VAR records / parameters | Tensor data+gradient hash fields | Raw API+VAR bytes | |---|---:|---:|---:|---:| | MNIST | 520 / 260 | 64 / 8 | 112 | 419,429 | | Transformer | 1,340 / 670 | 64 / 29 | 64 | 1,088,456 | -Transformer snapshots have `grad=None` and capture changed parameter values at -the current observers' assignment/zero-grad observation points. They do **not** -establish pre/post snapshots for each manual SGD update, or gradient snapshot -coverage. The final state check independently verifies that all three training -steps and their updates matched. We did not replace manual SGD or add synthetic -observers to improve coverage. MNIST's Adadelta path has data and gradient hashes. +Profile calls to both `tensor_hash` and `hash_tensor_cpu` exactly match 112 and +64 recorded hash fields. MNIST has 48 gradient hashes in addition to its 64 data +hashes. Transformer snapshots have `grad=None` at the existing observation points: +they capture changing parameter values, but **do not establish gradient snapshots +or pre/post observations for every manual SGD update**. We did not replace manual +SGD or add synthetic observations to improve those counts. Final model/loss/RNG +checks separately establish behavior parity for the completed training steps. -Logical hashed input volume, counting repeated snapshots, is 67,193,392 bytes -for MNIST and 214,496 bytes for Transformer. This is not physical memory traffic -or a time partition. No cross-process raw trace equality is claimed: identities -and timestamps remain intact in the retained raw records. +Logical hashed input volume, counting repeated snapshots, is 67,193,392 bytes for +MNIST and 214,496 bytes for Transformer; this is not physical memory traffic or +a time share. Raw identities and timestamps remain intact. There is no claim of +normalized raw trace equality across independent processes. | Ordinary median | MNIST baseline → candidate | Transformer baseline → candidate | |---|---:|---:| -| Setup | 2.702 → 2.709 s | 2.603 → 2.609 s | -| Whole process | 7.018 → 7.019 s | 7.018 → 6.817 s | -| Post-metrics process exit | 2.078 → 2.131 s | 2.039 → 2.040 s | -| Sampled main-process peak RSS | 561.492 → 561.285 MiB | 515.441 → 515.648 MiB | - -Post-metrics exit includes writer drain, Python shutdown and up to one polling -interval; it is **not a pure I/O drain timer**. The small training-body savings -do not materially reduce whole-process MNIST time in this short workload. Full -per-run distributions for setup/process/exit/RSS/bytes are in [`data/metadata-classification-20261027.json`](data/metadata-classification-20261027.json). - -## Reproduction and artifacts - -Remote root: -`/home/yuxuan/experiments/codex-instrumenter-performance-20261027-fixed`. -Local root: -`/Users/matt/gitrepos/traincheck/outputs/instrumenter-performance-20261027`. -Use a fresh copy of the experiment directory for reruns because run directories -are deliberately created exclusively. `baseline/source.tar` and -`candidate/source.tar` retain the exact source archives; the only experimental -adapter change in `source/tools/fresh_validation/benchmark_workload.py` is the -typed registration callback before optimizer construction, identical in both -variants. Retained harnesses are `component_runner.py` and `run_experiment.py`; -`analyze_experiment.py` rereads and validates the evidence. +| Setup | 2.576 → 2.636 s | 2.551 → 2.528 s | +| Whole process | 6.816 → 6.915 s | 6.719 → 6.816 s | +| Post-metrics exit | 2.272 → 2.234 s | 1.900 → 2.156 s | +| Sampled main-process peak RSS | 556.816 → 561.523 MiB | 515.715 → 515.727 MiB | + +Post-metrics exit includes background-writer drain, Python shutdown and polling +delay. It is not a pure I/O timer. These short jobs do **not** show a whole-process +speedup; startup/shutdown and host variation outweigh the training-body savings. +Full per-run distributions are retained in [`data/metadata-classification-20261027.json`](data/metadata-classification-20261027.json). + +## Artifacts and reproduction + +Local directory: +`/Users/matt/gitrepos/traincheck/outputs/instrumenter-performance-20261027/final-matched`. +Remote directory: +`/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched`. + +The source archives are exact Git exports. Their isolated workload adapters add +the same typed model-registration callback before optimizer construction; no +production source is patched in place. Data and original benchmark definitions +come from the existing read-only caches recorded in each manifest. + +With a fresh copy of the retained experiment structure (run directories must not +already exist): ```bash -/home/yuxuan/experiments/codex-inference-runtime-20261025/venv/bin/python run_experiment.py --root "$EXPERIMENT_ROOT" --phase smoke +/home/yuxuan/experiments/codex-inference-runtime-20261025/venv/bin/python correctness_driver.py --root "$EXPERIMENT_ROOT/candidate" /home/yuxuan/experiments/codex-inference-runtime-20261025/venv/bin/python run_experiment.py --root "$EXPERIMENT_ROOT" --phase paired python analyze_experiment.py --root "$LOCAL_EVIDENCE_ROOT" ``` -The runner sets `PYTHONPATH` to each isolated source tree, restricts native -threads, disables bytecode writes and CUDA visibility, and records exact child -commands. It aborts on execution, coverage, loss, model-state or RNG divergence. -No installation or shared source/environment edit was needed. +The runner records every child command and sets the isolated source path, native +thread limits, CPU visibility and deterministic seed configuration. It aborts on +execution, coverage, model/loss/RNG or resource-limit failure. The analyzer verifies +source archives, executed harness hashes, raw records, plain references, unchanged +production-file scope and profile hash activation. No installations or shared +source/environment edits were used. + +The benchmark copy has no available Git-head identity. Exact source/definition +and data hashes are therefore retained in each manifest and [`data/metadata-classification-20261027.json`](data/metadata-classification-20261027.json): + +| Input/environment artifact | SHA-256 | +|---|---| +| MNIST benchmark source | `4478c5de92610a1e03d61c356801d51ffca749f99dac00f234d6b6ea2bcb2aa7` | +| Transformer benchmark source | `e4451a33e9b16953004bc639d736d14f42a2b21f2939ccabe4cff10d3c40b74c` | +| MNIST image IDX file (first six samples used) | `ba891046e6505d7aadcbbe25680a0738ad16aec93bde7f9b65e87a2fc25776db` | +| MNIST label IDX file | `65a50cbbf4e906d70832878ad85ccda5333a97f0f4c3dd2ef09a8a9eef7101c5` | +| WikiText-2 training file | `9e9fa1ad55b1c2c95b08e37dd8e653f638fac2c6de904b79e813611eefbc985f` | +| Used 98-token prefix | `9b0b504c73ee103a9128f2072536e99b2696039399bd57669477f030db10eb35` | +| Runtime package inventory | `f9b751a6b4b74230992a297eec30164e8d78fcd6a2d177726075f381bfae3c37` | +| CPU inventory | `ab09fc242f18a43f3e25a1bb6d8aaf6b2940f71efa4c2800c82f825f13bdaaeb` | | Artifact | SHA-256 | |---|---| -| Baseline source archive | `ac1a830f490bef1af6b1b15ba49d116ff28a8de2e57f51dd80afc714f4e8d909` | -| Candidate source archive | `b945a9d4c2b43def8481660bc3fc6ae45dbfbd34e3a5e8a0c848e5ca379276e6` | -| Raw evidence archive, 24,826,924 bytes | `1de069eb0dd5d8dd4ce60e758a4d3dae5716fa69bda1f3e30570b983f4e69773` | -| Executed runner | `d17315872c4e412d3a2be075fa733eebb9c7d4f8189dff82d53dc6bae92c0818` | -| Executed driver | `3a89d821238902362b110abbc9d66bb2b87c1b1cef09144bf7407a5400cd72f4` | - -The raw archive retains every log, metrics/manifest file, trace, profile and -environment inventory. Large/raw data remains in outputs and the isolated -remote directory, not in commits. - -Scoped mypy passes for all three new harness/analysis modules with -`--check-untyped-defs --disallow-untyped-defs --disallow-any-generics` and -`--follow-imports=skip --ignore-missing-imports`. The latter two options mean -dependency typing is not fully checked. Production test/check results belong -to the focused implementation PRs; this report records the actual workload -validation only. - -## Asynchronous work: what this measurement supports - -The subclass API/VAR file writers already run in background threads. Producers -still inspect live values and build immutable serialized records before enqueueing -so later application mutations cannot change earlier observations. Final JSON -encoding contributes 0.27–0.52% of the measured profiles; moving just that encoding -to a worker has little headroom here. The larger synchronous work is metadata -inspection and conversion, including per-attribute warning scopes. - -Moving arbitrary object inspection to a worker after a shallow copy would change -which values, getters and exceptions are observed. A future asynchronous tensor -snapshot path needs owned copies, producer-stream ordering for CUDA, bounded byte -and item queues, per-destination ordering, and explicit flush/error behavior. -It would need separate contract tests and an explicit decision about delayed -serialization errors. This PR implements the measured synchronous classification -fast path only; it does not add asynchronous snapshot conversion. - -The separate CLI default validation also reproduced a pre-existing startup -recursion under `--use-full-instr` in `torch._ops.__getattr__` on unchanged main -and the subclass candidate. Those broader-flag runs were stopped at 45 seconds; -they are not included in the successful ordinary-collection measurements above. - -Plain-reference validation is included in the measurement JSON under -`plain_reference_validation`. Its raw supplementary archive is retained as -`plain-parity.tar.gz` (12,709 bytes), SHA-256 -`6093c207f75f30331248220838613369a50d6fa3a4f6f534e0075ce7f8f4d4c1`, -with its checker alongside the other local evidence. The original 28-case -archive is unchanged. +| Baseline source archive | `efc665f1b6914964f3ac93ac76a3e00791b5cf07c310818c65e7711496260537` | +| Candidate source archive | `793087ee08bbfb10ed60712d7d22da2153809236c26eb113d78592695754fe4e` | +| Raw evidence archive, 25,722,591 bytes | `875be9098fab644afed9798cd6005f66af6567719e6be1765d84e9bd74725584` | +| Executed runner | `21984b309b688e95de62d37b59e6e5f7c9a1685ef5293e54dfaacd60e9305875` | +| Executed paired driver | `5a0d4c4a02ea397bce646a482c2657bb80a1cd28bb5fa6e7cd0127071c6fda67` | +| Executed correctness driver | `752968c1c2bd4b646f2c30dc83788597ff5567d41b3759bebc2b9c658795d6fe` | + +Configured Ruff/Black/isort checks pass for the four harness/analysis modules. +Scoped mypy passes with `--check-untyped-defs --disallow-untyped-defs +--disallow-any-generics --follow-imports=skip --ignore-missing-imports`; dependency +typing is not fully checked by that invocation. The exact composed candidate also passes pinned production hooks, configured +mypy on all 78 production files, and strict installed-Torch checking of eight +new contract/test modules. The complete suite passes 246/247 tests; the sole +`test_inject_eval_loop` failure also occurs on unchanged main. Separate real CLI +runs complete ordinary, selective and full-flag collection, with exact training +parity and two active passing invariant decisions each. Detailed correctness +evidence is retained in `outputs/instrumenter-final-validation-20261027` and +`outputs/instrumenter-final-cli-20261027`, and described in PRs #154, #155 and #157. +Raw traces, profiles and checked harnesses remain in the experiment directories; +only the reviewed report and compact measurement evidence are committed here. + + +## Asynchronous snapshot work + +File writes already use background threads on the subclass path. Moving only +final JSON encoding off the producer offers little headroom in this study. +The larger synchronous work inspects live objects and prepares their metadata. +Moving those getters to a later worker after a shallow copy can observe changed +values and change when errors occur. A future asynchronous tensor snapshot path +needs owned snapshots, producer-stream ordering, bounded bytes/items, ordering +per destination, and explicit worker-error and drain behavior. This change adds +no asynchronous snapshot conversion. Per-attribute warning setup is the next +measured implementation target, provided getter and warning semantics remain +intact. Selective-checking overhead needs its own timing study.