diff --git a/docs/measurements/data/metadata-classification-20261027.json b/docs/measurements/data/metadata-classification-20261027.json new file mode 100644 index 00000000..87996b13 --- /dev/null +++ b/docs/measurements/data/metadata-classification-20261027.json @@ -0,0 +1,2557 @@ +{ + "artifacts": { + "driver_sha256": "5a0d4c4a02ea397bce646a482c2657bb80a1cd28bb5fa6e7cd0127071c6fda67", + "evidence_archive_bytes": 25722591, + "evidence_archive_sha256": "875be9098fab644afed9798cd6005f66af6567719e6be1765d84e9bd74725584", + "remote": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched", + "runner_sha256": "21984b309b688e95de62d37b59e6e5f7c9a1685ef5293e54dfaacd60e9305875" + }, + "limitations": [ + "CPU only, one native thread, three cold steps and five pairs on a shared host", + "Subclass tracker only; unchanged CPU hashing, tensor dump format, tracing coverage and sample policy", + "Timing excludes setup, diagnostics and final writer drain; process time and post-metrics exit separately retained", + "Post-metrics exit includes Python shutdown and polling delay; it is not a pure writer timer", + "Full raw traces retained; no normalized cross-process trace equivalence claim" + ], + "production_changed_files": [ + "traincheck/instrumentor/dumper.py" + ], + "smoke": [ + { + "logical_hash_input_bytes": 0, + "mode": "plain", + "records": { + "API": 0, + "VAR": 0 + }, + "tensor_hash_fields": 0, + "trace_bytes": { + "API": 0, + "VAR": 0 + }, + "tracked_variables": [], + "training_seconds": 0.007946331053972244, + "workload": "transformer" + }, + { + "logical_hash_input_bytes": 0, + "mode": "api", + "records": { + "API": 1338, + "VAR": 0 + }, + "tensor_hash_fields": 0, + "trace_bytes": { + "API": 1014134, + "VAR": 0 + }, + "tracked_variables": [], + "training_seconds": 0.7302097044885159, + "workload": "transformer" + }, + { + "logical_hash_input_bytes": 214496, + "mode": "var", + "records": { + "API": 0, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 0, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "training_seconds": 0.07497242465615273, + "workload": "transformer" + }, + { + "logical_hash_input_bytes": 214496, + "mode": "combined", + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 1034378, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "training_seconds": 0.7909529991447926, + "workload": "transformer" + }, + { + "logical_hash_input_bytes": 0, + "mode": "plain", + "records": { + "API": 0, + "VAR": 0 + }, + "tensor_hash_fields": 0, + "trace_bytes": { + "API": 0, + "VAR": 0 + }, + "tracked_variables": [], + "training_seconds": 0.03079724684357643, + "workload": "mnist" + }, + { + "logical_hash_input_bytes": 67193392, + "mode": "combined", + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "trace_bytes": { + "API": 363634, + "VAR": 55795 + }, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ], + "training_seconds": 0.6652194261550903, + "workload": "mnist" + } + ], + "sources": { + "baseline": { + "adapter_sha256": "a5b5e597d4921a5b4baceeada541a6a1765f13c97ab3bd1c2ef79dea331f0934", + "production_files_verified": 78, + "production_hashes": { + "traincheck/__init__.py": "47fdcb090c4c8c5822769d642dc23a520e8d8ff3e4724583450723e05408164b", + "traincheck/checker.py": "44dc20fc66ca82bb6b71fa02033aae3b7d489ae7afbc1611c44e857f399716b2", + "traincheck/checker_online.py": "2824c9ccd2f684c86884c68f2214a1a4be1d69c6098ce30b817d03f1fe1077d6", + "traincheck/collect_trace.py": "2e0a0b9a4e05fef450d09788e1747db2d3f1c4aa86f2828b63229cbbb7445d00", + "traincheck/config/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/config/config.py": "4c940a4d92f5de5fe8d3885e570078e479d9842d9626651fd60483e78f0160a9", + "traincheck/developer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/developer/annotations.py": "0ff28d35e085b40e1e366499feec96c8eb1e13bf3100dcdd7ab304c7f2f6e963", + "traincheck/infer_engine.py": "16bf736b498a5f86583b2f1207d31f37479db72510b04121abcb1158d0141f73", + "traincheck/inference_sampling.py": "d261b3416aba3ead6fa1e9b15091511eb3525137d531e4f0147146e76308976f", + "traincheck/instrumentor/__init__.py": "228b4f6a11cde8118cde9aba2f98b60a839f4331e7fa2b06c37cee99e79fcd50", + "traincheck/instrumentor/caches.py": "320eb1ffa1ce5f5b97fe4675a16d47e9591265779fdc29b94b72cdf5df93f168", + "traincheck/instrumentor/constructor_snapshot.py": "efdbdf75433ce2c3fcb0985107f59fa8e0a8ced6808b824a990030cfbf0845db", + "traincheck/instrumentor/control.py": "3612a39b61e85fb77b40008b7fcd2f7569b246e8985ca64a109ea53733cd29ef", + "traincheck/instrumentor/dumper.py": "cd4a56d4eb95049dcab85bacc1c64d7e55bfe45087fe5096ed247e4d0d7e8373", + "traincheck/instrumentor/proxy_wrapper/__init__.py": "7c71aaf70b978ee04e236497af6f803e4436049bc0acdf18c3edb48bfcece63d", + "traincheck/instrumentor/proxy_wrapper/dumper.py": "6907ac0a198443dd9138a37dc9b5de1232d3d7fb26412f8bba3fb08c85cd222d", + "traincheck/instrumentor/proxy_wrapper/hash.py": "e917bec817df1981c682d8b8645307432c6badb220b9f7c424b4089d9bf97bd7", + "traincheck/instrumentor/proxy_wrapper/proxy.py": "fe241558f0591738391bc57d0666e3b11e04d8579eb220ce5de855abc309baf3", + "traincheck/instrumentor/proxy_wrapper/proxy_basics.py": "f012e97fa51cf77be1806807ef6130b06c9cec6fde8d047d57d45b6ec9fcc6e5", + "traincheck/instrumentor/proxy_wrapper/proxy_config.py": "afb2f8dd04ed10e628ff21339ec8ebef36357fd2a0cc084e99e99e37756cf16b", + "traincheck/instrumentor/proxy_wrapper/proxy_handler.py": "4bab74647d35c64c5e9c702295dc4cb010302bf620401da0fad8539692b95ac7", + "traincheck/instrumentor/proxy_wrapper/proxy_methods.py": "6ae18928aa12e89b33102368c497aff048e3d6bed330eca2118fac051b8e3a97", + "traincheck/instrumentor/proxy_wrapper/proxy_observer.py": "10e83570fbe1f72089e4a594a308f83e045f7d74bb9e49193012645ac906d97b", + "traincheck/instrumentor/proxy_wrapper/proxy_registry.py": "1f7b69ddfad7e8140b05a75fc5e4b4d8acf6e818e1cf030fde714f8e79f5d07b", + "traincheck/instrumentor/proxy_wrapper/subclass.py": "f370473a0e9676701d03771babc25c2c01619cac3bb3517ad6a22560283bccfb", + "traincheck/instrumentor/proxy_wrapper/torch_proxy.py": "d6abd7f2eac207757fd90cbf55e179ae506b25996bed906c5a094564ff6faf2f", + "traincheck/instrumentor/proxy_wrapper/utils.py": "5f058609e89a96a3545561ed82d83f7dd6a153bb0376608654c199691984dc14", + "traincheck/instrumentor/replace_functions.py": "ca4f10b52083a4a5c251db194b7608406cbe36c85372dfdb9df17644f3d20b62", + "traincheck/instrumentor/source_file.py": "dad92c793efb41161d86250e89aa6a9636749564e567df8b9af92a477eb4858d", + "traincheck/instrumentor/torchscript_compat.py": "1b42072c80b29c2432f3ef83d6849365ecc0ae8c015dd2051a5b192ba209f88a", + "traincheck/instrumentor/tracer.py": "e5a74e766eea7b26141a0b14ee26d2c6f852a2819869bb9151becfe921f262c1", + "traincheck/instrumentor/types.py": "258679aa8d0b40247d900ba2e26ab03b25cb4626b046ab6d724973226c7b5469", + "traincheck/instrumentor/variable.py": "b01219cdd74f011055e0bc2a645ed283dbc47e66a834d5555b814fa815a9eab8", + "traincheck/invariant/DistinctArgumentRelation.py": "6e01540583668e30718ba90c341fd540cd9df53f20525a6cf502fcceee8ed7bc", + "traincheck/invariant/__init__.py": "773e9f767566ffd92658dc0e94107ac840672e446a41ba437a49cee9cc52197a", + "traincheck/invariant/base_cls.py": "8c419a0ea675146900be1b78a47de2da07f27442019b632ff224be870d48d5e6", + "traincheck/invariant/consistency_relation.py": "bc5b14810ea8a5e0e5f3e6c2c6be24d137a444d8609216d00eafd155b4aa3917", + "traincheck/invariant/consistency_transient_vars.py": "a702cd1b6787e8be1904bef9119773c0e037a83017959c771020dbbbbe1549e2", + "traincheck/invariant/contain_relation.py": "b32f819db4d5a11a11318bf7e38c64148f4c9360137ac5f09475b480c34c16a5", + "traincheck/invariant/cover_relation.py": "0eb0e1c30ad5b34a15ce32c7b0d15e894f7a3b9b68bfcaae5b901f5dc6a092c6", + "traincheck/invariant/lead_relation.py": "978a4db32b69dc1a90a84123aa93e52a3caaba97761321dbb844666cb47d4360", + "traincheck/invariant/precondition.py": "c14c7580643780954eab5a4d955b0b5d6a6768a57cd8e5e48328ad0d8f2f8703", + "traincheck/invariant/relation_pool.py": "8ac568cf1924162a791728ed5db8c41868a550f9e0cd5190fc9a19827f55d3e6", + "traincheck/invariant/symbolic_value.py": "28627e2c2a47a146087a96510f07f7deb8463c46371e418afa3fdeff33c7e6cb", + "traincheck/invariant/var_periodic_change_relation.py": "8086df3b9cfe34229b8bd46b18dce22a4b746902e38d1ba4bd02bc6f6c2f2ef4", + "traincheck/onlinechecker/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/onlinechecker/streamhandler_filesystem.py": "b5a298bd06a9408e857ff6721578cb4ae10f227762ba3d66aec685e574cfa511", + "traincheck/onlinechecker/utils.py": "65ba5d8a84b748adf4267800e704df1f4662c0f483534289af20096bf9a78a00", + "traincheck/parallel.py": "a431d169022264109c56b7057ebcdc8a587b698a8670a1257b765189d9e5bf76", + "traincheck/progress.py": "aab7d040324e283c8474aa82bc7550a777aec316131a26e1dc1b6a277333d1ae", + "traincheck/reporting/__init__.py": "299180153357c4e6f48cf75191e915d70cfca5ae6e7b3ac4d45144066b93ddca", + "traincheck/reporting/checker_report.py": "0d59b1753fc348701b49a0eb74e4b1d849a47d8234005ccbbb26e09e3cd0c808", + "traincheck/runner/__init__.py": "287ffa9cab82f01da0e82faea77589dc7d348710c862496458d2d60c364394ba", + "traincheck/runner/runner.py": "89027b488ded8640fddb387dd80d98f059cf40be5f94863966b259acea1da7c7", + "traincheck/script/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/script/num_invs_each_rel.py": "202f13d536929c52defa84cdce00dec2251ea09aabf3ae8ae57067c32d0b844d", + "traincheck/static_analyzer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/config.py": "840460167bd6d26db429076cd4dc227e6c17c4e5fb157172041c48574f44edf7", + "traincheck/static_analyzer/func_level/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/graph_generator/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/graph_generator/analyzer.py": "035684eb7ea4edf8ee0dbf4c6b12b91b428cd6dd57be5fe658253c21c8c420b1", + "traincheck/static_analyzer/graph_generator/anutils.py": "9e62b7ce5fced45a033b76de3d408bdff2fd8561ffa28bc58de82e4105b51d39", + "traincheck/static_analyzer/graph_generator/call_graph_parser.py": "69e2555a1595078a2d2bca174de32cd7d4b0f023c0a20b68a01ca00d1d3b8647", + "traincheck/static_analyzer/graph_generator/main.py": "a568488c50383a67a6ab8feecaf96e29b288861c1dce17481a1b5032d20748a4", + "traincheck/static_analyzer/graph_generator/node.py": "b6a0cea44a28d33f28fee79b7f7073e69914e95565f7a8d62f9e5a5fd6b240cd", + "traincheck/toolkit/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/toolkit/analyze_invariant.py": "706cf8c06062b6df3f1f86691f588c91696cbef6c27698b2f0b290ad8f53ff4b", + "traincheck/toolkit/analyze_trace.py": "48dfc81ef85d96678b5fe466b86ba80d892ecfc497387773fa7ab5995f22e822", + "traincheck/toolkit/detect_anomaly_from_trace_diff.py": "2ffc0723dd3709ccab09cb9b58d76c7bb0558dda09b66c47184e335b815cedd3", + "traincheck/trace/__init__.py": "ec84edd0ff32906c20956166a51b85b3847e22e48432c6a31698080cafbcbc2b", + "traincheck/trace/trace.py": "c4f6fae85de0a9d95591ab88d52ad8acf24f052f42978ec381a53042e1e49e67", + "traincheck/trace/trace_dict.py": "3aec874ef0ce2c34f6332dface604d5f79965976fd3d1c2b692f746f2af4e703", + "traincheck/trace/trace_pandas.py": "953d9cac029f83e015eb22e3bc9c8abc7b63919a0eef18b9794517392117ec24", + "traincheck/trace/trace_polars.py": "64136efcd60a851b0391e708108570b259c2288db6242564b636f69abd1b7b0e", + "traincheck/trace/types.py": "96c1b137f3d72fb05dcf806a4e7ea21fe453ae9cbc3e08352c55e95973fe96e0", + "traincheck/trace/utils.py": "6356156e4c5bbb6c111140302660f36ad1c126e5bc977bd55c81f097022102a3", + "traincheck/utils.py": "71034730f74a15978d37dc9fc9b748133389a69e0ea7f9c9109dd1bc573fd272" + }, + "source_archive_sha256": "efc665f1b6914964f3ac93ac76a3e00791b5cf07c310818c65e7711496260537", + "source_commit": "7a40649d2ce55c1a92380e26cd0f74b61c05824b", + "source_tree": "c84687d977b748a5080f034ace78c78b45cec172" + }, + "candidate": { + "adapter_sha256": "a5b5e597d4921a5b4baceeada541a6a1765f13c97ab3bd1c2ef79dea331f0934", + "production_files_verified": 78, + "production_hashes": { + "traincheck/__init__.py": "47fdcb090c4c8c5822769d642dc23a520e8d8ff3e4724583450723e05408164b", + "traincheck/checker.py": "44dc20fc66ca82bb6b71fa02033aae3b7d489ae7afbc1611c44e857f399716b2", + "traincheck/checker_online.py": "2824c9ccd2f684c86884c68f2214a1a4be1d69c6098ce30b817d03f1fe1077d6", + "traincheck/collect_trace.py": "2e0a0b9a4e05fef450d09788e1747db2d3f1c4aa86f2828b63229cbbb7445d00", + "traincheck/config/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/config/config.py": "4c940a4d92f5de5fe8d3885e570078e479d9842d9626651fd60483e78f0160a9", + "traincheck/developer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/developer/annotations.py": "0ff28d35e085b40e1e366499feec96c8eb1e13bf3100dcdd7ab304c7f2f6e963", + "traincheck/infer_engine.py": "16bf736b498a5f86583b2f1207d31f37479db72510b04121abcb1158d0141f73", + "traincheck/inference_sampling.py": "d261b3416aba3ead6fa1e9b15091511eb3525137d531e4f0147146e76308976f", + "traincheck/instrumentor/__init__.py": "228b4f6a11cde8118cde9aba2f98b60a839f4331e7fa2b06c37cee99e79fcd50", + "traincheck/instrumentor/caches.py": "320eb1ffa1ce5f5b97fe4675a16d47e9591265779fdc29b94b72cdf5df93f168", + "traincheck/instrumentor/constructor_snapshot.py": "efdbdf75433ce2c3fcb0985107f59fa8e0a8ced6808b824a990030cfbf0845db", + "traincheck/instrumentor/control.py": "3612a39b61e85fb77b40008b7fcd2f7569b246e8985ca64a109ea53733cd29ef", + "traincheck/instrumentor/dumper.py": "faf87dcfb0aa08dd97bb544aa80120925ec1eed05d03947a1de743fdf5da53e2", + "traincheck/instrumentor/proxy_wrapper/__init__.py": "7c71aaf70b978ee04e236497af6f803e4436049bc0acdf18c3edb48bfcece63d", + "traincheck/instrumentor/proxy_wrapper/dumper.py": "6907ac0a198443dd9138a37dc9b5de1232d3d7fb26412f8bba3fb08c85cd222d", + "traincheck/instrumentor/proxy_wrapper/hash.py": "e917bec817df1981c682d8b8645307432c6badb220b9f7c424b4089d9bf97bd7", + "traincheck/instrumentor/proxy_wrapper/proxy.py": "fe241558f0591738391bc57d0666e3b11e04d8579eb220ce5de855abc309baf3", + "traincheck/instrumentor/proxy_wrapper/proxy_basics.py": "f012e97fa51cf77be1806807ef6130b06c9cec6fde8d047d57d45b6ec9fcc6e5", + "traincheck/instrumentor/proxy_wrapper/proxy_config.py": "afb2f8dd04ed10e628ff21339ec8ebef36357fd2a0cc084e99e99e37756cf16b", + "traincheck/instrumentor/proxy_wrapper/proxy_handler.py": "4bab74647d35c64c5e9c702295dc4cb010302bf620401da0fad8539692b95ac7", + "traincheck/instrumentor/proxy_wrapper/proxy_methods.py": "6ae18928aa12e89b33102368c497aff048e3d6bed330eca2118fac051b8e3a97", + "traincheck/instrumentor/proxy_wrapper/proxy_observer.py": "10e83570fbe1f72089e4a594a308f83e045f7d74bb9e49193012645ac906d97b", + "traincheck/instrumentor/proxy_wrapper/proxy_registry.py": "1f7b69ddfad7e8140b05a75fc5e4b4d8acf6e818e1cf030fde714f8e79f5d07b", + "traincheck/instrumentor/proxy_wrapper/subclass.py": "f370473a0e9676701d03771babc25c2c01619cac3bb3517ad6a22560283bccfb", + "traincheck/instrumentor/proxy_wrapper/torch_proxy.py": "d6abd7f2eac207757fd90cbf55e179ae506b25996bed906c5a094564ff6faf2f", + "traincheck/instrumentor/proxy_wrapper/utils.py": "5f058609e89a96a3545561ed82d83f7dd6a153bb0376608654c199691984dc14", + "traincheck/instrumentor/replace_functions.py": "ca4f10b52083a4a5c251db194b7608406cbe36c85372dfdb9df17644f3d20b62", + "traincheck/instrumentor/source_file.py": "dad92c793efb41161d86250e89aa6a9636749564e567df8b9af92a477eb4858d", + "traincheck/instrumentor/torchscript_compat.py": "1b42072c80b29c2432f3ef83d6849365ecc0ae8c015dd2051a5b192ba209f88a", + "traincheck/instrumentor/tracer.py": "e5a74e766eea7b26141a0b14ee26d2c6f852a2819869bb9151becfe921f262c1", + "traincheck/instrumentor/types.py": "258679aa8d0b40247d900ba2e26ab03b25cb4626b046ab6d724973226c7b5469", + "traincheck/instrumentor/variable.py": "b01219cdd74f011055e0bc2a645ed283dbc47e66a834d5555b814fa815a9eab8", + "traincheck/invariant/DistinctArgumentRelation.py": "6e01540583668e30718ba90c341fd540cd9df53f20525a6cf502fcceee8ed7bc", + "traincheck/invariant/__init__.py": "773e9f767566ffd92658dc0e94107ac840672e446a41ba437a49cee9cc52197a", + "traincheck/invariant/base_cls.py": "8c419a0ea675146900be1b78a47de2da07f27442019b632ff224be870d48d5e6", + "traincheck/invariant/consistency_relation.py": "bc5b14810ea8a5e0e5f3e6c2c6be24d137a444d8609216d00eafd155b4aa3917", + "traincheck/invariant/consistency_transient_vars.py": "a702cd1b6787e8be1904bef9119773c0e037a83017959c771020dbbbbe1549e2", + "traincheck/invariant/contain_relation.py": "b32f819db4d5a11a11318bf7e38c64148f4c9360137ac5f09475b480c34c16a5", + "traincheck/invariant/cover_relation.py": "0eb0e1c30ad5b34a15ce32c7b0d15e894f7a3b9b68bfcaae5b901f5dc6a092c6", + "traincheck/invariant/lead_relation.py": "978a4db32b69dc1a90a84123aa93e52a3caaba97761321dbb844666cb47d4360", + "traincheck/invariant/precondition.py": "c14c7580643780954eab5a4d955b0b5d6a6768a57cd8e5e48328ad0d8f2f8703", + "traincheck/invariant/relation_pool.py": "8ac568cf1924162a791728ed5db8c41868a550f9e0cd5190fc9a19827f55d3e6", + "traincheck/invariant/symbolic_value.py": "28627e2c2a47a146087a96510f07f7deb8463c46371e418afa3fdeff33c7e6cb", + "traincheck/invariant/var_periodic_change_relation.py": "8086df3b9cfe34229b8bd46b18dce22a4b746902e38d1ba4bd02bc6f6c2f2ef4", + "traincheck/onlinechecker/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/onlinechecker/streamhandler_filesystem.py": "b5a298bd06a9408e857ff6721578cb4ae10f227762ba3d66aec685e574cfa511", + "traincheck/onlinechecker/utils.py": "65ba5d8a84b748adf4267800e704df1f4662c0f483534289af20096bf9a78a00", + "traincheck/parallel.py": "a431d169022264109c56b7057ebcdc8a587b698a8670a1257b765189d9e5bf76", + "traincheck/progress.py": "aab7d040324e283c8474aa82bc7550a777aec316131a26e1dc1b6a277333d1ae", + "traincheck/reporting/__init__.py": "299180153357c4e6f48cf75191e915d70cfca5ae6e7b3ac4d45144066b93ddca", + "traincheck/reporting/checker_report.py": "0d59b1753fc348701b49a0eb74e4b1d849a47d8234005ccbbb26e09e3cd0c808", + "traincheck/runner/__init__.py": "287ffa9cab82f01da0e82faea77589dc7d348710c862496458d2d60c364394ba", + "traincheck/runner/runner.py": "89027b488ded8640fddb387dd80d98f059cf40be5f94863966b259acea1da7c7", + "traincheck/script/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/script/num_invs_each_rel.py": "202f13d536929c52defa84cdce00dec2251ea09aabf3ae8ae57067c32d0b844d", + "traincheck/static_analyzer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/config.py": "840460167bd6d26db429076cd4dc227e6c17c4e5fb157172041c48574f44edf7", + "traincheck/static_analyzer/func_level/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/graph_generator/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/graph_generator/analyzer.py": "035684eb7ea4edf8ee0dbf4c6b12b91b428cd6dd57be5fe658253c21c8c420b1", + "traincheck/static_analyzer/graph_generator/anutils.py": "9e62b7ce5fced45a033b76de3d408bdff2fd8561ffa28bc58de82e4105b51d39", + "traincheck/static_analyzer/graph_generator/call_graph_parser.py": "69e2555a1595078a2d2bca174de32cd7d4b0f023c0a20b68a01ca00d1d3b8647", + "traincheck/static_analyzer/graph_generator/main.py": "a568488c50383a67a6ab8feecaf96e29b288861c1dce17481a1b5032d20748a4", + "traincheck/static_analyzer/graph_generator/node.py": "b6a0cea44a28d33f28fee79b7f7073e69914e95565f7a8d62f9e5a5fd6b240cd", + "traincheck/toolkit/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/toolkit/analyze_invariant.py": "706cf8c06062b6df3f1f86691f588c91696cbef6c27698b2f0b290ad8f53ff4b", + "traincheck/toolkit/analyze_trace.py": "48dfc81ef85d96678b5fe466b86ba80d892ecfc497387773fa7ab5995f22e822", + "traincheck/toolkit/detect_anomaly_from_trace_diff.py": "2ffc0723dd3709ccab09cb9b58d76c7bb0558dda09b66c47184e335b815cedd3", + "traincheck/trace/__init__.py": "ec84edd0ff32906c20956166a51b85b3847e22e48432c6a31698080cafbcbc2b", + "traincheck/trace/trace.py": "c4f6fae85de0a9d95591ab88d52ad8acf24f052f42978ec381a53042e1e49e67", + "traincheck/trace/trace_dict.py": "3aec874ef0ce2c34f6332dface604d5f79965976fd3d1c2b692f746f2af4e703", + "traincheck/trace/trace_pandas.py": "953d9cac029f83e015eb22e3bc9c8abc7b63919a0eef18b9794517392117ec24", + "traincheck/trace/trace_polars.py": "64136efcd60a851b0391e708108570b259c2288db6242564b636f69abd1b7b0e", + "traincheck/trace/types.py": "96c1b137f3d72fb05dcf806a4e7ea21fe453ae9cbc3e08352c55e95973fe96e0", + "traincheck/trace/utils.py": "6356156e4c5bbb6c111140302660f36ad1c126e5bc977bd55c81f097022102a3", + "traincheck/utils.py": "71034730f74a15978d37dc9fc9b748133389a69e0ea7f9c9109dd1bc573fd272" + }, + "source_archive_sha256": "793087ee08bbfb10ed60712d7d22da2153809236c26eb113d78592695754fe4e", + "source_commit": "035f7f1976c5d84bd01f5d96bb095fb07c9cc059", + "source_tree": "89fdd92ebc6a57e5878d27cfb1b18dc69ee28a0b" + } + }, + "validation": { + "cases": [ + { + "label": "mnist-combined-r1", + "logical_hash_input_bytes": 67193392, + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "trace_bytes": { + "API": 363634, + "VAR": 55795 + }, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ], + "variant": "baseline", + "workload": "mnist" + }, + { + "label": "mnist-combined-r1", + "logical_hash_input_bytes": 67193392, + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "trace_bytes": { + "API": 363634, + "VAR": 55795 + }, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ], + "variant": "candidate", + "workload": "mnist" + }, + { + "label": "mnist-combined-r2", + "logical_hash_input_bytes": 67193392, + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "trace_bytes": { + "API": 363634, + "VAR": 55795 + }, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ], + "variant": "candidate", + "workload": "mnist" + }, + { + "label": "mnist-combined-r2", + "logical_hash_input_bytes": 67193392, + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "trace_bytes": { + "API": 363634, + "VAR": 55795 + }, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ], + "variant": "baseline", + "workload": "mnist" + }, + { + "label": "mnist-combined-r3", + "logical_hash_input_bytes": 67193392, + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "trace_bytes": { + "API": 363634, + "VAR": 55795 + }, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ], + "variant": "baseline", + "workload": "mnist" + }, + { + "label": "mnist-combined-r3", + "logical_hash_input_bytes": 67193392, + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "trace_bytes": { + "API": 363634, + "VAR": 55795 + }, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ], + "variant": "candidate", + "workload": "mnist" + }, + { + "label": "mnist-combined-r4", + "logical_hash_input_bytes": 67193392, + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "trace_bytes": { + "API": 363634, + "VAR": 55795 + }, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ], + "variant": "candidate", + "workload": "mnist" + }, + { + "label": "mnist-combined-r4", + "logical_hash_input_bytes": 67193392, + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "trace_bytes": { + "API": 363634, + "VAR": 55795 + }, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ], + "variant": "baseline", + "workload": "mnist" + }, + { + "label": "mnist-combined-r5", + "logical_hash_input_bytes": 67193392, + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "trace_bytes": { + "API": 363634, + "VAR": 55795 + }, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ], + "variant": "baseline", + "workload": "mnist" + }, + { + "label": "mnist-combined-r5", + "logical_hash_input_bytes": 67193392, + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "trace_bytes": { + "API": 363634, + "VAR": 55795 + }, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ], + "variant": "candidate", + "workload": "mnist" + }, + { + "label": "mnist-combined-r1-profile", + "logical_hash_input_bytes": 67193392, + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "trace_bytes": { + "API": 363634, + "VAR": 55795 + }, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ], + "variant": "baseline", + "workload": "mnist" + }, + { + "label": "mnist-combined-r1-profile", + "logical_hash_input_bytes": 67193392, + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "trace_bytes": { + "API": 363634, + "VAR": 55795 + }, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ], + "variant": "candidate", + "workload": "mnist" + }, + { + "label": "transformer-combined-r1", + "logical_hash_input_bytes": 214496, + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 1034378, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "variant": "baseline", + "workload": "transformer" + }, + { + "label": "transformer-combined-r1", + "logical_hash_input_bytes": 214496, + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 1034378, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "variant": "candidate", + "workload": "transformer" + }, + { + "label": "transformer-combined-r2", + "logical_hash_input_bytes": 214496, + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 1034378, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "variant": "candidate", + "workload": "transformer" + }, + { + "label": "transformer-combined-r2", + "logical_hash_input_bytes": 214496, + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 1034378, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "variant": "baseline", + "workload": "transformer" + }, + { + "label": "transformer-combined-r3", + "logical_hash_input_bytes": 214496, + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 1034378, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "variant": "baseline", + "workload": "transformer" + }, + { + "label": "transformer-combined-r3", + "logical_hash_input_bytes": 214496, + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 1034378, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "variant": "candidate", + "workload": "transformer" + }, + { + "label": "transformer-combined-r4", + "logical_hash_input_bytes": 214496, + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 1034378, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "variant": "candidate", + "workload": "transformer" + }, + { + "label": "transformer-combined-r4", + "logical_hash_input_bytes": 214496, + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 1034378, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "variant": "baseline", + "workload": "transformer" + }, + { + "label": "transformer-combined-r5", + "logical_hash_input_bytes": 214496, + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 1034378, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "variant": "baseline", + "workload": "transformer" + }, + { + "label": "transformer-combined-r5", + "logical_hash_input_bytes": 214496, + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 1034378, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "variant": "candidate", + "workload": "transformer" + }, + { + "label": "transformer-combined-r1-profile", + "logical_hash_input_bytes": 214496, + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 1034378, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "variant": "baseline", + "workload": "transformer" + }, + { + "label": "transformer-combined-r1-profile", + "logical_hash_input_bytes": 214496, + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "trace_bytes": { + "API": 1034378, + "VAR": 54078 + }, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ], + "variant": "candidate", + "workload": "transformer" + } + ], + "exact_model_loss_rng_parity": true, + "reference": "Each workload's candidate plain correctness case; every paired and profiled run checked against it", + "source_and_raw_trace_hashes_verified": true + }, + "workloads": { + "mnist": { + "coverage": { + "logical_hash_input_bytes": 67193392, + "records": { + "API": 520, + "VAR": 64 + }, + "tensor_hash_fields": 112, + "tracked_variables": [ + "model.conv1.bias", + "model.conv1.weight", + "model.conv2.bias", + "model.conv2.weight", + "model.fc1.bias", + "model.fc1.weight", + "model.fc2.bias", + "model.fc2.weight" + ] + }, + "median_reduction_percent_of_baseline_training": 9.96277605477705, + "median_reduction_seconds": 0.0665290355682373, + "paired_reduction_percent": [ + 7.064396033143514, + 13.47503614924583, + 10.059949814656402, + 9.825393357719033, + 7.328966441316239 + ], + "paired_reductions_seconds": [ + 0.04763709753751755, + 0.08998306840658188, + 0.06571457535028458, + 0.06551166623830795, + 0.04950897395610809 + ], + "profiles": { + "baseline": { + "interpretation": "Separate cProfile run; cumulative rows overlap and are not an exclusive breakdown of ordinary timing.", + "rows": [ + { + "calls": 1503819, + "cumulative_seconds": 0.16993218200000001, + "file": "~", + "function": "", + "line": 0, + "percent_of_profile_total": 12.385472890531892, + "self_seconds": 0.09892717200000001 + }, + { + "calls": 211616, + "cumulative_seconds": 0.089078695, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "__enter__", + "line": 466, + "percent_of_profile_total": 6.492482760248784, + "self_seconds": 0.08047354400000001 + }, + { + "calls": 112, + "cumulative_seconds": 0.221655714, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/dumper.py", + "function": "dump_tensor", + "line": 302, + "percent_of_profile_total": 16.155332112304016, + "self_seconds": 0.0005632370000000001 + }, + { + "calls": 112, + "cumulative_seconds": 0.22108786000000002, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/proxy_wrapper/hash.py", + "function": "tensor_hash", + "line": 95, + "percent_of_profile_total": 16.11394418778022, + "self_seconds": 0.010103038 + }, + { + "calls": 468, + "cumulative_seconds": 1.3077217060000002, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/dumper.py", + "function": "convert_var_to_dict", + "line": 343, + "percent_of_profile_total": 95.31303339601158, + "self_seconds": 0.223835801 + }, + { + "calls": 260, + "cumulative_seconds": 0.599447363, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/tracer.py", + "function": "to_dict_args_kwargs", + "line": 166, + "percent_of_profile_total": 43.69060042868943, + "self_seconds": 0.000508692 + }, + { + "calls": 211616, + "cumulative_seconds": 0.6243659410000001, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/dumper.py", + "function": "safe_getattr", + "line": 328, + "percent_of_profile_total": 45.50678597198814, + "self_seconds": 0.18085563400000002 + }, + { + "calls": 112, + "cumulative_seconds": 0.184003886, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/proxy_wrapper/hash.py", + "function": "hash_tensor_cpu", + "line": 47, + "percent_of_profile_total": 13.411086204998654, + "self_seconds": 0.158479586 + }, + { + "calls": 64, + "cumulative_seconds": 0.42377681500000003, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/proxy_wrapper/subclass.py", + "function": "dump_trace", + "line": 218, + "percent_of_profile_total": 30.88688788694803, + "self_seconds": 0.000185841 + }, + { + "calls": 1780, + "cumulative_seconds": 0.00044063900000000005, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/utils.py", + "function": "safe_getattr", + "line": 21, + "percent_of_profile_total": 0.03211588484758633, + "self_seconds": 0.000238251 + }, + { + "calls": 260, + "cumulative_seconds": 0.293872481, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/tracer.py", + "function": "to_dict_return_value", + "line": 210, + "percent_of_profile_total": 21.418836643307788, + "self_seconds": 0.000125961 + }, + { + "calls": 520, + "cumulative_seconds": 0.006746617000000001, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/dumper.py", + "function": "dump_trace_API", + "line": 241, + "percent_of_profile_total": 0.4917258224595833, + "self_seconds": 0.00042564800000000004 + }, + { + "calls": 64, + "cumulative_seconds": 0.0010291760000000001, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/dumper.py", + "function": "dump_trace_VAR", + "line": 250, + "percent_of_profile_total": 0.07501128566445436, + "self_seconds": 5.3818000000000005e-05 + }, + { + "calls": 584, + "cumulative_seconds": 0.004231365, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/dumper.py", + "function": "serialize", + "line": 85, + "percent_of_profile_total": 0.3084021865702017, + "self_seconds": 0.00040759300000000004 + }, + { + "calls": 203335, + "cumulative_seconds": 0.084410277, + "file": "/home/yuxuan/experiments/codex-inference-runtime-20261025/venv/lib/python3.11/site-packages/torch/nn/parameter.py", + "function": "__instancecheck__", + "line": 21, + "percent_of_profile_total": 6.152226053719406, + "self_seconds": 0.059368754 + }, + { + "calls": 211616, + "cumulative_seconds": 0.067194185, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "__exit__", + "line": 487, + "percent_of_profile_total": 4.897434652600911, + "self_seconds": 0.059458518 + }, + { + "calls": 211616, + "cumulative_seconds": 0.158761849, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "_add_filter", + "line": 181, + "percent_of_profile_total": 11.571325417572863, + "self_seconds": 0.089265255 + }, + { + "calls": 211616, + "cumulative_seconds": 0.033080641, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "__init__", + "line": 440, + "percent_of_profile_total": 2.4110758626457103, + "self_seconds": 0.033080641 + }, + { + "calls": 211616, + "cumulative_seconds": 0.22810855700000002, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "simplefilter", + "line": 165, + "percent_of_profile_total": 16.62564627588816, + "self_seconds": 0.06152440000000001 + } + ], + "sha256": "5e59942c772a2e5dc341de52946b26bfe79a96e2df96cada624a7f902b7496c1", + "total_seconds": 1.3720282100000003 + }, + "candidate": { + "interpretation": "Separate cProfile run; cumulative rows overlap and are not an exclusive breakdown of ordinary timing.", + "rows": [ + { + "calls": 464813, + "cumulative_seconds": 0.020077905, + "file": "~", + "function": "", + "line": 0, + "percent_of_profile_total": 1.7884348567809505, + "self_seconds": 0.018422498000000002 + }, + { + "calls": 211616, + "cumulative_seconds": 0.082044563, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "__enter__", + "line": 466, + "percent_of_profile_total": 7.3081009337657825, + "self_seconds": 0.074606007 + }, + { + "calls": 112, + "cumulative_seconds": 0.195671804, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/proxy_wrapper/hash.py", + "function": "hash_tensor_cpu", + "line": 47, + "percent_of_profile_total": 17.429421782940512, + "self_seconds": 0.172274382 + }, + { + "calls": 112, + "cumulative_seconds": 0.231207348, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/dumper.py", + "function": "dump_tensor", + "line": 318, + "percent_of_profile_total": 20.594742345233897, + "self_seconds": 0.0005136580000000001 + }, + { + "calls": 260, + "cumulative_seconds": 0.46156553600000005, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/tracer.py", + "function": "to_dict_args_kwargs", + "line": 166, + "percent_of_profile_total": 41.11384595510252, + "self_seconds": 0.000501468 + }, + { + "calls": 112, + "cumulative_seconds": 0.23068943700000002, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/proxy_wrapper/hash.py", + "function": "tensor_hash", + "line": 95, + "percent_of_profile_total": 20.548609539788796, + "self_seconds": 0.009633974 + }, + { + "calls": 468, + "cumulative_seconds": 1.06152485, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/dumper.py", + "function": "convert_var_to_dict", + "line": 362, + "percent_of_profile_total": 94.55508645345071, + "self_seconds": 0.146122746 + }, + { + "calls": 211616, + "cumulative_seconds": 0.5983217670000001, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/dumper.py", + "function": "safe_getattr", + "line": 344, + "percent_of_profile_total": 53.29537636887765, + "self_seconds": 0.17498206900000002 + }, + { + "calls": 64, + "cumulative_seconds": 0.38794965600000003, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/proxy_wrapper/subclass.py", + "function": "dump_trace", + "line": 218, + "percent_of_profile_total": 34.55652805741332, + "self_seconds": 0.000167806 + }, + { + "calls": 1780, + "cumulative_seconds": 0.00044975900000000004, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/utils.py", + "function": "safe_getattr", + "line": 21, + "percent_of_profile_total": 0.04006218142535009, + "self_seconds": 0.00022905 + }, + { + "calls": 584, + "cumulative_seconds": 0.00550764, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/dumper.py", + "function": "serialize", + "line": 101, + "percent_of_profile_total": 0.49059179005982123, + "self_seconds": 0.0016963430000000001 + }, + { + "calls": 260, + "cumulative_seconds": 0.22334777700000003, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/tracer.py", + "function": "to_dict_return_value", + "line": 210, + "percent_of_profile_total": 19.894652832122613, + "self_seconds": 0.00012765900000000002 + }, + { + "calls": 64, + "cumulative_seconds": 0.0022816850000000003, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/dumper.py", + "function": "dump_trace_VAR", + "line": 266, + "percent_of_profile_total": 0.20324057645427865, + "self_seconds": 5.284e-05 + }, + { + "calls": 520, + "cumulative_seconds": 0.006337504000000001, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/dumper.py", + "function": "dump_trace_API", + "line": 257, + "percent_of_profile_total": 0.5645117385797324, + "self_seconds": 0.000423195 + }, + { + "calls": 2817, + "cumulative_seconds": 0.0019264170000000001, + "file": "/home/yuxuan/experiments/codex-inference-runtime-20261025/venv/lib/python3.11/site-packages/torch/nn/parameter.py", + "function": "__instancecheck__", + "line": 21, + "percent_of_profile_total": 0.1715951595296117, + "self_seconds": 0.001401609 + }, + { + "calls": 211616, + "cumulative_seconds": 0.063982831, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "__exit__", + "line": 487, + "percent_of_profile_total": 5.699256231958701, + "self_seconds": 0.057104195 + }, + { + "calls": 211616, + "cumulative_seconds": 0.15220593500000001, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "_add_filter", + "line": 181, + "percent_of_profile_total": 13.557709310953294, + "self_seconds": 0.085092887 + }, + { + "calls": 211616, + "cumulative_seconds": 0.031969863, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "__init__", + "line": 440, + "percent_of_profile_total": 2.847708331905725, + "self_seconds": 0.031969863 + }, + { + "calls": 211616, + "cumulative_seconds": 0.219395593, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "simplefilter", + "line": 165, + "percent_of_profile_total": 19.54261293423426, + "self_seconds": 0.05925521500000001 + } + ], + "sha256": "43ec809c22a641250f2ee269119905729cf082c8c5aa51f7489291d6e8f71c1c", + "total_seconds": 1.122652297 + } + }, + "speedup_ratio_of_medians": 1.1106517462248529, + "variants": { + "baseline": { + "peak_RSS_bytes": { + "max": 588607488.0, + "median": 583864320.0, + "min": 582459392.0, + "values": [ + 582459392.0, + 583032832.0, + 588607488.0, + 583864320.0, + 588369920.0 + ] + }, + "post_metrics_exit_seconds": { + "max": 2.2820894457399845, + "median": 2.2715826109051704, + "min": 2.241224192082882, + "values": [ + 2.241224192082882, + 2.2820894457399845, + 2.275044210255146, + 2.243912100791931, + 2.2715826109051704 + ] + }, + "process_seconds": { + "max": 6.9157307259738445, + "median": 6.815988626331091, + "min": 6.813528638333082, + "values": [ + 6.813528638333082, + 6.813662972301245, + 6.819815680384636, + 6.815988626331091, + 6.9157307259738445 + ] + }, + "setup_seconds": { + "max": 2.5969117917120457, + "median": 2.5764283053576946, + "min": 2.5584622099995613, + "values": [ + 2.568533480167389, + 2.5584622099995613, + 2.5764283053576946, + 2.5915246196091175, + 2.5969117917120457 + ] + }, + "trace_bytes": { + "max": 419429.0, + "median": 419429.0, + "min": 419429.0, + "values": [ + 419429.0, + 419429.0, + 419429.0, + 419429.0, + 419429.0 + ] + }, + "training_seconds": { + "max": 0.6755246371030807, + "median": 0.6677760817110538, + "min": 0.6532296538352966, + "values": [ + 0.6743265427649021, + 0.6677760817110538, + 0.6532296538352966, + 0.6667587123811245, + 0.6755246371030807 + ] + } + }, + "candidate": { + "peak_RSS_bytes": { + "max": 594067456.0, + "median": 588800000.0, + "min": 584409088.0, + "values": [ + 587386880.0, + 594067456.0, + 588951552.0, + 588800000.0, + 584409088.0 + ] + }, + "post_metrics_exit_seconds": { + "max": 2.4097439981997013, + "median": 2.23392204195261, + "min": 2.1729552559554577, + "values": [ + 2.4097439981997013, + 2.23392204195261, + 2.2292166277766228, + 2.1729552559554577, + 2.3070859499275684 + ] + }, + "process_seconds": { + "max": 6.918491534888744, + "median": 6.9149907901883125, + "min": 6.7165322713553905, + "values": [ + 6.9149907901883125, + 6.916295755654573, + 6.7165322713553905, + 6.7182507291436195, + 6.918491534888744 + ] + }, + "setup_seconds": { + "max": 2.7161961905658245, + "median": 2.6359554640948772, + "min": 2.568043265491724, + "values": [ + 2.568043265491724, + 2.7161961905658245, + 2.5984866246581078, + 2.6359554640948772, + 2.673058681190014 + ] + }, + "trace_bytes": { + "max": 419429.0, + "median": 419429.0, + "min": 419429.0, + "values": [ + 419429.0, + 419429.0, + 419429.0, + 419429.0, + 419429.0 + ] + }, + "training_seconds": { + "max": 0.6266894452273846, + "median": 0.6012470461428165, + "min": 0.577793013304472, + "values": [ + 0.6266894452273846, + 0.577793013304472, + 0.587515078485012, + 0.6012470461428165, + 0.6260156631469727 + ] + } + } + }, + "workload_manifest": { + "adapter_sha256": "a5b5e597d4921a5b4baceeada541a6a1765f13c97ab3bd1c2ef79dea331f0934", + "api_disabled_ablation": "var mode keeps Instrumentor wrappers with funcs_to_instr=[]; uses production no-dump fast path", + "batch_size": 2, + "benchmark_git_head": null, + "benchmark_source": "/home/yuxuan/experiments/codex-trace-impact-20261013/benchmarks/instr-overhead/overhead-e2e/mnist/main.py", + "benchmark_source_sha256": "4478c5de92610a1e03d61c356801d51ffca749f99dac00f234d6b6ea2bcb2aa7", + "cpu_threads": 1, + "cuda_available": false, + "data": { + "count": 6, + "files": [ + { + "path": "/home/yuxuan/gitrepos/data/MNIST/raw/train-images-idx3-ubyte", + "sha256": "ba891046e6505d7aadcbbe25680a0738ad16aec93bde7f9b65e87a2fc25776db" + }, + { + "path": "/home/yuxuan/gitrepos/data/MNIST/raw/train-labels-idx1-ubyte", + "sha256": "65a50cbbf4e906d70832878ad85ccda5333a97f0f4c3dd2ef09a8a9eef7101c5" + } + ], + "kind": "cached_mnist_training_prefix", + "normalization": "(uint8 / 255 - 0.1307) / 0.3081", + "ordering": "first samples in IDX order; no shuffle" + }, + "deterministic_algorithms": true, + "device": "cpu", + "excluded_phases": [ + "model initialization", + "data preparation", + "metric extraction", + "checkpoint writing" + ], + "extracted_definitions_sha256": "f839142a57ce449054e05ffa79d746e5282e2599e101810e6ccf6e7a9dd33ce5", + "extracted_symbols": [ + "Net" + ], + "initial_state_hash": "877e9abeb783ae5ed703a9666a7276e921e0424b78bb2d50a41461221d90ddb0", + "mode": "combined", + "model_config": { + "architecture": "exact Net class" + }, + "model_tracker_style": "subclass", + "models_to_track": [ + "model" + ], + "observation_files": [ + { + "path": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/static_analyzer/func_level/nn_func_level.log", + "sha256": "0ea56ca02089d5b19a1c0697dea00d107fccd18e17d485ff97bc04a5378dfcba" + }, + { + "path": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/static_analyzer/func_level/optim_func_level.log", + "sha256": "300c47f9babd033ddf68487dde01a828746ff795b5f0111761b2f8fda08ad900" + } + ], + "plain_baseline": "Imports TrainCheck (including global Module.__setattr__ hook), but no API wrappers or proxied parameters; imports/setup excluded", + "precision": "float32", + "profiled": false, + "python": "3.11.4 (main, Jul 5 2023, 13:45:01) [GCC 11.2.0]", + "scan_proxy_in_args": true, + "seed": 42, + "setup_seconds": 2.568533480167389, + "source_commit": "7a40649d2ce55c1a92380e26cd0f74b61c05824b", + "source_tree": "c84687d977b748a5080f034ace78c78b45cec172", + "steps": 3, + "tensor_dump_format": { + "dump_tensor_full": false, + "dump_tensor_hash": true, + "dump_tensor_stats": false + }, + "timing": "Cold three-step CPU training body including start_step and wrapper-disable assignment; excludes setup/loss extraction/state digest/drain", + "topology": "Static observers only in VAR-enabled modes, matching normal collect_trace activation; interaction is not assumed zero", + "torch_version": "2.16.0.dev20261012+cpu", + "tracked_parameter_names": [ + "conv1.weight", + "conv1.bias", + "conv2.weight", + "conv2.bias", + "fc1.weight", + "fc1.bias", + "fc2.weight", + "fc2.bias" + ], + "training_only_trace": true, + "update": "Exact benchmark forward, NLL loss, backward, Adadelta(lr=1.0). Epoch scheduler/evaluation omitted.", + "use_full_instr": false, + "variant": "final_correctness_without_metadata_fastpath", + "workload_kind": "bounded_model_micro_workload", + "wrapped_functions": 21032 + } + }, + "transformer": { + "coverage": { + "logical_hash_input_bytes": 214496, + "records": { + "API": 1340, + "VAR": 64 + }, + "tensor_hash_fields": 64, + "tracked_variables": [ + "model.decoder.bias", + "model.decoder.weight", + "model.encoder.layers.0.linear1.bias", + "model.encoder.layers.0.linear1.weight", + "model.encoder.layers.0.linear2.bias", + "model.encoder.layers.0.linear2.weight", + "model.encoder.layers.0.norm1.bias", + "model.encoder.layers.0.norm1.weight", + "model.encoder.layers.0.norm2.bias", + "model.encoder.layers.0.norm2.weight", + "model.encoder.layers.0.self_attn.in_proj_bias", + "model.encoder.layers.0.self_attn.in_proj_weight", + "model.encoder.layers.0.self_attn.out_proj.bias", + "model.encoder.layers.0.self_attn.out_proj.weight", + "model.encoder.layers.1.linear1.bias", + "model.encoder.layers.1.linear1.weight", + "model.encoder.layers.1.linear2.bias", + "model.encoder.layers.1.linear2.weight", + "model.encoder.layers.1.norm1.bias", + "model.encoder.layers.1.norm1.weight", + "model.encoder.layers.1.norm2.bias", + "model.encoder.layers.1.norm2.weight", + "model.encoder.layers.1.self_attn.in_proj_bias", + "model.encoder.layers.1.self_attn.in_proj_weight", + "model.encoder.layers.1.self_attn.out_proj.bias", + "model.encoder.layers.1.self_attn.out_proj.weight", + "model.encoder.norm.bias", + "model.encoder.norm.weight", + "model.input_emb.weight" + ] + }, + "median_reduction_percent_of_baseline_training": 18.43201701823739, + "median_reduction_seconds": 0.17684413865208626, + "paired_reduction_percent": [ + 15.601390824394477, + 20.54804037418375, + 17.7208926939699, + 15.512118752865149, + 25.14732482059326 + ], + "paired_reductions_seconds": [ + 0.14744044840335846, + 0.19879167526960373, + 0.16855186223983765, + 0.14882946759462357, + 0.257242564111948 + ], + "profiles": { + "baseline": { + "interpretation": "Separate cProfile run; cumulative rows overlap and are not an exclusive breakdown of ordinary timing.", + "rows": [ + { + "calls": 3622987, + "cumulative_seconds": 0.41540509400000003, + "file": "~", + "function": "", + "line": 0, + "percent_of_profile_total": 15.272859297011918, + "self_seconds": 0.24020582000000001 + }, + { + "calls": 516009, + "cumulative_seconds": 0.211889453, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "__enter__", + "line": 466, + "percent_of_profile_total": 7.790366196592234, + "self_seconds": 0.190857086 + }, + { + "calls": 64, + "cumulative_seconds": 0.005668298, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/dumper.py", + "function": "dump_tensor", + "line": 302, + "percent_of_profile_total": 0.20840167599758433, + "self_seconds": 7.4091e-05 + }, + { + "calls": 64, + "cumulative_seconds": 0.005591704, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/proxy_wrapper/hash.py", + "function": "tensor_hash", + "line": 95, + "percent_of_profile_total": 0.20558560705213386, + "self_seconds": 0.00060932 + }, + { + "calls": 1773, + "cumulative_seconds": 2.641869315, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/dumper.py", + "function": "convert_var_to_dict", + "line": 343, + "percent_of_profile_total": 97.1314481018094, + "self_seconds": 0.538952239 + }, + { + "calls": 670, + "cumulative_seconds": 1.513930065, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/tracer.py", + "function": "to_dict_args_kwargs", + "line": 166, + "percent_of_profile_total": 55.6614283315965, + "self_seconds": 0.0012697680000000001 + }, + { + "calls": 516009, + "cumulative_seconds": 1.5134438030000001, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/dumper.py", + "function": "safe_getattr", + "line": 328, + "percent_of_profile_total": 55.64355033439629, + "self_seconds": 0.44812079600000004 + }, + { + "calls": 64, + "cumulative_seconds": 0.0046741460000000006, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/proxy_wrapper/hash.py", + "function": "hash_tensor_cpu", + "line": 47, + "percent_of_profile_total": 0.17185050261249582, + "self_seconds": 0.004206617 + }, + { + "calls": 64, + "cumulative_seconds": 0.20849510200000002, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/proxy_wrapper/subclass.py", + "function": "dump_trace", + "line": 218, + "percent_of_profile_total": 7.66556886989486, + "self_seconds": 0.000155234 + }, + { + "calls": 8292, + "cumulative_seconds": 0.002601659, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/utils.py", + "function": "safe_getattr", + "line": 21, + "percent_of_profile_total": 0.09565306834153729, + "self_seconds": 0.001095569 + }, + { + "calls": 1404, + "cumulative_seconds": 0.01060878, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/dumper.py", + "function": "serialize", + "line": 85, + "percent_of_profile_total": 0.39004433646390013, + "self_seconds": 0.000932733 + }, + { + "calls": 670, + "cumulative_seconds": 0.950235758, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/tracer.py", + "function": "to_dict_return_value", + "line": 210, + "percent_of_profile_total": 34.936540838190744, + "self_seconds": 0.000306486 + }, + { + "calls": 1340, + "cumulative_seconds": 0.015419796000000001, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/dumper.py", + "function": "dump_trace_API", + "line": 241, + "percent_of_profile_total": 0.566927026409135, + "self_seconds": 0.001075404 + }, + { + "calls": 64, + "cumulative_seconds": 0.001013873, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/instrumentor/dumper.py", + "function": "dump_trace_VAR", + "line": 250, + "percent_of_profile_total": 0.03727623926065616, + "self_seconds": 4.5441e-05 + }, + { + "calls": 495303, + "cumulative_seconds": 0.207537147, + "file": "/home/yuxuan/experiments/codex-inference-runtime-20261025/venv/lib/python3.11/site-packages/torch/nn/parameter.py", + "function": "__instancecheck__", + "line": 21, + "percent_of_profile_total": 7.630348522000259, + "self_seconds": 0.146889277 + }, + { + "calls": 1, + "cumulative_seconds": 1.39e-06, + "file": "/home/yuxuan/experiments/codex-inference-runtime-20261025/venv/lib/python3.11/site-packages/torch/nn/parameter.py", + "function": "__instancecheck__", + "line": 240, + "percent_of_profile_total": 5.110499300436254e-05, + "self_seconds": 1.044e-06 + }, + { + "calls": 516009, + "cumulative_seconds": 0.162974535, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "__exit__", + "line": 487, + "percent_of_profile_total": 5.991951418031825, + "self_seconds": 0.144182988 + }, + { + "calls": 516009, + "cumulative_seconds": 0.38795989000000003, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "_add_filter", + "line": 181, + "percent_of_profile_total": 14.263803931239758, + "self_seconds": 0.216126051 + }, + { + "calls": 516009, + "cumulative_seconds": 0.07956756400000001, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "__init__", + "line": 440, + "percent_of_profile_total": 2.9253955407152294, + "self_seconds": 0.07956756400000001 + }, + { + "calls": 516009, + "cumulative_seconds": 0.553407844, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "simplefilter", + "line": 165, + "percent_of_profile_total": 20.346693522431192, + "self_seconds": 0.146483431 + } + ], + "sha256": "2428566e23bf2a60b055d3755c2ada906bb809a40ae26a05455788e5727459df", + "total_seconds": 2.7198907939999986 + }, + "candidate": { + "interpretation": "Separate cProfile run; cumulative rows overlap and are not an exclusive breakdown of ordinary timing.", + "rows": [ + { + "calls": 1154741, + "cumulative_seconds": 0.053038279, + "file": "~", + "function": "", + "line": 0, + "percent_of_profile_total": 2.483533581874309, + "self_seconds": 0.04782687 + }, + { + "calls": 516009, + "cumulative_seconds": 0.20110108200000001, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "__enter__", + "line": 466, + "percent_of_profile_total": 9.416619466447228, + "self_seconds": 0.182952516 + }, + { + "calls": 64, + "cumulative_seconds": 0.00432889, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/proxy_wrapper/hash.py", + "function": "hash_tensor_cpu", + "line": 47, + "percent_of_profile_total": 0.20270159382886233, + "self_seconds": 0.0039002060000000002 + }, + { + "calls": 64, + "cumulative_seconds": 0.005282504, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/dumper.py", + "function": "dump_tensor", + "line": 318, + "percent_of_profile_total": 0.2473548600697501, + "self_seconds": 7.26e-05 + }, + { + "calls": 670, + "cumulative_seconds": 1.184774024, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/tracer.py", + "function": "to_dict_args_kwargs", + "line": 166, + "percent_of_profile_total": 55.47740482937538, + "self_seconds": 0.0012466880000000001 + }, + { + "calls": 64, + "cumulative_seconds": 0.005207354, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/proxy_wrapper/hash.py", + "function": "tensor_hash", + "line": 95, + "percent_of_profile_total": 0.24383593841171788, + "self_seconds": 0.000586207 + }, + { + "calls": 1773, + "cumulative_seconds": 2.056601617, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/dumper.py", + "function": "convert_var_to_dict", + "line": 362, + "percent_of_profile_total": 96.30099763147493, + "self_seconds": 0.35871008800000004 + }, + { + "calls": 516009, + "cumulative_seconds": 1.4721962940000002, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/dumper.py", + "function": "safe_getattr", + "line": 344, + "percent_of_profile_total": 68.93604023727663, + "self_seconds": 0.433983473 + }, + { + "calls": 64, + "cumulative_seconds": 0.161195886, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/proxy_wrapper/subclass.py", + "function": "dump_trace", + "line": 218, + "percent_of_profile_total": 7.548046499415692, + "self_seconds": 0.000153255 + }, + { + "calls": 8292, + "cumulative_seconds": 0.002652792, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/utils.py", + "function": "safe_getattr", + "line": 21, + "percent_of_profile_total": 0.12421779405262211, + "self_seconds": 0.001105293 + }, + { + "calls": 64, + "cumulative_seconds": 0.000806478, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/dumper.py", + "function": "dump_trace_VAR", + "line": 266, + "percent_of_profile_total": 0.037763578189307936, + "self_seconds": 4.4666e-05 + }, + { + "calls": 1340, + "cumulative_seconds": 0.015288563000000002, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/dumper.py", + "function": "dump_trace_API", + "line": 257, + "percent_of_profile_total": 0.7158916228993976, + "self_seconds": 0.001078509 + }, + { + "calls": 1404, + "cumulative_seconds": 0.010575724, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/dumper.py", + "function": "serialize", + "line": 101, + "percent_of_profile_total": 0.4952115001060667, + "self_seconds": 0.000939728 + }, + { + "calls": 670, + "cumulative_seconds": 0.742331382, + "file": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/candidate/source/traincheck/instrumentor/tracer.py", + "function": "to_dict_return_value", + "line": 210, + "percent_of_profile_total": 34.759893247595116, + "self_seconds": 0.00031331 + }, + { + "calls": 8937, + "cumulative_seconds": 0.006075963, + "file": "/home/yuxuan/experiments/codex-inference-runtime-20261025/venv/lib/python3.11/site-packages/torch/nn/parameter.py", + "function": "__instancecheck__", + "line": 21, + "percent_of_profile_total": 0.28450881961546626, + "self_seconds": 0.004308247 + }, + { + "calls": 1, + "cumulative_seconds": 1.334e-06, + "file": "/home/yuxuan/experiments/codex-inference-runtime-20261025/venv/lib/python3.11/site-packages/torch/nn/parameter.py", + "function": "__instancecheck__", + "line": 240, + "percent_of_profile_total": 6.246495664424423e-05, + "self_seconds": 1.068e-06 + }, + { + "calls": 516009, + "cumulative_seconds": 0.15780037300000002, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "__exit__", + "line": 487, + "percent_of_profile_total": 7.389050568133857, + "self_seconds": 0.141191683 + }, + { + "calls": 516009, + "cumulative_seconds": 0.37356121800000003, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "_add_filter", + "line": 181, + "percent_of_profile_total": 17.492117905802896, + "self_seconds": 0.20996710500000001 + }, + { + "calls": 516009, + "cumulative_seconds": 0.07808938900000001, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "__init__", + "line": 440, + "percent_of_profile_total": 3.6565594439734044, + "self_seconds": 0.07808938900000001 + }, + { + "calls": 516009, + "cumulative_seconds": 0.540450281, + "file": "/home/yuxuan/miniconda3/lib/python3.11/warnings.py", + "function": "simplefilter", + "line": 165, + "percent_of_profile_total": 25.306749153699105, + "self_seconds": 0.146992578 + } + ], + "sha256": "99d75dabca1f2ee57a623d97c35581f222b6fd148a2c13b4f4f8575a2df1280f", + "total_seconds": 2.1355974160000004 + } + }, + "speedup_ratio_of_medians": 1.225971224792435, + "variants": { + "baseline": { + "peak_RSS_bytes": { + "max": 540844032.0, + "median": 540766208.0, + "min": 539996160.0, + "values": [ + 540766208.0, + 540528640.0, + 540844032.0, + 540782592.0, + 539996160.0 + ] + }, + "post_metrics_exit_seconds": { + "max": 2.292447991669178, + "median": 1.9001667201519012, + "min": 1.8594940602779388, + "values": [ + 1.9001667201519012, + 1.9286032617092133, + 1.8995989076793194, + 1.8594940602779388, + 2.292447991669178 + ] + }, + "process_seconds": { + "max": 7.21809833869338, + "median": 6.719264965504408, + "min": 6.713514465838671, + "values": [ + 6.716304361820221, + 6.719264965504408, + 6.713514465838671, + 7.1309390515089035, + 7.21809833869338 + ] + }, + "setup_seconds": { + "max": 2.5845856182277203, + "median": 2.550613649189472, + "min": 2.526260081678629, + "values": [ + 2.561651110649109, + 2.526260081678629, + 2.550613649189472, + 2.5845856182277203, + 2.535349767655134 + ] + }, + "trace_bytes": { + "max": 1088456.0, + "median": 1088456.0, + "min": 1088456.0, + "values": [ + 1088456.0, + 1088456.0, + 1088456.0, + 1088456.0, + 1088456.0 + ] + }, + "training_seconds": { + "max": 1.022942066192627, + "median": 0.9594399705529213, + "min": 0.9450468234717846, + "values": [ + 0.9450468234717846, + 0.9674483388662338, + 0.9511476941406727, + 0.9594399705529213, + 1.022942066192627 + ] + } + }, + "candidate": { + "peak_RSS_bytes": { + "max": 540979200.0, + "median": 540778496.0, + "min": 540712960.0, + "values": [ + 540966912.0, + 540979200.0, + 540778496.0, + 540745728.0, + 540712960.0 + ] + }, + "post_metrics_exit_seconds": { + "max": 2.2137519530951977, + "median": 2.1560278981924057, + "min": 2.0495513416826725, + "values": [ + 2.1560278981924057, + 2.0495513416826725, + 2.2079070173203945, + 2.2137519530951977, + 2.0717223808169365 + ] + }, + "process_seconds": { + "max": 6.919477663934231, + "median": 6.816005241125822, + "min": 6.717189069837332, + "values": [ + 6.816975295543671, + 6.815951332449913, + 6.816005241125822, + 6.919477663934231, + 6.717189069837332 + ] + }, + "setup_seconds": { + "max": 2.55265936255455, + "median": 2.5275897532701492, + "min": 2.5160773061215878, + "values": [ + 2.5471372343599796, + 2.5160773061215878, + 2.5275897532701492, + 2.5250496082007885, + 2.55265936255455 + ] + }, + "trace_bytes": { + "max": 1088456.0, + "median": 1088456.0, + "min": 1088456.0, + "values": [ + 1088456.0, + 1088456.0, + 1088456.0, + 1088456.0, + 1088456.0 + ] + }, + "training_seconds": { + "max": 0.8106105029582977, + "median": 0.782595831900835, + "min": 0.7656995020806789, + "values": [ + 0.7976063750684261, + 0.7686566635966301, + 0.782595831900835, + 0.8106105029582977, + 0.7656995020806789 + ] + } + } + }, + "workload_manifest": { + "adapter_sha256": "a5b5e597d4921a5b4baceeada541a6a1765f13c97ab3bd1c2ef79dea331f0934", + "api_disabled_ablation": "var mode keeps Instrumentor wrappers with funcs_to_instr=[]; uses production no-dump fast path", + "batch_size": 2, + "benchmark_git_head": null, + "benchmark_source": "/home/yuxuan/experiments/codex-trace-impact-20261013/benchmarks/false-positive/RNN/trainset/word_language_model-RNN_TANH-3epo/model.py", + "benchmark_source_sha256": "e4451a33e9b16953004bc639d736d14f42a2b21f2939ccabe4cff10d3c40b74c", + "cpu_threads": 1, + "cuda_available": false, + "data": { + "adaptation": "Whitespace tokenization plus ; vocabulary and tensors are bounded to the chosen prefix, unlike the full benchmark corpus.", + "kind": "wikitext2_training_prefix_with_prefix_only_vocabulary", + "path": "/home/yuxuan/experiments/codex-trace-impact-20261013/benchmarks/false-positive/RNN/trainset/word_language_model-RNN_TANH-3epo/data/wikitext-2/train.txt", + "prefix_sha256": "9b0b504c73ee103a9128f2072536e99b2696039399bd57669477f030db10eb35", + "sha256": "9e9fa1ad55b1c2c95b08e37dd8e653f638fac2c6de904b79e813611eefbc985f", + "token_count": 98, + "vocabulary_size": 60 + }, + "deterministic_algorithms": true, + "device": "cpu", + "excluded_phases": [ + "model initialization", + "data preparation", + "metric extraction", + "checkpoint writing" + ], + "extracted_definitions_sha256": "1f2db95454690be6a91aaa141e3a5a246308f9abca11444c9113e4adfb78b95d", + "extracted_symbols": [ + "PositionalEncoding", + "TransformerModel" + ], + "initial_state_hash": "e197d3c2a3f351146bc265c5b3eb54dcd1d921499ce24d1cc61bdc5e6dce886d", + "mode": "combined", + "model_config": { + "clip": 0.25, + "dropout_argument": 0.2, + "lr": 1.0, + "nhead": 2, + "nhid": 64, + "ninp": 32, + "nlayers": 2, + "ntoken": 60, + "sequence_length": 16 + }, + "model_tracker_style": "subclass", + "models_to_track": [ + "model" + ], + "observation_files": [ + { + "path": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/static_analyzer/func_level/nn_func_level.log", + "sha256": "0ea56ca02089d5b19a1c0697dea00d107fccd18e17d485ff97bc04a5378dfcba" + }, + { + "path": "/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched/baseline/source/traincheck/static_analyzer/func_level/optim_func_level.log", + "sha256": "300c47f9babd033ddf68487dde01a828746ff795b5f0111761b2f8fda08ad900" + } + ], + "plain_baseline": "Imports TrainCheck (including global Module.__setattr__ hook), but no API wrappers or proxied parameters; imports/setup excluded", + "precision": "float32", + "profiled": false, + "python": "3.11.4 (main, Jul 5 2023, 13:45:01) [GCC 11.2.0]", + "scan_proxy_in_args": true, + "seed": 42, + "setup_seconds": 2.561651110649109, + "source_commit": "7a40649d2ce55c1a92380e26cd0f74b61c05824b", + "source_tree": "c84687d977b748a5080f034ace78c78b45cec172", + "steps": 3, + "tensor_dump_format": { + "dump_tensor_full": false, + "dump_tensor_hash": true, + "dump_tensor_stats": false + }, + "timing": "Cold three-step CPU training body including start_step and wrapper-disable assignment; excludes setup/loss extraction/state digest/drain", + "topology": "Static observers only in VAR-enabled modes, matching normal collect_trace activation; interaction is not assumed zero", + "torch_version": "2.16.0.dev20261012+cpu", + "tracked_parameter_names": [ + "encoder.layers.0.self_attn.in_proj_weight", + "encoder.layers.0.self_attn.in_proj_bias", + "encoder.layers.0.self_attn.out_proj.weight", + "encoder.layers.0.self_attn.out_proj.bias", + "encoder.layers.0.linear1.weight", + "encoder.layers.0.linear1.bias", + "encoder.layers.0.linear2.weight", + "encoder.layers.0.linear2.bias", + "encoder.layers.0.norm1.weight", + "encoder.layers.0.norm1.bias", + "encoder.layers.0.norm2.weight", + "encoder.layers.0.norm2.bias", + "encoder.layers.1.self_attn.in_proj_weight", + "encoder.layers.1.self_attn.in_proj_bias", + "encoder.layers.1.self_attn.out_proj.weight", + "encoder.layers.1.self_attn.out_proj.bias", + "encoder.layers.1.linear1.weight", + "encoder.layers.1.linear1.bias", + "encoder.layers.1.linear2.weight", + "encoder.layers.1.linear2.bias", + "encoder.layers.1.norm1.weight", + "encoder.layers.1.norm1.bias", + "encoder.layers.1.norm2.weight", + "encoder.layers.1.norm2.bias", + "encoder.norm.weight", + "encoder.norm.bias", + "decoder.weight", + "decoder.bias", + "input_emb.weight" + ], + "training_only_trace": true, + "update": "Adapted language-model forward/NLL/backward/clipped manual SGD. Model classes copied exactly; the benchmark main.py training path is RNN-specific and is not executed. Transformer dimensions/vocabulary reduced.", + "use_full_instr": false, + "variant": "final_correctness_without_metadata_fastpath", + "workload_kind": "bounded_model_micro_workload", + "wrapped_functions": 21032 + } + } + } +} diff --git a/docs/measurements/metadata-classification-20261027.md b/docs/measurements/metadata-classification-20261027.md new file mode 100644 index 00000000..0cccc6ce --- /dev/null +++ b/docs/measurements/metadata-classification-20261027.md @@ -0,0 +1,232 @@ +# Final matched metadata-classification measurements + +With the reviewed correctness changes on both sides, the metadata fast path +reduces median three-step CPU training time by **9.96% on MNIST** and **18.43% +on the small Transformer**. Both API and subclass model tracing are enabled +under ordinary collection (`use_full_instr=False`, retaining normal exclusions). +Every one of the ten matched pairs improves. Hashing is unchanged. + +These final numbers supersede the initial study for the current implementation. +The initial evidence remains in the parent output directory and in repository +checkpoint `bcf9bb82c3c11812336d1199dc699aa4ee6473e2`. Do not compare timings +across the studies to attribute additional improvements. + +## Exact sources and experiment + +| Variant | Commit | Tree | +|---|---|---| +| Baseline: all current correctness/default changes, metadata fast path removed | `7a40649d2ce55c1a92380e26cd0f74b61c05824b` | `c84687d977b748a5080f034ace78c78b45cec172` | +| Candidate: all current changes including metadata fast path | `035f7f1976c5d84bd01f5d96bb095fb07c9cc059` | `89fdd92ebc6a57e5878d27cfb1b18dc69ee28a0b` | + +All **78 production Python files per variant** match the respective exact Git +archive byte for byte. The only production difference is +`traincheck/instrumentor/dumper.py`. Both sides include the parameter identity, +alias/ownership lifecycle, default-tracker and constructor-snapshot fixes. + +The optimization recognizes exact built-in callable types before repeatedly +checking them against framework tensor/module classes. Attribute enumeration, +getter evaluation, warning scope/restoration, supported metadata fields and the +generic fallback remain present. No hash algorithm, backend, dump format, +sampling or observation-depth change is part of the comparison. + +Conditions: labmach Intel Core i9-13900K; Python 3.11.4; PyTorch +`2.16.0.dev20261012+cpu`; one PyTorch/native thread; CPU only; float32; batch 2; +three **cold** steps. Initialization seeds are 42, training seeds 43 for Python, +NumPy and PyTorch. No affinity or exclusive host reservation was used. + +- MNIST uses the exact benchmark `Net` (1,199,882 parameters), the first six + cached MNIST training images, NLL loss and Adadelta with learning rate 1. +- Transformer uses exact benchmark model definitions with bounded dimensions: + embedding 32, two layers, two heads, feedforward 64, sequence length 16 and + vocabulary 60 from the WikiText-2 prefix. Gradient clipping and the original + manual SGD updates remain present. +- Model registration occurs before optimizer construction. Both-tracer runs use + explicit subclass tracking, depth-3 static observers, argument scanning, + hash-format tensor dumps on CPU, sampling interval 1, warmup 0, and disabled + API descriptors. This does not measure `--use-full-instr`, Proxy/VarSampler + modes, CUDA, MPS or distributed training. + +The timer includes `start_step`, the training body and disabling instrumentation +on exit. It excludes imports/setup, loss extraction, model/RNG verification and +final writer shutdown. RNG capture runs **after the timer with wrappers disabled** +and adds no API trace records. Every case uses a fresh process and has a 90-second, +4-GiB sampled main-process RSS, 256-MiB trace limit; no case hit a limit. + +## Ordinary measurements + +Five pairs per workload alternate baseline/candidate then candidate/baseline, +ending baseline/candidate. Profiles are separate from these measurements. +Percent reduction uses the baseline median **training-body time** as denominator, +not added overhead or whole-process time. + +| Workload | Baseline median (range), ms | Candidate median (range), ms | Reduction | Ratio of medians | +|---|---:|---:|---:|---:| +| MNIST | 667.776 (653.230–675.525) | 601.247 (577.793–626.689) | **66.529 ms / 9.96%** | 1.111× | +| Transformer | 959.440 (945.047–1,022.942) | 782.596 (765.700–810.611) | **176.844 ms / 18.43%** | 1.226× | + +| Workload | Baseline runs, ms | Candidate runs, ms | Paired reductions, ms | +|---|---|---|---| +| MNIST | 674.327, 667.776, 653.230, 666.759, 675.525 | 626.689, 577.793, 587.515, 601.247, 626.016 | 47.637, 89.983, 65.715, 65.512, 49.509 | +| Transformer | 945.047, 967.448, 951.148, 959.440, 1,022.942 | 797.606, 768.657, 782.596, 810.611, 765.700 | 147.440, 198.792, 168.552, 148.829, 257.243 | + +Paired reductions range from 7.06%–13.48% for MNIST and 15.51%–25.15% for +Transformer. No outliers were discarded. Five short pairs on a shared machine +do not establish a confidence interval, steady-state scaling or GPU performance. + +## Profiles and remaining work + +One separate profile per variant/workload has total time **1.372028 → 1.122652 s** +for MNIST and **2.719891 → 2.135597 s** for Transformer. Every percentage below +uses that variant's **profiled total**, not its ordinary median. Cumulative rows +overlap; they cannot be added into an exclusive cost partition. + +| Cumulative component | MNIST baseline → candidate, ms (% of profile total) | Transformer baseline → candidate, ms (% of profile total) | +|---|---|---| +| Metadata conversion, `convert_var_to_dict` | 1,307.722 (95.31%) → 1,061.525 (94.56%) | 2,641.869 (97.13%) → 2,056.602 (96.30%) | +| Built-in `isinstance` | 169.932 (12.39%) → 20.078 (1.79%) | 415.405 (15.27%) → 53.038 (2.48%) | +| Attribute lookup, dumper `safe_getattr` | 624.366 (45.51%) → 598.322 (53.30%) | 1,513.444 (55.64%) → 1,472.196 (68.94%) | +| Warning `simplefilter`, nested in attribute lookup | 228.109 (16.63%) → 219.396 (19.54%) | 553.408 (20.35%) → 540.450 (25.31%) | +| API argument conversion | 599.447 (43.69%) → 461.566 (41.11%) | 1,513.930 (55.66%) → 1,184.774 (55.48%) | +| API return conversion | 293.872 (21.42%) → 223.348 (19.90%) | 950.236 (34.94%) → 742.331 (34.76%) | +| `ProxyParameter.dump_trace` | 423.777 (30.89%) → 387.950 (34.56%) | 208.495 (7.67%) → 161.196 (7.55%) | +| `tensor_hash`, nested in snapshots | 221.088 (16.11%) → 230.689 (20.55%) | 5.592 (0.21%) → 5.207 (0.24%) | +| Final JSON `serialize` | 4.231 (0.31%) → 5.508 (0.49%) | 10.609 (0.39%) → 10.576 (0.50%) | + +`isinstance` calls fall from **1,503,819 to 464,813** for MNIST and from +**3,622,987 to 1,154,741** for Transformer. Parameter `__instancecheck__` calls +fall from 203,335 to 2,817 and 495,303 to 8,937 respectively; Transformer also +has one separate instance-check entry in each profile. + +Attribute lookups and warning scopes stay at **211,616 for MNIST** and **516,009 +for Transformer**. Their repeated setup remains a concrete next instrumentation +bottleneck, subject to preserving getter side effects and warning restoration. +Final JSON encoding is small in these profiles. Hashing was not optimized: +both variants use the existing CPU implementation, with unchanged call counts. +The small profile hash-time differences are not a hash-speedup claim. +Metadata conversion and JSON encoding still run synchronously in the producer; +the background threads handle buffered file writing. The small JSON row does +not imply that all preparation of a trace record is cheap or asynchronous. + +## Correctness, coverage and resources + +All **30 cases** passed: six candidate correctness cases (Transformer plain, +API, VAR and combined; MNIST plain and combined), twenty ordinary paired cases +and four profiles. The six correctness cases complete before measurement starts. +Every paired/profiled run is checked directly against the corresponding plain +candidate run: exact initial/final model-state digests and shapes, every loss +and step ID, and Python/NumPy/PyTorch RNG state before training and after each +step. Only elapsed seconds are excluded from loss-row comparison. + +All retained raw trace checksums, sizes, counts and API pre/post multiplicities +were reread locally. Every tracked parameter has changing data hashes across +the three steps. Counts and coverage signatures agree across ordinary/profiled +baseline and candidate runs: + +| Workload | API records / complete pairs | VAR records / parameters | Tensor data+gradient hash fields | Raw API+VAR bytes | +|---|---:|---:|---:|---:| +| MNIST | 520 / 260 | 64 / 8 | 112 | 419,429 | +| Transformer | 1,340 / 670 | 64 / 29 | 64 | 1,088,456 | + +Profile calls to both `tensor_hash` and `hash_tensor_cpu` exactly match 112 and +64 recorded hash fields. MNIST has 48 gradient hashes in addition to its 64 data +hashes. Transformer snapshots have `grad=None` at the existing observation points: +they capture changing parameter values, but **do not establish gradient snapshots +or pre/post observations for every manual SGD update**. We did not replace manual +SGD or add synthetic observations to improve those counts. Final model/loss/RNG +checks separately establish behavior parity for the completed training steps. + +Logical hashed input volume, counting repeated snapshots, is 67,193,392 bytes for +MNIST and 214,496 bytes for Transformer; this is not physical memory traffic or +a time share. Raw identities and timestamps remain intact. There is no claim of +normalized raw trace equality across independent processes. + +| Ordinary median | MNIST baseline → candidate | Transformer baseline → candidate | +|---|---:|---:| +| Setup | 2.576 → 2.636 s | 2.551 → 2.528 s | +| Whole process | 6.816 → 6.915 s | 6.719 → 6.816 s | +| Post-metrics exit | 2.272 → 2.234 s | 1.900 → 2.156 s | +| Sampled main-process peak RSS | 556.816 → 561.523 MiB | 515.715 → 515.727 MiB | + +Post-metrics exit includes background-writer drain, Python shutdown and polling +delay. It is not a pure I/O timer. These short jobs do **not** show a whole-process +speedup; startup/shutdown and host variation outweigh the training-body savings. +Full per-run distributions are retained in [`data/metadata-classification-20261027.json`](data/metadata-classification-20261027.json). + +## Artifacts and reproduction + +Local directory: +`/Users/matt/gitrepos/traincheck/outputs/instrumenter-performance-20261027/final-matched`. +Remote directory: +`/home/yuxuan/experiments/codex-instrumenter-performance-20261027-final-matched`. + +The source archives are exact Git exports. Their isolated workload adapters add +the same typed model-registration callback before optimizer construction; no +production source is patched in place. Data and original benchmark definitions +come from the existing read-only caches recorded in each manifest. + +With a fresh copy of the retained experiment structure (run directories must not +already exist): + +```bash +/home/yuxuan/experiments/codex-inference-runtime-20261025/venv/bin/python correctness_driver.py --root "$EXPERIMENT_ROOT/candidate" +/home/yuxuan/experiments/codex-inference-runtime-20261025/venv/bin/python run_experiment.py --root "$EXPERIMENT_ROOT" --phase paired +python analyze_experiment.py --root "$LOCAL_EVIDENCE_ROOT" +``` + +The runner records every child command and sets the isolated source path, native +thread limits, CPU visibility and deterministic seed configuration. It aborts on +execution, coverage, model/loss/RNG or resource-limit failure. The analyzer verifies +source archives, executed harness hashes, raw records, plain references, unchanged +production-file scope and profile hash activation. No installations or shared +source/environment edits were used. + +The benchmark copy has no available Git-head identity. Exact source/definition +and data hashes are therefore retained in each manifest and [`data/metadata-classification-20261027.json`](data/metadata-classification-20261027.json): + +| Input/environment artifact | SHA-256 | +|---|---| +| MNIST benchmark source | `4478c5de92610a1e03d61c356801d51ffca749f99dac00f234d6b6ea2bcb2aa7` | +| Transformer benchmark source | `e4451a33e9b16953004bc639d736d14f42a2b21f2939ccabe4cff10d3c40b74c` | +| MNIST image IDX file (first six samples used) | `ba891046e6505d7aadcbbe25680a0738ad16aec93bde7f9b65e87a2fc25776db` | +| MNIST label IDX file | `65a50cbbf4e906d70832878ad85ccda5333a97f0f4c3dd2ef09a8a9eef7101c5` | +| WikiText-2 training file | `9e9fa1ad55b1c2c95b08e37dd8e653f638fac2c6de904b79e813611eefbc985f` | +| Used 98-token prefix | `9b0b504c73ee103a9128f2072536e99b2696039399bd57669477f030db10eb35` | +| Runtime package inventory | `f9b751a6b4b74230992a297eec30164e8d78fcd6a2d177726075f381bfae3c37` | +| CPU inventory | `ab09fc242f18a43f3e25a1bb6d8aaf6b2940f71efa4c2800c82f825f13bdaaeb` | + +| Artifact | SHA-256 | +|---|---| +| Baseline source archive | `efc665f1b6914964f3ac93ac76a3e00791b5cf07c310818c65e7711496260537` | +| Candidate source archive | `793087ee08bbfb10ed60712d7d22da2153809236c26eb113d78592695754fe4e` | +| Raw evidence archive, 25,722,591 bytes | `875be9098fab644afed9798cd6005f66af6567719e6be1765d84e9bd74725584` | +| Executed runner | `21984b309b688e95de62d37b59e6e5f7c9a1685ef5293e54dfaacd60e9305875` | +| Executed paired driver | `5a0d4c4a02ea397bce646a482c2657bb80a1cd28bb5fa6e7cd0127071c6fda67` | +| Executed correctness driver | `752968c1c2bd4b646f2c30dc83788597ff5567d41b3759bebc2b9c658795d6fe` | + +Configured Ruff/Black/isort checks pass for the four harness/analysis modules. +Scoped mypy passes with `--check-untyped-defs --disallow-untyped-defs +--disallow-any-generics --follow-imports=skip --ignore-missing-imports`; dependency +typing is not fully checked by that invocation. The exact composed candidate also passes pinned production hooks, configured +mypy on all 78 production files, and strict installed-Torch checking of eight +new contract/test modules. The complete suite passes 246/247 tests; the sole +`test_inject_eval_loop` failure also occurs on unchanged main. Separate real CLI +runs complete ordinary, selective and full-flag collection, with exact training +parity and two active passing invariant decisions each. Detailed correctness +evidence is retained in `outputs/instrumenter-final-validation-20261027` and +`outputs/instrumenter-final-cli-20261027`, and described in PRs #154, #155 and #157. +Raw traces, profiles and checked harnesses remain in the experiment directories; +only the reviewed report and compact measurement evidence are committed here. + + +## Asynchronous snapshot work + +File writes already use background threads on the subclass path. Moving only +final JSON encoding off the producer offers little headroom in this study. +The larger synchronous work inspects live objects and prepares their metadata. +Moving those getters to a later worker after a shallow copy can observe changed +values and change when errors occur. A future asynchronous tensor snapshot path +needs owned snapshots, producer-stream ordering, bounded bytes/items, ordering +per destination, and explicit worker-error and drain behavior. This change adds +no asynchronous snapshot conversion. Per-attribute warning setup is the next +measured implementation target, provided getter and warning semantics remain +intact. Selective-checking overhead needs its own timing study. diff --git a/tests/test_metadata_classification.py b/tests/test_metadata_classification.py new file mode 100644 index 00000000..c417ad14 --- /dev/null +++ b/tests/test_metadata_classification.py @@ -0,0 +1,227 @@ +"""Record contracts for metadata classification, including custom type behavior.""" + +import unittest +import warnings +from contextlib import ExitStack +from unittest.mock import patch + +import torch + +from traincheck.instrumentor import dumper + + +class AttributeHolder: + def __init__(self, value: object) -> None: + self.value = value + + +class CustomParameter(torch.nn.Parameter): + pass + + +class CustomTensor(torch.Tensor): + pass + + +class ChangingClass: + """An object allowed by Python's isinstance protocol, not an actual tensor.""" + + def __init__(self, error: Exception | None = None) -> None: + self.reads = 0 + self.error = error + + def __getattribute__(self, name: str) -> object: + if name == "__class__": + self.reads += 1 + if self.reads == 1: + return object + if self.error is not None: + raise self.error + return torch.nn.Parameter + return super().__getattribute__(name) + + +class MetadataClassificationTests(unittest.TestCase): + def setUp(self) -> None: + contexts = ExitStack() + self.addCleanup(contexts.close) + for name in ( + "skip_attrs_due_to_errs", + "skip_type_due_to_errs", + "skip_type_due_to_recursion", + ): + contexts.enter_context(patch.object(dumper, name, {})) + + def test_callable_attributes_keep_metadata_values_and_order(self) -> None: + def function() -> None: + return None + + callables: tuple[object, ...] = ( + function, + len, + self.setUp, + object().__str__, + object.__str__, + str.upper, + dict.__dict__["fromkeys"], + ) + for callback in callables: + with self.subTest(callback_type=type(callback)): + holder = AttributeHolder(callback) + holder.__dict__["_TRAINCHECK_callback"] = callback + holder.__dict__["name"] = "observed" + holder.__dict__["dtype"] = torch.float32 + holder.__dict__["shape"] = torch.Size([2, 3]) + record = dumper.convert_var_to_dict( + holder, + include_tensor_data=False, + dump_config={ + "value": None, + "name": None, + "_TRAINCHECK_callback": None, + "dtype": None, + "shape": None, + }, + ) + self.assertEqual( + list(record), ["name", "_TRAINCHECK_callback", "dtype", "shape"] + ) + self.assertEqual(record["name"], "observed") + self.assertIs(record["_TRAINCHECK_callback"], callback) + self.assertEqual(record["dtype"], "torch.float32") + self.assertEqual(record["shape"], (2, 3)) + + def test_tensor_and_parameter_subclasses_keep_identity_and_values(self) -> None: + flagged = torch.tensor([3.0, 4.0]) + flagged.__dict__["_is_param"] = True + values = ( + torch.tensor([3.0, 4.0]), + torch.tensor([3.0, 4.0]).as_subclass(CustomTensor), + torch.nn.Parameter(torch.tensor([3.0, 4.0])), + CustomParameter(torch.tensor([3.0, 4.0])), + flagged, + ) + with patch.dict( + dumper.tensor_dump_format, + dump_tensor_hash=False, + dump_tensor_stats=False, + dump_tensor_full=True, + ): + for value in values: + with self.subTest(tensor_type=type(value)): + holder = AttributeHolder(value) + metadata = dumper.convert_var_to_dict( + holder, False, {"value": None} + ) + self.assertEqual(metadata, {"_TRAINCHECK_value_ID": id(value)}) + before = dumper.convert_var_to_dict(holder, True, {"value": None}) + self.assertEqual( + before, + {"_TRAINCHECK_value_ID": id(value), "value": [3.0, 4.0]}, + ) + self.assertEqual(list(before), ["_TRAINCHECK_value_ID", "value"]) + with torch.no_grad(): + value.add_(1) + after = dumper.convert_var_to_dict(holder, True, {"value": None}) + self.assertEqual( + after, + {"_TRAINCHECK_value_ID": id(value), "value": [4.0, 5.0]}, + ) + self.assertEqual(before["value"], [3.0, 4.0]) + + def test_dynamic_class_keeps_parameter_fallback(self) -> None: + # Parameter.__instancecheck__ makes a different number of __class__ reads + # across supported PyTorch versions. Preserve the framework's exact + # lookup behavior, including side effects, without pinning that count. + native = ChangingClass() + self.assertFalse(isinstance(native, torch.Tensor)) + self.assertTrue(isinstance(native, torch.nn.Parameter)) + value = ChangingClass() + record = dumper.convert_var_to_dict( + AttributeHolder(value), False, {"value": None} + ) + self.assertEqual(record, {"_TRAINCHECK_value_ID": id(value)}) + self.assertEqual(value.reads, native.reads) + + def test_dynamic_class_exception_still_propagates(self) -> None: + error = ValueError("dynamic class failed") + value = ChangingClass(error) + with self.assertRaises(ValueError) as caught: + dumper.convert_var_to_dict(AttributeHolder(value), False, {"value": None}) + self.assertIs(caught.exception, error) + + def test_custom_metaclass_does_not_add_classification_side_effects(self) -> None: + visits: list[str] = [] + + class Metaclass(type): + def __hash__(cls) -> int: + visits.append("hash") + if len(visits) > 1: + raise ValueError("unexpected repeated type hashing") + return type.__hash__(cls) + + class Value(metaclass=Metaclass): + pass + + self.assertEqual( + dumper.convert_var_to_dict( + AttributeHolder(Value()), False, {"value": None} + ), + {}, + ) + self.assertEqual(visits, ["hash"]) + + def test_getters_warning_restoration_and_failed_attribute_backoff(self) -> None: + visits: list[str] = [] + + class Getters: + @property + def callback(self) -> object: + visits.append("callback") + warnings.warn("metadata warning", UserWarning) + warnings.simplefilter("error", RuntimeWarning) + return len + + @property + def broken(self) -> object: + visits.append("broken") + raise ValueError("missing metadata") + + @property + def value(self) -> int: + visits.append("value") + warnings.warn("later warning", RuntimeWarning) + return 7 + + with warnings.catch_warnings(record=True) as observed: + warnings.simplefilter("always") + filters = list(warnings.filters) + holder = Getters() + config = {"callback": None, "broken": None, "value": None} + self.assertEqual( + dumper.convert_var_to_dict(holder, False, config), {"value": 7} + ) + self.assertEqual( + dumper.convert_var_to_dict(holder, False, config), {"value": 7} + ) + self.assertEqual(warnings.filters, filters) + self.assertEqual(observed, []) + self.assertEqual(visits, ["callback", "broken", "value", "callback", "value"]) + + def test_dict_subclass_keeps_dynamic_keys_and_missing_fields(self) -> None: + class Metadata(dict[str, object]): + pass + + value = Metadata(first=1, callback=len) + self.assertEqual(dumper.convert_var_to_dict(value, False), {"first": 1}) + value["second"] = 2 + result = dumper.convert_var_to_dict(value, False) + self.assertEqual(result, {"first": 1, "second": 2}) + self.assertEqual(list(result), ["first", "second"]) + self.assertEqual( + dumper.convert_var_to_dict(value, False, {"missing": None}), {} + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/traincheck/instrumentor/dumper.py b/traincheck/instrumentor/dumper.py index 5a852d33..c8d7ce3a 100644 --- a/traincheck/instrumentor/dumper.py +++ b/traincheck/instrumentor/dumper.py @@ -3,8 +3,10 @@ import os import threading import time +import types +from collections.abc import Mapping from queue import Empty, Queue -from typing import Callable, Hashable +from typing import Callable, Hashable, cast import orjson import torch @@ -45,6 +47,20 @@ logger = logging.getLogger(__name__) +# Exact callable types have no tensor/module metadata. Subclasses and arbitrary +# callable objects retain the generic path, including dynamic __class__ behavior. +_METADATA_CALLABLE_TYPES: frozenset[type] = frozenset( + { + types.FunctionType, + types.BuiltinFunctionType, + types.MethodType, + types.MethodWrapperType, + types.WrapperDescriptorType, + types.MethodDescriptorType, + types.ClassMethodDescriptorType, + } +) + def _json_default(o): try: @@ -325,22 +341,29 @@ class NOT_FOUND: pass -def safe_getattr(obj, attr_name): +def safe_getattr(obj: object, attr_name: str) -> object: import warnings try: with warnings.catch_warnings(): warnings.simplefilter("ignore") - attr = getattr(obj, attr_name, NOT_FOUND) + attr: object = getattr(obj, attr_name, NOT_FOUND) if attr is NOT_FOUND: if issubclass(type(obj), dict): - attr = dict.get(obj, attr_name, NOT_FOUND) + # The concrete-type check validates this cast without + # invoking an object's potentially dynamic __class__. + mapping = cast(dict[object, object], obj) + attr = dict.get(mapping, attr_name, NOT_FOUND) return attr except Exception: return NOT_FOUND -def convert_var_to_dict(var, include_tensor_data=True, dump_config=None) -> dict: +def convert_var_to_dict( + var: object, + include_tensor_data: bool = True, + dump_config: Mapping[str, object] | None = None, +) -> dict[str, object]: """ TODO: variables can be nested and thus this should be a recursive function (dump_config supports this). But this function is not recursive yet, so it only dumps the first level of attributes of the variable. @@ -364,7 +387,10 @@ def convert_var_to_dict(var, include_tensor_data=True, dump_config=None) -> dict name for name in dir(var) if not name.startswith("__") ] # dir() won't get called on primitive vars (look at the logic below checking for primitive types) whose dump_config is always None, so no need to check for primitive types here. if issubclass(type(var), dict): - attr_names += [k for k in var.keys() if isinstance(k, str)] + # The concrete-type check excludes objects spoofing __class__; + # mypy cannot narrow through issubclass(type(...), ...). + mapping = cast(dict[object, object], var) + attr_names += [k for k in mapping.keys() if isinstance(k, str)] except Exception as e: if type(var) not in skip_type_due_to_errs: skip_type_due_to_errs[type(var)] = 0 @@ -415,9 +441,17 @@ def convert_var_to_dict(var, include_tensor_data=True, dump_config=None) -> dict continue attr_name = str(attr_name) - if type(attr) in primitive_types: + attr_type = type(attr) + if attr_type in primitive_types: result[attr_name] = attr + elif type(attr_type) is type and attr_type in _METADATA_CALLABLE_TYPES: + # These exact built-in types cannot match the framework branches + # below. Retain explicitly requested TrainCheck metadata unchanged. + # The metaclass guard avoids extra user-defined __hash__/__eq__ calls. + if "_TRAINCHECK" in attr_name: + result[attr_name] = attr + elif isinstance(attr, torch.Tensor): result[f"_TRAINCHECK_{attr_name}_ID"] = id(attr) if include_tensor_data: