diff --git a/docs/experiments/dynamic-type-names-results.json b/docs/experiments/dynamic-type-names-results.json new file mode 100644 index 00000000..ab69fc52 --- /dev/null +++ b/docs/experiments/dynamic-type-names-results.json @@ -0,0 +1,67 @@ +{ + "baseline_commit": "9ac61d1140cf8679c24442a174ec12f4de069980", + "python": "3.10.16", + "torch": "2.2.2", + "platform": "macOS-27.0.1-arm64-arm-64bit", + "method": "Identical seven-test suite, candidate module globals, exact parent typename function extracted by AST versus candidate function.", + "before": { + "tests": 7, + "failures": 3, + "errors": 0, + "passed": 4, + "failed_tests": [ + "test_dynamic_typename.DynamicTypenameTests.test_dynamic_callable_metadata_uses_class_name", + "test_dynamic_typename.DynamicTypenameTests.test_non_string_name_values_fall_back_without_truth_testing", + "test_dynamic_typename.DynamicTypenameTests.test_real_pytorch_dynamic_ops_wrapper" + ] + }, + "after": { + "tests": 7, + "failures": 0, + "errors": 0, + "passed": 7, + "failed_tests": [] + }, + "linux_followup": { + "fix_commit": "2f034ae5d0fab7e792b7c0a5bdc6a1acfeb8962c", + "utils_sha256": "854c11519e7aa6d0cbbfd79694d2ad4895fb1aaf72e9efbb0068048598b70750", + "runtime": { + "python": "3.11.4 (main, Jul 5 2023, 13:45:01) [GCC 11.2.0]", + "torch": "2.16.0.dev20261012+cpu" + }, + "ops_wrapper_names": { + "instrumentation": "torch._inductor.virtualized.OpsWrapper", + "runtime": "torch._inductor.virtualized.OpsWrapper" + }, + "tests": { + "v3-integration-tests": { + "tests_passed": 29, + "exit_code": 0, + "source_unchanged": true, + "stdout_sha256": "be7f9c929651de7f09f383a8f3b0176ec8d3285f0ad053f4d49e6e837a40309b" + }, + "v3-coverage-tests": { + "tests_passed": 11, + "exit_code": 0, + "source_unchanged": true, + "stdout_sha256": "a33575d7feb6117b55c84c6919d21911e1f76065db033ebbdd9270671c536239" + } + }, + "scope": "Exact PR146 utility in composed source; seven type-name tests included in 29 integration tests. No training or timing claim from these preflight results.", + "collection_smoke": { + "workload": "Bounded benchmark MNIST Net / cached real MNIST / batch2 / CPUfloat32 / seed42", + "completed_steps": 10, + "success": true, + "source_unchanged": true, + "state_shapes_unchanged": true, + "source_stack": "PR144 plus PR142/143/145/146 production fixes; immutable integration-v3, signed74b harness", + "evidence_sha256": { + "phase-result.json": "03a50887287fb8e98398735e738412edd6c356bcc21dc39ee15929d5ffcf324c", + "metrics.json": "d99a6fd45dc534964835937a8a623ad98309052a3771e9750c3f8dfb0bfe0713", + "adaptation.json": "8a533afe8ea099fbb255124ba589f2edf068f1c17d5d3774d73c0ff88a63fa8e", + "invocation.json": "8ecea3da01502bff67024e8aa303f3d7e6ab72e7a7d1191b3aef53d61868fc75" + }, + "scope": "Collection completed with exact narrow PR146 utility in composed stack. Separate later shape-only fault-selector failure does not affect this collection result. No fix-isolated training timing claim." + } + } +} diff --git a/docs/experiments/dynamic-type-names.md b/docs/experiments/dynamic-type-names.md new file mode 100644 index 00000000..a6df3f75 --- /dev/null +++ b/docs/experiments/dynamic-type-names.md @@ -0,0 +1,101 @@ +# Dynamic type-name metadata + +## Problem and change + +PyTorch `torch._inductor.virtualized.ops` returns callable values for dynamic +`__qualname__` and `__name__` attributes. TrainCheck rejected the first value but +accepted the second, then asserted that its type name was a string. A fresh +MNIST trace-collection attempt failed before training reached its first step. + +Validate `__name__` as a nonempty string before using it. Otherwise, use the +existing reported-class fallback. This does not call dynamic metadata, truth-test +non-string values, change valid name precedence, or replace the reported +`__class__` with the object's actual type. + +## Validation + +An identical seven-test suite was run against the exact parent `typename` +function extracted from commit `9ac61d1140cf8679c24442a174ec12f4de069980` and against +the changed function, with identical module globals. Python 3.10.16 / PyTorch +2.2.2 on macOS arm64: + +| Result | Parent | Changed | +| --- | ---: | ---: | +| Tests passed | 4 | 7 | +| Tests failed | 3 | 0 | +| Test errors | 0 | 0 | + +The three previously failing tests cover callable dynamic metadata, non-string +names (including an unsafe truth-value implementation), and the real PyTorch +`OpsWrapper`. The remaining tests preserve qualified/simple names, reported-class +proxies, functions, descriptors, tensors, and parameters. Both instrumentation +and runtime modes are exercised for dynamic metadata. + +Reproduce the candidate suite with: + +```sh +python -m unittest tests.test_dynamic_typename -v +``` + +The first successful Linux training pilot used an existing broader temporary +workaround. That pilot is not evidence for this narrower production change; the +follow-up below uses the exact signed fix. This is a compatibility fix, with no +performance improvement claim. + +## Matching Linux runtime follow-up + +The exact signed utility change was also tested on Python 3.11.4 and PyTorch +2.16.0.dev20261012+cpu. All seven type-name regressions passed within a **29-test +integration suite** (also covering trace integrity, TorchScript, and precondition +state isolation). A separate 11-test checker-observer suite passed. The real +`OpsWrapper` returned `torch._inductor.virtualized.OpsWrapper` in both modes. + +The disposable integration source combined the signed tracing, TorchScript, +cache, sampling, state-isolation, and type-name changes. Its `utils.py` matched +commit `2f034ae5d0fab7e792b7c0a5bdc6a1acfeb8962c` exactly, and all source hashes +were unchanged after testing. Compact evidence includes runtime versions and +log/source hashes. These preflight results do not establish training overhead +or cross-workload invariant generalization. + +The composed source also completed a fresh **10-step MNIST trace collection** +with cached real MNIST, batch size 2, CPU float32, and seed 42. Its source hashes +and model-state shapes stayed unchanged. This supports collection compatibility +in the identified stack; it does not isolate training behavior to this one fix. +The study later stopped at a shape-only negative-control selector because no +shape invariant existed, after collection and healthy checks had succeeded. +That harness issue and its preserved evidence are separate from the type-name +fix. Evidence hashes for the successful collection are included in the JSON. + +## Contract hardening (2026-10-25) + +`typename` now declares its existing `object`/`bool` inputs and string result. +The dynamic-name guards, reported-class fallback, and final string assertions +are unchanged. All dynamic-name test helpers now have explicit annotations. +Two additional tests preserve the negative contract: an invalid fallback class +name still fails the final assertion, and an unexpected metadata exception +propagates unchanged. This does not change `safe_getattr`'s pre-existing framework +compatibility exception handling. + +The composed branch includes the signed #142 and #143 hardening checkpoints. +Validation uses Python 3.10.16/PyTorch 2.2.2 with one OpenMP/MKL/OpenBLAS thread: + +```sh +python -m unittest discover -s tests -p 'test_dynamic_typename.py' +python -m unittest discover -s tests -p 'test_torchscript*.py' +python -m unittest discover -s tests -p 'test_trac*.py' +pre-commit run --files traincheck/utils.py tests/test_dynamic_typename.py traincheck/instrumentor/tracer.py traincheck/instrumentor/dumper.py traincheck/instrumentor/torchscript_compat.py tests/test_torchscript_compat.py tests/test_torchscript_contracts.py tests/test_tracing_contracts.py +mypy --follow-imports=silent --ignore-missing-imports --check-untyped-defs --disallow-untyped-defs --disallow-any-generics --warn-return-any traincheck/instrumentor/torchscript_compat.py tests/test_dynamic_typename.py tests/test_torchscript_contracts.py tests/test_tracing_contracts.py +``` + +All 37 tests passed: 9 dynamic-name, 13 TorchScript (including six fresh-process +controls), and 15 tracing tests. All pinned hooks and the stronger scoped check +passed using mypy 1.9. The existing general utility/serialization modules are +not claimed to be strictly typed in full; see the #143 evidence for the remaining +legacy tracer/dumper diagnostics. No runtime benchmark was rerun for this +annotation/contract update, so earlier performance numbers retain their original +revision and workload scope. + +With `--check-untyped-defs`, `utils.py` reports the same single pre-existing +`threading.__excepthook__` stub diagnostic at both original head +`2333682cb2f3f48d338ff27a6d70682f0f4e4702` and this update; no new diagnostic +was introduced by the `typename` annotation. diff --git a/docs/experiments/reservoir-cap-validation.json b/docs/experiments/reservoir-cap-validation.json new file mode 100644 index 00000000..b777c692 --- /dev/null +++ b/docs/experiments/reservoir-cap-validation.json @@ -0,0 +1,262 @@ +{ + "baseline_commit": "9eb4f320d3db34d372145261704b805b88e39bda", + "checkpoint_verified_date": "2026-10-25", + "combined_changed_production_source_sha256": { + "traincheck/infer_engine.py": "476a3ec99bef0ab823b23f94bc95842c20b6d9b7b4a57ab4a2efa2987f30f286", + "traincheck/invariant/base_cls.py": "2a2092626065dd871549dd7c969f085c4bc38fe5fffd92469d2ed775bda084ae", + "traincheck/invariant/contain_relation.py": "b32f819db4d5a11a11318bf7e38c64148f4c9360137ac5f09475b480c34c16a5", + "traincheck/parallel.py": "6843319dd8c5c397e93f361a29fe564161e7307fbe4f0f8368090d252d4a7bc4" + }, + "combined_sampling_pr": 139, + "combined_source_manifest_sha256": "60f431cd032532f6b3521169c29ce3ed3943c30fb5b55449ef7ebb608d6cc811", + "combined_test_source_sha256": { + "tests/test_example_reservoir.py": "0381a81e714980f5cff16699db951d984c4f42cd95b6fa47b1611d4d38c2f636", + "tests/test_inference_sampling.py": "046554588613da74e5dbad524acd9ca9ed7eee9c96611d1341eab3cc6cd16e8e" + }, + "controlled_probe": { + "after": { + "cap": 2, + "constructed_num_seen": 5, + "constructed_retained": 2, + "merged_num_seen": 20, + "merged_retained": 2 + }, + "before": { + "cap": 2, + "constructed_num_seen": 5, + "constructed_retained": 5, + "merged_num_seen": 10, + "merged_retained": 4 + } + }, + "limits": [ + "Per-occurrence marginal inclusion only for correlated source reservoirs; no joint uniformity claim.", + "Optional containment generalization is skipped for already-sampled inputs; generalized hypotheses can change above the cap.", + "Retained reservoirs are bounded; temporary unions and total hypothesis/process memory are not.", + "Saved raw benchmark traces are external to the checked-in regression tests." + ], + "recorded_date": "2026-10-13", + "regression_tests": { + "reservoir_standalone_passed": 21, + "reservoir_with_sampling_passed": 21, + "sampling_with_reservoir_passed": 11, + "start_methods": [ + "fork", + "spawn" + ], + "worker_counts": [ + 1, + 2, + 4 + ] + }, + "replay": { + "comparison": "Full structural JSON multiset; preserves multiplicity and array order", + "complete_runs": 9, + "disabled_relations": [ + "FunctionLeadRelation", + "FunctionCoverRelation" + ], + "input_hashes_unchanged": true, + "native_threads": 1, + "repeats_per_workload_and_workers": 1, + "runs": [ + { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "8e39cf710d943898ca007478953584d363db135ce3cb458619e71836ae179820", + "distinct_records": 21, + "records": 21 + }, + "invariants": { + "canonical_multiset_sha256": "5691e461a1fba5bf5314b18a62bf72f82fc95e56f71b26eca87218e53f321e84", + "distinct_records": 193, + "records": 193 + }, + "matches_sampling_only": true, + "workers": 1, + "workload": "mnist" + }, + { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "8e39cf710d943898ca007478953584d363db135ce3cb458619e71836ae179820", + "distinct_records": 21, + "records": 21 + }, + "invariants": { + "canonical_multiset_sha256": "5691e461a1fba5bf5314b18a62bf72f82fc95e56f71b26eca87218e53f321e84", + "distinct_records": 193, + "records": 193 + }, + "matches_sampling_only": true, + "workers": 2, + "workload": "mnist" + }, + { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "8e39cf710d943898ca007478953584d363db135ce3cb458619e71836ae179820", + "distinct_records": 21, + "records": 21 + }, + "invariants": { + "canonical_multiset_sha256": "5691e461a1fba5bf5314b18a62bf72f82fc95e56f71b26eca87218e53f321e84", + "distinct_records": 193, + "records": 193 + }, + "matches_sampling_only": true, + "workers": 4, + "workload": "mnist" + }, + { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "b3424febf9358e01c3e079362e17bc68a939a933e14f1cfab954884482073bd7", + "distinct_records": 10, + "records": 10 + }, + "invariants": { + "canonical_multiset_sha256": "21332736553d2fe20c1fdb1e2c852597950e3b799bc399d47007a2b1f7a71b09", + "distinct_records": 518, + "records": 518 + }, + "matches_sampling_only": true, + "workers": 1, + "workload": "transformer" + }, + { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "b3424febf9358e01c3e079362e17bc68a939a933e14f1cfab954884482073bd7", + "distinct_records": 10, + "records": 10 + }, + "invariants": { + "canonical_multiset_sha256": "21332736553d2fe20c1fdb1e2c852597950e3b799bc399d47007a2b1f7a71b09", + "distinct_records": 518, + "records": 518 + }, + "matches_sampling_only": true, + "workers": 2, + "workload": "transformer" + }, + { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "b3424febf9358e01c3e079362e17bc68a939a933e14f1cfab954884482073bd7", + "distinct_records": 10, + "records": 10 + }, + "invariants": { + "canonical_multiset_sha256": "21332736553d2fe20c1fdb1e2c852597950e3b799bc399d47007a2b1f7a71b09", + "distinct_records": 518, + "records": 518 + }, + "matches_sampling_only": true, + "workers": 4, + "workload": "transformer" + }, + { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "dfba6a7ed565ce66163b9eb98921ad05857649e37f78075b48fdf9341b912204", + "distinct_records": 86, + "records": 86 + }, + "invariants": { + "canonical_multiset_sha256": "596f82fcfb946a7fe2fe138fa9e61302bafb93465fdc529317ccf93d2da1022a", + "distinct_records": 512, + "records": 512 + }, + "matches_sampling_only": true, + "workers": 1, + "workload": "dcgan" + }, + { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "dfba6a7ed565ce66163b9eb98921ad05857649e37f78075b48fdf9341b912204", + "distinct_records": 86, + "records": 86 + }, + "invariants": { + "canonical_multiset_sha256": "596f82fcfb946a7fe2fe138fa9e61302bafb93465fdc529317ccf93d2da1022a", + "distinct_records": 512, + "records": 512 + }, + "matches_sampling_only": true, + "workers": 2, + "workload": "dcgan" + }, + { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "dfba6a7ed565ce66163b9eb98921ad05857649e37f78075b48fdf9341b912204", + "distinct_records": 86, + "records": 86 + }, + "invariants": { + "canonical_multiset_sha256": "596f82fcfb946a7fe2fe138fa9e61302bafb93465fdc529317ccf93d2da1022a", + "distinct_records": 512, + "records": 512 + }, + "matches_sampling_only": true, + "workers": 4, + "workload": "dcgan" + } + ], + "source_hashes_unchanged": true + }, + "runtime": { + "packages": { + "numpy": "1.26.4", + "pandas": "2.3.3", + "torch": "2.2.2+cu121" + }, + "platform": "Linux", + "python": "3.10.16" + }, + "saved_evidence_sha256": { + "baseline-probe.json": "69d232dbeae11972ddbb9d27abb153d5f88dd0e2025417c041800583740d9289", + "cap-source-manifest.json": "6910fabc4a993976f9cf90cd98d68d2797e70ff306fb5ff9d4052fab2fb2f8b1", + "cap-tests.log": "7f512ee57813789d5bb907cdccbceaeea7782c95cf95b7dac62fccf9448e3a82", + "combined-reservoir-tests.log": "6b0f38d6f17bafb808ba505c604ff60306a6362452233d22b6c77700a0210dbd", + "combined-sampling-tests.log": "dac8a7daaff443800210eeecbadfa8b257e9cbbe7ccd30859f8951c7c0a9246c", + "combined-source-manifest.json": "60f431cd032532f6b3521169c29ce3ed3943c30fb5b55449ef7ebb608d6cc811", + "fixed-probe.json": "72161a6b9fe85d4b0df03f9ae4b0921bbe27def313da4ffa738fef10ca8a7304", + "replay/summary.json": "aa081af685c92f9e62c3ca7f72b9082cd2ed61af7c4f1a37b543eb8e709d914c", + "sampling-only-comparison.json": "e526d5cc2c9f8fa37c2aeef21a05e9481338dc142789f27edff129d3121d0a15" + }, + "schema_version": 1, + "standalone_source_sha256": { + "tests/test_example_reservoir.py": "0381a81e714980f5cff16699db951d984c4f42cd95b6fa47b1611d4d38c2f636", + "traincheck/infer_engine.py": "741ac7701ad4fef3089fce778ddd4d422c5245432cf582bec06095e0ed6ec71c", + "traincheck/invariant/base_cls.py": "2a2092626065dd871549dd7c969f085c4bc38fe5fffd92469d2ed775bda084ae", + "traincheck/invariant/contain_relation.py": "67a6613f0048542174f6655fcb1f984fabf5c5f2fa9303eab13aed93103b410d", + "traincheck/parallel.py": "6843319dd8c5c397e93f361a29fe564161e7307fbe4f0f8368090d252d4a7bc4" + }, + "trace_inputs": { + "dcgan": { + "file_count": 1, + "file_sha256": [ + "da74105c67ff699b91d96eac1fac89f3626ba9199594c9c373eb1ff6a69b15d1" + ], + "total_bytes": 3401128 + }, + "mnist": { + "file_count": 1, + "file_sha256": [ + "6e3dd86b5ed0d7b5d556a167eaf95226852dd816a33f872668fbaf1b26638c20" + ], + "total_bytes": 472863 + }, + "transformer": { + "file_count": 1, + "file_sha256": [ + "467e587463e72e874f5abfd14452cabdf966fd804f448eecf85f0155ea49ffb6" + ], + "total_bytes": 1564970 + } + } +} diff --git a/docs/experiments/reservoir-cap-validation.md b/docs/experiments/reservoir-cap-validation.md new file mode 100644 index 00000000..4e71de54 --- /dev/null +++ b/docs/experiments/reservoir-cap-validation.md @@ -0,0 +1,46 @@ +# Example reservoir cap validation + +Iterable construction and stage/cross-trace merging previously appended retained examples directly. This bypassed `MAX_EXAMPLE_LIST_SIZE` and left the represented population count stale. The fix uses bounded insertion and merges according to the original stream populations, preserving duplicate occurrences and exact concatenation order when complete inputs fit. + +Results below were recorded on 2026-10-13. The pending source and saved evidence hashes were independently rechecked on 2026-10-25. The baseline is `9eb4f320d3db34d372145261704b805b88e39bda`; combined validation includes the deterministic sampling change in #139. + +## Controlled fixture + +The fixture sets capacity to 2, merges two streams containing ten occurrences each, and separately constructs a reservoir from five occurrences. + +| Measurement | Before | After | +|---|---:|---:| +| Retained after merge | 4 | 2 | +| `num_seen` after merge | 10 | 20 | +| Retained from five-item iterable | 5 | 2 | +| `num_seen` for iterable | 5 | 5 | + +These measure reservoir correctness. They do not measure whole-process memory savings. + +## Regression and replay checks + +Run the portable regression suite from the repository root: + +```bash +python -m unittest discover -s tests -p test_example_reservoir.py -v +``` + +All 21 reservoir tests passed on Linux with Python 3.10.16, PyTorch 2.2.2+cu121, pandas 2.3.3, and NumPy 1.26.4. Coverage includes cap/count accounting, duplicates, atomic rejection, continuation RNG state, stage/cross-trace call sites, actual inference with 1/2/4 workers under fork and spawn, and fixed-seed marginal-inclusion sweeps. With #139, all 21 reservoir and 11 sampling tests passed; its sampling fixture explicitly disables the cap to exercise precondition downsampling separately. + +Nine complete saved-trace inference replays used one pass per workload and worker count, with native thread counts set to 1 and `FunctionLeadRelation`/`FunctionCoverRelation` disabled. Every invariant and failed-hypothesis JSON multiset matched across workers and against sampling-only outputs; multiplicity and array order were retained in comparisons. + +| Workload | Workers tested | Invariants | Failed hypotheses | +|---|---|---:|---:| +| MNIST | 1, 2, 4 | 193 | 21 | +| Small transformer | 1, 2, 4 | 518 | 10 | +| DCGAN | 1, 2, 4 | 512 | 86 | + +Input and source hashes remained unchanged. Compact counts, canonical output hashes, and source/evidence hashes are recorded in [reservoir-cap-validation.json](reservoir-cap-validation.json). Its combined-source manifest and test hashes identify the validated combination with #139, including the uncapped sampling fixture. Raw saved traces are external to this note; the synthetic regression suite runs directly from the checkout. + +Publication validation on 2026-10-25 reran all 32 combined tests successfully on macOS. Production Python ASTs match the previously validated combined source; publication changes are formatting and narrow lint comments for intentional exact-integer checks. + +## Scope + +The merge RNG must advance independently of source selection and must not reset across nested merges. Existing equal-seed source reservoirs can still be correlated: the guarantee is marginal occurrence inclusion, not a uniform joint sample across correlated reservoirs. + +Optional containment generalization is skipped when an input has already been sampled, because discarded set overlap cannot be reconstructed. Complete inputs retain exact set deduplication before capping the known union. Consequently, generalized hypotheses can change above the cap. Temporary unions, total hypothesis counts, and total process memory are outside this per-reservoir bound. diff --git a/docs/experiments/torchscript-builtin-validation.json b/docs/experiments/torchscript-builtin-validation.json new file mode 100644 index 00000000..13020e1c --- /dev/null +++ b/docs/experiments/torchscript-builtin-validation.json @@ -0,0 +1,208 @@ +{ + "historical_cpu_study": { + "linux": { + "controls": [ + { + "eager_output_and_gradients_equal_baseline": true, + "eager_records": 0, + "exception_type": null, + "graph_has_aten_tanh": true, + "mode": "baseline", + "registered_operator": "aten::tanh", + "script_compilation": "passed", + "scripted_output_and_gradients_equal_baseline": true, + "scripted_records": 0 + }, + { + "eager_output_and_gradients_equal_baseline": true, + "eager_records": 2, + "exception_type": "TypeError", + "graph_has_aten_tanh": false, + "mode": "buggy-unprimed", + "registered_operator": null, + "script_compilation": "failed", + "scripted_output_and_gradients_equal_baseline": null, + "scripted_records": null + }, + { + "eager_output_and_gradients_equal_baseline": true, + "eager_records": 2, + "exception_type": "TypeError", + "graph_has_aten_tanh": false, + "mode": "buggy-warm", + "registered_operator": null, + "script_compilation": "failed", + "scripted_output_and_gradients_equal_baseline": null, + "scripted_records": null + }, + { + "eager_output_and_gradients_equal_baseline": true, + "eager_records": 2, + "exception_type": null, + "graph_has_aten_tanh": true, + "mode": "fixed-unprimed", + "registered_operator": "aten::tanh", + "script_compilation": "passed", + "scripted_output_and_gradients_equal_baseline": true, + "scripted_records": 0 + }, + { + "eager_output_and_gradients_equal_baseline": true, + "eager_records": 2, + "exception_type": null, + "graph_has_aten_tanh": true, + "mode": "fixed-warm", + "registered_operator": "aten::tanh", + "script_compilation": "passed", + "scripted_output_and_gradients_equal_baseline": true, + "scripted_records": 0 + }, + { + "eager_output_and_gradients_equal_baseline": true, + "eager_records": 2, + "exception_type": null, + "graph_has_aten_tanh": true, + "mode": "integrated-fixed-warm", + "registered_operator": "aten::tanh", + "script_compilation": "passed", + "scripted_output_and_gradients_equal_baseline": true, + "scripted_records": 0 + } + ], + "device": "cpu", + "helper_test_log_sha256": "feabb4add9e2ca3374e538560634d1ec97dc4f4ce89aab12f3d726e36ea9b2f6", + "helper_unit_tests_passed": 4, + "native_threads": 1, + "python": "3.10.16", + "registry_already_initialized_after_import": true, + "semantic_trace_checks_passed": 13, + "summary_sha256": "68c1888b7df4a9a0248e6d218ea5e82d2daeb90c3efcf50b852faaf7e3bfa7c2", + "torch": "2.2.2+cu121" + }, + "macos": { + "controls": [ + { + "eager_output_and_gradients_equal_baseline": true, + "eager_records": 0, + "exception_type": null, + "graph_has_aten_tanh": true, + "mode": "baseline", + "registered_operator": "aten::tanh", + "script_compilation": "passed", + "scripted_output_and_gradients_equal_baseline": true, + "scripted_records": 0 + }, + { + "eager_output_and_gradients_equal_baseline": true, + "eager_records": 2, + "exception_type": "TypeError", + "graph_has_aten_tanh": false, + "mode": "buggy-unprimed", + "registered_operator": null, + "script_compilation": "failed", + "scripted_output_and_gradients_equal_baseline": null, + "scripted_records": null + }, + { + "eager_output_and_gradients_equal_baseline": true, + "eager_records": 2, + "exception_type": "TypeError", + "graph_has_aten_tanh": false, + "mode": "buggy-warm", + "registered_operator": null, + "script_compilation": "failed", + "scripted_output_and_gradients_equal_baseline": null, + "scripted_records": null + }, + { + "eager_output_and_gradients_equal_baseline": true, + "eager_records": 2, + "exception_type": null, + "graph_has_aten_tanh": true, + "mode": "fixed-unprimed", + "registered_operator": "aten::tanh", + "script_compilation": "passed", + "scripted_output_and_gradients_equal_baseline": true, + "scripted_records": 0 + }, + { + "eager_output_and_gradients_equal_baseline": true, + "eager_records": 2, + "exception_type": null, + "graph_has_aten_tanh": true, + "mode": "fixed-warm", + "registered_operator": "aten::tanh", + "script_compilation": "passed", + "scripted_output_and_gradients_equal_baseline": true, + "scripted_records": 0 + }, + { + "eager_output_and_gradients_equal_baseline": true, + "eager_records": 2, + "exception_type": null, + "graph_has_aten_tanh": true, + "mode": "integrated-fixed-warm", + "registered_operator": "aten::tanh", + "script_compilation": "passed", + "scripted_output_and_gradients_equal_baseline": true, + "scripted_records": 0 + } + ], + "device": "cpu", + "helper_test_log_sha256": "d094b1bd2c8318c96097f80c50da282add5b2972970a77046de8db6c8935f6ca", + "helper_unit_tests_passed": 4, + "native_threads": 1, + "python": "3.10.16", + "registry_already_initialized_after_import": true, + "semantic_trace_checks_passed": 13, + "summary_sha256": "4e8775e72f31a344f6e82f59f6b3009fddebc9f71c9463f6f5a009b5885d8c43", + "torch": "2.2.2" + } + }, + "historical_source_sha256": { + "bias_gelu_fixture.py": "c6d7503996e5fa2872114aa7ba8b3b37151f03e8f7cc2302a76e6c451fb70059", + "builtin_compat_candidate.py": "744506a0c035396aea443f39394a700ef7908bc7d36b8117ba76fae97c92e845", + "candidate-src/traincheck/instrumentor/torchscript_compat.py": "744506a0c035396aea443f39394a700ef7908bc7d36b8117ba76fae97c92e845", + "candidate-src/traincheck/instrumentor/tracer.py": "a8e39a9dea2c4940543174c2bca97146665f0f08068bdf350b3f16802d4e79bd", + "probe.py": "58b0ce8f8a1fb65b96cd5a756bb82ef5ff8d315b3673081ea58d49ebffb0b29c", + "run.py": "f4c549ea1dd21c47f4e98f35df4126b23895391eacb2f89c182aaf7f2ccf8d12", + "test_candidate.py": "6f5fbbab5053e2303a53209ec25d71c1c240e9c06b9038af844adf762f636eff", + "torchscript-builtin-identity.patch": "8ae809f16bf6464d5a152d9612e37bd81edf04ef6680fee7e3042b42c8f026a5" + }, + "limits": [ + "Registry was already initialized after imports; no genuinely cold-registry result is claimed.", + "Scripted operators bypass Python recorder hooks; successful graphs emit no inner tanh API records.", + "Registered wrappers remain alive for the process lifetime; repeated creation retains additional objects.", + "Full Megatron startup, checkpoint-shape compatibility, and other PyTorch versions/operators are not established.", + "The converted portable checkout test ran locally on macOS; Linux evidence comes from the prior equivalent focused study." + ], + "linux_saved_evidence_archive_sha256": "81f5ede08024c3e099ad61b7489ffa420e4da490c743cded9d9236fd6453a339", + "portable_checkout_validation": { + "fresh_probe_processes": 6, + "fresh_process_regression_methods": 1, + "helper_test_methods": 4, + "log_sha256": "4e23c888ea4641c17ac45c1a413ba0a9b74afbbda23c87e364458b283a4b2438", + "platform": "macOS", + "python": "3.10.16", + "semantic_trace_checks": 13, + "source_includes_trace_integrity_fix": true, + "test_methods_passed": 5, + "tested_base_commit": "7d6c683db92d83ffd6db9fc9c11fbc0ebdd2c5d4", + "tested_base_pr": 142, + "timeout_seconds_per_probe": 90, + "torch": "2.2.2" + }, + "recorded_date": "2026-10-25", + "schema_version": 1, + "source": { + "base_dumper_sha256": "dd813a342b4531a6543995a23591f558db00d04142611a2d9656b1258833fac3", + "base_tracer_sha256": "bce6546cfa13a263150dff891f2f8d7d4af8c5ccb28dc2faedcedc691c5857da", + "candidate_dumper_sha256": "dd813a342b4531a6543995a23591f558db00d04142611a2d9656b1258833fac3", + "candidate_tracer_sha256": "ad57320f0a3764c02c73cf4bfa623f1ef73b9835dfa6cb1365dc2988551de428", + "helper_implementation_ast_unchanged_except_docstring": true, + "helper_sha256": "0d0c348fa02d5f93bb74e874829e5fb4c57365b82ce973ae8ff1f21a42c2d64e", + "integration_patch_sha256": "b374501d22408ebace2d1e2e69c03d2a98abcf3ebfa2e4e056b62adf27874397", + "portable_test_sha256": "653be1032409579030ebd3ec5694cc3e43f34df8e4f5750a76a9ffc64dd0a2e2", + "validated_helper_before_docstring_sha256": "744506a0c035396aea443f39394a700ef7908bc7d36b8117ba76fae97c92e845" + } +} diff --git a/docs/experiments/torchscript-builtin-validation.md b/docs/experiments/torchscript-builtin-validation.md new file mode 100644 index 00000000..b87e8442 --- /dev/null +++ b/docs/experiments/torchscript-builtin-validation.md @@ -0,0 +1,79 @@ +# TorchScript builtin identity validation + +TorchScript records builtin operators by function ID. Replacing `torch.tanh` with a TrainCheck wrapper loses the `aten::tanh` lookup. TorchScript attempts to inspect Python source, follows the wrapper metadata to the C builtin, and raises `TypeError: ... expected, got builtin_function_or_method`. + +The compatibility helper looks up the original mapping before the wrapper is published and registers the wrapper with that same operator. It guesses no operator names. Missing private APIs and unregistered Python functions are unchanged. Registered wrappers are retained because the registry stores only IDs; otherwise a later unrelated callable could reuse a stale ID. + +## Controls and results + +The 2026-10-25 CPU experiment uses a real Python source file containing a representative tanh-based bias-GELU expression. Each control runs in a fresh interpreter, with a 90-second timeout and one native/inter-op thread. + +| Control | Compilation | Eager tanh records | Scripted operator | +|---|---|---:|---| +| Unwrapped baseline | Pass | 0 | `aten::tanh` | +| Wrapper without compatibility, no explicit priming | TypeError | 2, one pair | Unregistered wrapper | +| Wrapper without compatibility, explicitly primed | TypeError | 2, one pair | Unregistered wrapper | +| Wrapper plus helper, no explicit priming | Pass | 2, one pair | `aten::tanh` | +| Wrapper plus helper, explicitly primed | Pass | 2, one pair | `aten::tanh` | +| Helper integrated into the actual wrapper | Pass | 2, one pair | `aten::tanh` | + +The original study passed all 13 semantic/trace checks and four helper unit tests on macOS with PyTorch 2.2.2 and Linux with PyTorch 2.2.2+cu121; both used Python 3.10.16 and CPU tensors. Candidate eager outputs and both gradients exactly matched the unwrapped baseline within each platform. Successful scripted outputs and both gradients also exactly matched that platform's scripted baseline. Every eager pair retained arguments, return snapshots, and a shared call ID. + +The portable checkout test was additionally run on macOS against the trace-integrity source plus this integration: all five test methods passed, comprising the four helper contracts and six fresh-process controls with the same 13 semantic/trace checks. The helper implementation is unchanged from the Linux-tested version apart from its module docstring and formatting. Linux's earlier study is separate evidence; the converted checkout test was not rerun there. + +Run the portable tests from the repository root: + +```bash +python -m unittest discover -s tests -p test_torchscript_compat.py -v +``` + +Compact source hashes, control outcomes, and validation details are recorded in [torchscript-builtin-validation.json](torchscript-builtin-validation.json). + +## Limits + +TrainCheck imports had already initialized the registry in both tested environments. “No explicit priming” therefore does not prove a genuinely uninitialized registry path. + +Scripted graphs call `aten::tanh` directly and produce no inner Python tanh records. Eager tracing remains intact; complete compiled-graph tracing is outside this change. Full instrument-before-import Megatron startup and checkpoint-shape compatibility were not validated by this focused fixture. + +Registered wrappers remain alive for the process lifetime; repeatedly creating new wrappers retains additional objects. The helper depends on feature-detected private APIs and does not establish compatibility with every future PyTorch version or wrapped operator. + +## Contract hardening (2026-10-25) + +Runtime checkpoint `d4ebb12e3c5fd7531080d5623447f2af0a1ab3e9` replaces the +untyped cache and status dictionaries with a typed retention registry and a +required, discriminated result: `preserved` has a nonempty string operator, +`not-registered` has no mapping, and `unsupported-private-api` reports missing +private support. Present noncallable callbacks and nonstring/empty operator +metadata raise `TypeError` before registration; lookup/registration exceptions +propagate unchanged. Failed registration does not retain the wrapper. + +The wrapper publishes `_traincheck_torchscript_builtin` metadata and warns once +per process when private support is unavailable, including under an `always` +warning filter and concurrent callers. Unsupported support still permits eager +instrumentation. Its factory and nested callables now have explicit annotations; +`var_dump_config=None` is passed unchanged to the existing registry. Disabled +tracing passes explicit false flags for its unused tracing options. + +Validation used Python 3.10.16/PyTorch 2.2.2, one OpenMP/MKL/OpenBLAS thread, +and the repository-pinned hooks (including mypy 1.9): + +```sh +python -m unittest discover -s tests -p 'test_torchscript*.py' +python -m unittest discover -s tests -p 'test_trac*.py' +pre-commit run --files traincheck/instrumentor/tracer.py traincheck/instrumentor/torchscript_compat.py tests/test_torchscript_compat.py tests/test_torchscript_contracts.py +mypy --follow-imports=silent --ignore-missing-imports --check-untyped-defs --disallow-untyped-defs --disallow-any-generics --warn-return-any traincheck/instrumentor/torchscript_compat.py tests/test_torchscript_contracts.py tests/test_tracing_contracts.py +``` + +All 13 TorchScript tests passed, including 8 new negative/capability contracts and +the existing six fresh-process controls for outputs, gradients, compiled graphs, +and eager records. All 15 tracing tests and all pinned hooks passed. The stronger +check above passed without adding `Any`, casts, or ignore directives. Callable +ellipsis parameters describe arbitrary framework functions retained or passed +through without invocation by the compatibility module. + +A separate `--check-untyped-defs` check of the legacy tracer/dumper files found +7 errors at original head `9ac61d1140cf8679c24442a174ec12f4de069980` and +4 after hardening: existing logger-key type inference, two unannotated queue +locals, and owner-class traversal assignment. Legacy serialization/proxy helpers +remain partially annotated; this is not a repository-wide strictness claim. +No timing or benchmark improvement is claimed for this contract-only change. diff --git a/docs/experiments/trace-integrity-results.json b/docs/experiments/trace-integrity-results.json new file mode 100644 index 00000000..d44c4508 --- /dev/null +++ b/docs/experiments/trace-integrity-results.json @@ -0,0 +1,125 @@ +{ + "measurement_date": "2026-10-13", + "traincheck_base": "9eb4f320d3db34d372145261704b805b88e39bda", + "benchmark_source": "ae6601978217bc0f06b9a78479cf00f0bc7242c9", + "controlled_reentry": { + "buggy": { + "application_calls": 100, + "events": 400, + "metadata_helper_events": 200, + "application_results_match": true, + "recursion_errors": 0, + "scope": "Controlled injected conversion helper, not a natural workload frequency" + }, + "fixed": { + "application_calls": 100, + "events": 200, + "metadata_helper_events": 0, + "application_results_match": true, + "recursion_errors": 0, + "scope": "Controlled injected conversion helper, not a natural workload frequency" + } + }, + "controlled_buffer_interleavings": { + "scope": "Controlled forced interleavings, not a natural workload frequency or performance benchmark", + "trials_per_mode": 50, + "records_preloaded_per_trial": 100, + "results": { + "buggy": { + "duplicate": { + "records": 10000, + "duplicates": 5000, + "missing": 0 + }, + "loss": { + "records": 5000, + "duplicates": 0, + "missing": 50 + } + }, + "fixed": { + "duplicate": { + "records": 5000, + "duplicates": 0, + "missing": 0 + }, + "loss": { + "records": 5050, + "duplicates": 0, + "missing": 0 + } + } + } + }, + "natural_workload_matrix": { + "runs_completed": 45, + "instrumented_runs": 36, + "conditions": [ + "baseline", + "buggy", + "guard", + "buffer", + "fixed" + ], + "seeds": [ + 42, + 43, + 44 + ], + "training_steps": 5, + "batch_size": 2, + "native_threads": 1, + "runtime": { + "python": "3.10.16", + "torch": "2.2.2+cu121", + "gpu": "RTX A2000 12 GB" + }, + "scope": "Adapted small model workloads with ordinary local tensors; not full published benchmark reproductions. All instrumented variants share the same dynamic-type-name compatibility patch.", + "instrumentation": "Default API coverage, use_full_instr=False, sampling interval one, no model/proxy tracking; training bodies only.", + "workloads": [ + { + "name": "mnist", + "parameters": 1199882, + "instrumented_runs": 12, + "records_per_run": 654, + "raw_bytes_per_run": 472863, + "instrumented_variants_and_seeds_equal": true + }, + { + "name": "transformer", + "parameters": 23522, + "instrumented_runs": 12, + "records_per_run": 2006, + "raw_bytes_per_run": 1564970, + "instrumented_variants_and_seeds_equal": true + }, + { + "name": "dcgan", + "parameters": 190672, + "instrumented_runs": 12, + "records_per_run": 4596, + "raw_bytes_per_run": 3401128, + "instrumented_variants_and_seeds_equal": true + } + ], + "total_instrumented_records": 87072, + "total_instrumented_raw_bytes": 65267532, + "all_recorded_losses_and_final_model_state_hashes_match_baseline": true, + "observed_duplicate_records": 0, + "observed_recursion_error_events": 0, + "natural_trace_size_reduction_observed": false + }, + "evidence_artifact_sha256": { + "matrix.csv": "f196211cc478ec532338de06247895cfa5bcd379770fd66583c5b80189127cfe", + "matrix-summary.json": "a6a18173e3a0550500160b0e0b3169553ee714bb0ce175048cc5998f19c2ba06", + "controlled-races.json": "6b0ab42d2d8f1ff37557627bd8b261610f7d84adf65f88e2025739dfd875c4aa", + "finite-reentry-buggy.json": "25f5dfeab0f68c080682f21f0fd6e0314e7c4254a2a9e538b9081aad3a382475", + "finite-reentry-fixed.json": "18470fb8d6c929296feecf3485242775dd63331e89c0669244d7ed763bdf6595" + }, + "limitations": [ + "Injected helpers and forced interleavings establish possible failure severity, not natural occurrence rates.", + "All 45 ordinary-tensor runs completed; none of the 36 instrumented runs showed trace inflation or a trace-size reduction from these fixes.", + "No general performance improvement or invariant-checking overhead claim is made.", + "Historical incomplete and completed TorchTitan traces are not comparable percentage-size-reduction measurements." + ] +} diff --git a/docs/experiments/trace-integrity.md b/docs/experiments/trace-integrity.md new file mode 100644 index 00000000..4cdff2aa --- /dev/null +++ b/docs/experiments/trace-integrity.md @@ -0,0 +1,68 @@ +# Recorder re-entry and buffer integrity + +Recorder-owned argument conversion, tensor metadata inspection, and JSON encoding can call instrumented APIs again. The wrapper previously did not honor the conversion helper's local suppression flag, so recorder work could generate more records or recurse. Independently, the producer and periodic flusher accessed the same buffer without synchronization: overlapping flushes could duplicate a batch, and a producer append could be cleared before delivery. + +The recorder now uses an exception-safe, thread-local suppression depth around its own work. It releases suppression before executing application code, preserving nested application calls and generator execution. A class context manager preserves frozen framework exceptions. The buffer snapshots mutable values synchronously, then locks append, queue handoff, and clear as one ordered operation. Buffers still belong to individual producer threads for trace routing. + +## Controlled impact + +These tests deliberately inject a conversion helper or force a thread interleaving; they do not estimate how often an ordinary training job triggers either defect. + +| Controlled trigger | Buggy | Fixed | +|---|---:|---:| +| 100 application calls with a wrapped serialization helper | 400 records, including 200 recorder-owned records | 200 records, zero recorder-owned records | +| Two overlapping flushes; 50 trials × 100 records | 10,000 emitted, including 5,000 duplicates | 5,000 emitted, zero duplicates | +| Append during paused flush; 50 trials, 5,050 expected records | 50 records missing | Zero records missing | + +The finite re-entry case preserves application return values and raises no `RecursionError`: self-tracing can inflate a trace without an obvious recursion failure. The buffer race can cause loss as well as duplication; valid pre/post pairing alone cannot detect a completely missing pair. + +## Ordinary workload control + +All 45 runs completed: MNIST CNN, a small Transformer, and reduced-width DCGAN × five conditions × seeds 42/43/44. Each run used five training steps, batch size two, one GPU, and one native CPU thread. Conditions were no instrumentation, the buggy snapshot, guard only, buffer lock only, and both fixes. + +None of the 36 instrumented runs triggered either defect. Each instrumented condition had the same record count and raw byte size at every seed: + +| Workload | Records per run | Raw bytes per run | Buggy → fixed size change | +|---|---:|---:|---:| +| MNIST CNN | 654 | 472,863 | 0% | +| Small Transformer | 2,006 | 1,564,970 | 0% | +| DCGAN | 4,596 | 3,401,128 | 0% | + +All recorded losses and canonical final model-state hashes matched their corresponding uninstrumented baselines. These are adapted micro-workloads with ordinary local tensors on PyTorch 2.2.2+cu121, not full published benchmark reproductions or DTensor coverage. The observations support these correctness repairs; they do not establish a general performance improvement or invariant-checking overhead result. + +## Validation and provenance + +Run the eleven portable regression tests from the checkout: + +```sh +OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 python -m unittest tests.test_trace_integrity -v +``` + +The suite covers recorder re-entry, JSON emission re-entry, real tensor snapshots, nested application calls, generators, original and frozen exceptions, thread-local suppression, overlapping flushes, and concurrent producer ordering. + +[Sanitized results](trace-integrity-results.json) retain controlled counts, workload settings, base/source revisions, and evidence-file hashes. Measurements used TrainCheck base `9eb4f320d3db34d372145261704b805b88e39bda` and adapted benchmark source `ae6601978217bc0f06b9a78479cf00f0bc7242c9`; all instrumented conditions shared the same dynamic-type-name compatibility patch. No private paths, raw training traces, or checkpoint data are included. + +## Contract hardening (2026-10-25) + +Runtime checkpoint `aba4ee8e164f4d24eedae97ea641cd9a6f69c343` makes the +thread-local suppression state, context-manager exception/traceback interface, +and buffer queue/record types explicit. Suppression starts at zero independently +in each thread; process/thread IDs retain their existing lazy initialization. +`__exit__` returns `Literal[False]`, preserving application exceptions. Buffer +serialization and queue failures propagate; a failed enqueue retains its batch +for a later successful flush without dropping or duplicating records. + +Validation used Python 3.10.16, PyTorch 2.2.2, and one OpenMP/MKL/OpenBLAS thread: + +```sh +python -m unittest discover -s tests -p 'test_trac*.py' +pre-commit run --files traincheck/instrumentor/tracer.py traincheck/instrumentor/dumper.py tests/test_tracing_contracts.py +mypy --follow-imports=silent --ignore-missing-imports --check-untyped-defs --disallow-untyped-defs --disallow-any-generics tests/test_tracing_contracts.py +``` + +All 15 tests passed (11 existing regressions and 4 state/write-failure contract +tests). All repository-pinned hooks passed; the stronger check of the new test +module used pinned mypy 1.9. Legacy serializer and proxy-wrapper annotations are +outside this narrowly scoped change; this does not claim repository-wide strict +typing. No benchmark timings were rerun for this typing/contract checkpoint; +the experiment numbers above remain evidence for their original revisions. diff --git a/docs/measurements/data/fresh-run-invariant-validation-mnist.jsonl b/docs/measurements/data/fresh-run-invariant-validation-mnist.jsonl new file mode 100644 index 00000000..9fbcea68 --- /dev/null +++ b/docs/measurements/data/fresh-run-invariant-validation-mnist.jsonl @@ -0,0 +1,204 @@ +{"index":0,"relation":"APIContainRelation","description":"Adadelta.step() always calls Adadelta._init_group()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":1,"relation":"APIContainRelation","description":"Adadelta.step() always calls Optimizer._accelerator_graph_capture_health_check()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":2,"relation":"APIContainRelation","description":"Adadelta.step() always calls Optimizer._optimizer_step_code()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":3,"relation":"APIContainRelation","description":"Adadelta.step() always calls TorchBindOpOverload.__call__()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":4,"relation":"APIContainRelation","description":"Adadelta.step() always calls _VariableFunctionsClass.is_complex()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":5,"relation":"APIContainRelation","description":"Adadelta.step() always calls _VariableFunctionsClass.zeros()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,1,9,0,1,1,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,1,9,0,1,1,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,1,19,0,1,1,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,1,19,0,1,1,0,20,0,null,true,false,true,false,false,true]}} +{"index":6,"relation":"APIContainRelation","description":"Adadelta.step() always calls _VariableFunctionsClass.zeros_like()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,1,9,0,1,1,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,1,9,0,1,1,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,1,19,0,1,1,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,1,19,0,1,1,0,20,0,null,true,false,true,false,false,true]}} +{"index":7,"relation":"APIContainRelation","description":"Adadelta.step() always calls _foreach_utils._get_foreach_kernels_supported_devices()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":8,"relation":"APIContainRelation","description":"Adadelta.step() always calls _foreach_utils._get_fused_kernels_supported_devices()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":9,"relation":"APIContainRelation","description":"Adadelta.step() always calls _ops._contains_fake_script_object()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":10,"relation":"APIContainRelation","description":"Adadelta.step() always calls _ops._must_dispatch_in_python()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":11,"relation":"APIContainRelation","description":"Adadelta.step() always calls accelerator.current_accelerator()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":12,"relation":"APIContainRelation","description":"Adadelta.step() always calls adadelta._single_tensor_adadelta()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":13,"relation":"APIContainRelation","description":"Adadelta.step() always calls adadelta.adadelta()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":14,"relation":"APIContainRelation","description":"Adadelta.step() always calls compiler.is_compiling()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":15,"relation":"APIContainRelation","description":"Adadelta.step() always calls optimizer._default_to_fused_or_foreach()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":16,"relation":"APIContainRelation","description":"Adadelta.step() always calls optimizer._get_scalar_dtype()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,1,9,0,1,1,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,1,9,0,1,1,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,1,19,0,1,1,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,1,19,0,1,1,0,20,0,null,true,false,true,false,false,true]}} +{"index":17,"relation":"APIContainRelation","description":"Adadelta.step() always calls optimizer._to_scalar()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":18,"relation":"APIContainRelation","description":"Adadelta.step() always calls torch.is_grad_enabled()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":19,"relation":"APIContainRelation","description":"Conv2d.forward() always calls Conv2d._conv_forward()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":20,"relation":"APIContainRelation","description":"Conv2d.forward() always calls _VariableFunctionsClass.conv2d()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":21,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",1,1,0,0,1,1,0,1,39,null,true,false,false,false,false,true]}} +{"index":22,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":23,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":24,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":25,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":26,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":27,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":28,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":29,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":30,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":31,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":32,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":33,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":34,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":35,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":36,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":37,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":38,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":39,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":40,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":41,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":42,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with ndim=4","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":43,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":44,"relation":"ConsistentOutputRelation","description":"Conv2d.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":45,"relation":"DistinctArgumentRelation","description":"Conv2d.forward() receives distinct arguments at each step","identity_clauses":[],"runs":{"reference":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"control":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"holdout":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true],"fault":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true]}} +{"index":46,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",1,1,0,0,1,1,0,1,39,null,true,false,false,false,false,true]}} +{"index":47,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":48,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":49,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":50,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",1,1,0,0,1,1,0,1,39,null,true,false,false,false,false,true]}} +{"index":51,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":52,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].grad_fn consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true]}} +{"index":53,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].grad_fn consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true]}} +{"index":54,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":55,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":56,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":57,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":58,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":59,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":60,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":61,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":62,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":63,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":64,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":65,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":66,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":67,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":68,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":69,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":70,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].itemsize consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":71,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":72,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":73,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].ndim consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":74,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":75,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":76,"relation":"ConsistentInputOutputRelation","description":"Conv2d.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":77,"relation":"APIContainRelation","description":"Dropout.forward() always calls functional.dropout()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":78,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":79,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":80,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":81,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":82,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":83,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":84,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":85,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":86,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":87,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":88,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":89,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":90,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":91,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":92,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":93,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":94,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":95,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":96,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":97,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":98,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":99,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":100,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":101,"relation":"DistinctArgumentRelation","description":"Dropout.forward() receives distinct arguments at each step","identity_clauses":[],"runs":{"reference":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"control":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"holdout":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true],"fault":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true]}} +{"index":102,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":103,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":104,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":105,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":106,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":107,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":108,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":109,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":110,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":111,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_leaf consistent with output[0].is_leaf","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":112,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":113,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":114,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":115,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":116,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":117,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":118,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":119,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":120,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":121,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":122,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":123,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":124,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":125,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":126,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":127,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].nbytes consistent with output[0].nbytes","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":128,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":129,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].requires_grad consistent with output[0].requires_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":130,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":131,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":132,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":133,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.2 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true]}} +{"index":134,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.2 consistent with output[0].shape.3","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true]}} +{"index":135,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.3 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true]}} +{"index":136,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.3 consistent with output[0].shape.3","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,10,10,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,20,20,0,40,0,null,true,false,true,false,false,true]}} +{"index":137,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":138,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":139,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":140,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":141,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":142,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":143,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":144,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":145,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":146,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":147,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":148,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":149,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":150,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":151,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":152,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":153,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":154,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":155,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":156,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":157,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":158,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with ndim=2","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":159,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":160,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":161,"relation":"DistinctArgumentRelation","description":"Linear.forward() receives distinct arguments at each step","identity_clauses":[],"runs":{"reference":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"control":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"holdout":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true],"fault":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true]}} +{"index":162,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":163,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":164,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":165,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":166,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":167,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":168,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":169,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":170,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":171,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_leaf consistent with output[0].is_leaf","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":172,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":173,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":174,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":175,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":176,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":177,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":178,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":179,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":180,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":181,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":182,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":183,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":184,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":185,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":186,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":187,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":188,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].ndim consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":189,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].requires_grad consistent with output[0].requires_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":190,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":191,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].shape.0 consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":192,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":193,"relation":"APIContainRelation","description":"Optimizer.zero_grad() always calls TorchBindOpOverload.__call__()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":194,"relation":"APIContainRelation","description":"Optimizer.zero_grad() always calls _ops._contains_fake_script_object()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":195,"relation":"APIContainRelation","description":"Optimizer.zero_grad() always calls _ops._must_dispatch_in_python()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":196,"relation":"APIContainRelation","description":"Optimizer.zero_grad() always calls compiler.is_exporting()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":197,"relation":"APIContainRelation","description":"adadelta.adadelta() always calls _VariableFunctionsClass.is_complex()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":198,"relation":"APIContainRelation","description":"adadelta.adadelta() always calls _foreach_utils._get_foreach_kernels_supported_devices()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":199,"relation":"APIContainRelation","description":"adadelta.adadelta() always calls _foreach_utils._get_fused_kernels_supported_devices()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":200,"relation":"APIContainRelation","description":"adadelta.adadelta() always calls adadelta._single_tensor_adadelta()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":201,"relation":"APIContainRelation","description":"adadelta.adadelta() always calls compiler.is_compiling()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":202,"relation":"APIContainRelation","description":"adadelta.adadelta() always calls optimizer._default_to_fused_or_foreach()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":203,"relation":"APIContainRelation","description":"adadelta.adadelta() always calls optimizer._to_scalar()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} diff --git a/docs/measurements/data/fresh-run-invariant-validation-transformer.jsonl b/docs/measurements/data/fresh-run-invariant-validation-transformer.jsonl new file mode 100644 index 00000000..b5594e51 --- /dev/null +++ b/docs/measurements/data/fresh-run-invariant-validation-transformer.jsonl @@ -0,0 +1,564 @@ +{"index":0,"relation":"APIContainRelation","description":"Dropout.forward() always calls functional.dropout()","identity_clauses":[],"runs":{"reference":[70,"parent_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"parent_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"parent_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"parent_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":1,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":2,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":3,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":4,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":5,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":6,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":7,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":8,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":9,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":10,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":11,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":12,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":13,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":14,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":15,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":16,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":17,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":18,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":19,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":20,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":21,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":22,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with ndim=3","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":23,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":24,"relation":"ConsistentOutputRelation","description":"Dropout.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":25,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":26,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":27,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":28,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":29,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":30,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":31,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":32,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":33,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":34,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_leaf consistent with output[0].is_leaf","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":35,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":36,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":37,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":38,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":39,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":40,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":41,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":42,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":43,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":44,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":45,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":46,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":47,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":48,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":49,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":50,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].nbytes consistent with output[0].nbytes","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":51,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":52,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].requires_grad consistent with output[0].requires_grad","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":53,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":54,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":55,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":56,"relation":"ConsistentInputOutputRelation","description":"Dropout.forward(): input[0].shape.2 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"control":[70,"complete_function_call",70,70,0,0,70,70,0,70,0,null,true,false,true,false,false,true],"holdout":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true],"fault":[140,"complete_function_call",140,140,0,0,140,140,0,140,0,null,true,false,true,false,false,true]}} +{"index":57,"relation":"APIContainRelation","description":"Embedding.forward() always calls _VariableFunctionsClass.embedding()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":58,"relation":"APIContainRelation","description":"Embedding.forward() always calls functional.embedding()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":59,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":60,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":61,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":62,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":63,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":64,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":65,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":66,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":67,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":68,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":69,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":70,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":71,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":72,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":73,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":74,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":75,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":76,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":77,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":78,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":79,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":80,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with nbytes=4096","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":81,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with ndim=3","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":82,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":83,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":84,"relation":"ConsistentOutputRelation","description":"Embedding.forward() consistently returns tensors with shape=(16, 2, 32)","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",1,1,0,0,1,1,0,1,19,null,true,false,false,false,false,true]}} +{"index":85,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":86,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":87,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].grad_fn consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":88,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].grad_fn consistent with output[0].name","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":89,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":90,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":91,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":92,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":93,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":94,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":95,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":96,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":97,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":98,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":99,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":100,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":101,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":102,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":103,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":104,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":105,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":106,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].ndim consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":107,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":108,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",1,1,0,0,1,1,0,1,19,null,true,false,false,false,false,true]}} +{"index":109,"relation":"ConsistentInputOutputRelation","description":"Embedding.forward(): input[0].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":110,"relation":"APIContainRelation","description":"LayerNorm.forward() always calls _VariableFunctionsClass.layer_norm()","identity_clauses":[],"runs":{"reference":[50,"parent_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"parent_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"parent_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"parent_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":111,"relation":"APIContainRelation","description":"LayerNorm.forward() always calls functional.layer_norm()","identity_clauses":[],"runs":{"reference":[50,"parent_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"parent_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"parent_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"parent_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":112,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":113,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":114,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":115,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":116,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":117,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":118,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":119,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":120,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":121,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":122,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":123,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":124,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":125,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":126,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":127,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":128,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":129,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":130,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":131,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":132,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":133,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with nbytes=4096","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":134,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with ndim=3","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":135,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":136,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":137,"relation":"ConsistentOutputRelation","description":"LayerNorm.forward() consistently returns tensors with shape=(16, 2, 32)","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":138,"relation":"DistinctArgumentRelation","description":"LayerNorm.forward() receives distinct arguments at each step","identity_clauses":[],"runs":{"reference":[100,"within_step_argument_pair",1,1,0,0,100,100,0,null,null,0,true,false,true,false,false,true],"control":[100,"within_step_argument_pair",1,1,0,0,100,100,0,null,null,0,true,false,true,false,false,true],"holdout":[200,"within_step_argument_pair",1,1,0,0,200,200,0,null,null,0,true,false,true,false,false,true],"fault":[200,"within_step_argument_pair",1,1,0,0,200,200,0,null,null,0,true,false,true,false,false,true]}} +{"index":139,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":140,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":141,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":142,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":143,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":144,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":145,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":146,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":147,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":148,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_leaf consistent with output[0].is_leaf","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":149,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":150,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":151,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":152,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":153,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":154,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":155,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":156,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":157,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":158,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":159,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":160,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":161,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":162,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":163,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":164,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].nbytes consistent with output[0].nbytes","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":165,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":166,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].requires_grad consistent with output[0].requires_grad","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":167,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":168,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":169,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":170,"relation":"ConsistentInputOutputRelation","description":"LayerNorm.forward(): input[0].shape.2 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":171,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":172,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":173,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":174,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":175,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":176,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":177,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":178,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":179,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":180,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":181,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":182,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":183,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":184,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":185,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":186,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":187,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":188,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":189,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":190,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":191,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":192,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with ndim=3","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":193,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":194,"relation":"ConsistentOutputRelation","description":"Linear.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":195,"relation":"DistinctArgumentRelation","description":"Linear.forward() receives distinct arguments at each step","identity_clauses":[],"runs":{"reference":[100,"within_step_argument_pair",1,1,0,0,100,100,0,null,null,0,true,false,true,false,false,true],"control":[100,"within_step_argument_pair",1,1,0,0,100,100,0,null,null,0,true,false,true,false,false,true],"holdout":[200,"within_step_argument_pair",1,1,0,0,200,200,0,null,null,0,true,false,true,false,false,true],"fault":[200,"within_step_argument_pair",1,1,0,0,200,200,0,null,null,0,true,false,true,false,false,true]}} +{"index":196,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":197,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":198,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":199,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":200,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":201,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":202,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":203,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":204,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":205,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_leaf consistent with output[0].is_leaf","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":206,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":207,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":208,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":209,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":210,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":211,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":212,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":213,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":214,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":215,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":216,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":217,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":218,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":219,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":220,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":221,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":222,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].requires_grad consistent with output[0].requires_grad","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":223,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":224,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":225,"relation":"ConsistentInputOutputRelation","description":"Linear.forward(): input[0].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"control":[50,"complete_function_call",50,50,0,0,50,50,0,50,0,null,true,false,true,false,false,true],"holdout":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true],"fault":[100,"complete_function_call",100,100,0,0,100,100,0,100,0,null,true,false,true,false,false,true]}} +{"index":226,"relation":"APIContainRelation","description":"MultiheadAttention.forward() always calls Tensor.unflatten()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":227,"relation":"APIContainRelation","description":"MultiheadAttention.forward() always calls _VariableFunctionsClass.is_floating_point()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":228,"relation":"APIContainRelation","description":"MultiheadAttention.forward() always calls functional._canonical_mask()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":229,"relation":"APIContainRelation","description":"MultiheadAttention.forward() always calls functional._in_projection_packed()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":230,"relation":"APIContainRelation","description":"MultiheadAttention.forward() always calls functional._mha_shape_check()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":231,"relation":"APIContainRelation","description":"MultiheadAttention.forward() always calls functional._none_or_dtype()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":232,"relation":"APIContainRelation","description":"MultiheadAttention.forward() always calls functional.multi_head_attention_forward()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":233,"relation":"APIContainRelation","description":"MultiheadAttention.forward() always calls mha.get_fastpath_enabled()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":234,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":235,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":236,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":237,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":238,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":239,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":240,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":241,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":242,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":243,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":244,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":245,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":246,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":247,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":248,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":249,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":250,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":251,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":252,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":253,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":254,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":255,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with nbytes=4096","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":256,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with ndim=3","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":257,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":258,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":259,"relation":"ConsistentOutputRelation","description":"MultiheadAttention.forward() consistently returns tensors with shape=(16, 2, 32)","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":260,"relation":"DistinctArgumentRelation","description":"MultiheadAttention.forward() receives distinct arguments at each step","identity_clauses":[],"runs":{"reference":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"control":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"holdout":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true],"fault":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true]}} +{"index":261,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":262,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":263,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":264,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":265,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":266,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":267,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":268,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":269,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":270,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_leaf consistent with output[0].is_leaf","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":271,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":272,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":273,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":274,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":275,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":276,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":277,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":278,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":279,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":280,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":281,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":282,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":283,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":284,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":285,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":286,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].nbytes consistent with output[0].nbytes","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":287,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":288,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].requires_grad consistent with output[0].requires_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":289,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":290,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":291,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":292,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[0].shape.2 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":293,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":294,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":295,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":296,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":297,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":298,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":299,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":300,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":301,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":302,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].nbytes consistent with output[0].nbytes","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":303,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":304,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":305,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":306,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[1].shape.2 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":307,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":308,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":309,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":310,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":311,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":312,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":313,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":314,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":315,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":316,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].nbytes consistent with output[0].nbytes","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":317,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":318,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":319,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":320,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[2].shape.2 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":321,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":322,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":323,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":324,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":325,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":326,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":327,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].grad_fn consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":328,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].grad_fn consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":329,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":330,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":331,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":332,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].ndim consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":333,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":334,"relation":"ConsistentInputOutputRelation","description":"MultiheadAttention.forward(): input[3].shape.1 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":335,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls Dropout.forward()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":336,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls LayerNorm.forward()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":337,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls Linear.forward()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":338,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls ModuleList._get_abs_string_index()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":339,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls MultiheadAttention.forward()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":340,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls Tensor.unflatten()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":341,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls TransformerEncoderLayer._ff_block()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":342,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls TransformerEncoderLayer._sa_block()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":343,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls TransformerEncoderLayer.forward()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":344,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls _VariableFunctionsClass.full()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":345,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls _VariableFunctionsClass.is_floating_point()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":346,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls _VariableFunctionsClass.layer_norm()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":347,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls _VariableFunctionsClass.relu()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":348,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls _VariableFunctionsClass.triu()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":349,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls functional._canonical_mask()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":350,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls functional._in_projection_packed()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":351,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls functional._mha_shape_check()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":352,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls functional._none_or_dtype()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":353,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls functional.dropout()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":354,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls functional.layer_norm()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":355,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls functional.multi_head_attention_forward()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":356,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls mha.get_fastpath_enabled()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":357,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls transformer._detect_is_causal_mask()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":358,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls transformer._generate_square_subsequent_mask()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":359,"relation":"APIContainRelation","description":"TransformerEncoder.forward() always calls transformer._get_seq_len()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":360,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":361,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":362,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":363,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":364,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":365,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":366,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":367,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":368,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":369,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":370,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":371,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":372,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":373,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":374,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":375,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":376,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":377,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":378,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":379,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":380,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":381,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with nbytes=4096","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":382,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with ndim=3","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":383,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":384,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":385,"relation":"ConsistentOutputRelation","description":"TransformerEncoder.forward() consistently returns tensors with shape=(16, 2, 32)","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":386,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":387,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":388,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":389,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":390,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":391,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":392,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":393,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":394,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":395,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_leaf consistent with output[0].is_leaf","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":396,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":397,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":398,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":399,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":400,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":401,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":402,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":403,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":404,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":405,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":406,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":407,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":408,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":409,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":410,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":411,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].nbytes consistent with output[0].nbytes","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":412,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":413,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].requires_grad consistent with output[0].requires_grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":414,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":415,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":416,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":417,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[0].shape.2 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":418,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":419,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":420,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":421,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":422,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":423,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":424,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].grad_fn consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":425,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].grad_fn consistent with output[0].name","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":426,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":427,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":428,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":429,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].ndim consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":430,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":431,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoder.forward(): input[1].shape.1 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":432,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls Dropout.forward()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":433,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls LayerNorm.forward()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":434,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls Linear.forward()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":435,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls MultiheadAttention.forward()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":436,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls Tensor.unflatten()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":437,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls TransformerEncoderLayer._ff_block()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":438,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls TransformerEncoderLayer._sa_block()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":439,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls _VariableFunctionsClass.is_floating_point()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":440,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls _VariableFunctionsClass.layer_norm()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":441,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls _VariableFunctionsClass.relu()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":442,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls functional._canonical_mask()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":443,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls functional._in_projection_packed()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":444,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls functional._mha_shape_check()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":445,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls functional._none_or_dtype()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":446,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls functional.dropout()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":447,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls functional.layer_norm()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":448,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls functional.multi_head_attention_forward()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":449,"relation":"APIContainRelation","description":"TransformerEncoderLayer.forward() always calls mha.get_fastpath_enabled()","identity_clauses":[],"runs":{"reference":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"parent_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":450,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":451,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":452,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":453,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":454,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":455,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":456,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_leaf=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":457,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":458,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":459,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":460,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":461,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":462,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":463,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":464,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":465,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":466,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":467,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":468,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":469,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":470,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":471,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with nbytes=4096","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":472,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with ndim=3","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":473,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with requires_grad=True","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":474,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":475,"relation":"ConsistentOutputRelation","description":"TransformerEncoderLayer.forward() consistently returns tensors with shape=(16, 2, 32)","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":476,"relation":"DistinctArgumentRelation","description":"TransformerEncoderLayer.forward() receives distinct arguments at each step","identity_clauses":[],"runs":{"reference":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"control":[10,"within_step_argument_pair",1,1,0,0,10,10,0,null,null,0,true,false,true,false,false,true],"holdout":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true],"fault":[20,"within_step_argument_pair",1,1,0,0,20,20,0,null,null,0,true,false,true,false,false,true]}} +{"index":477,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":478,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":479,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":480,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":481,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":482,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":483,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_cpu consistent with output[0].is_cpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":484,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_cuda consistent with output[0].is_cuda","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":485,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_ipu consistent with output[0].is_ipu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":486,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_leaf consistent with output[0].is_leaf","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":487,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_maia consistent with output[0].is_maia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":488,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_meta consistent with output[0].is_meta","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":489,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_mkldnn consistent with output[0].is_mkldnn","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":490,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_mps consistent with output[0].is_mps","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":491,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_mtia consistent with output[0].is_mtia","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":492,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_nested consistent with output[0].is_nested","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":493,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_quantized consistent with output[0].is_quantized","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":494,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_sparse consistent with output[0].is_sparse","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":495,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_sparse_csr consistent with output[0].is_sparse_csr","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":496,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_vulkan consistent with output[0].is_vulkan","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":497,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_xla consistent with output[0].is_xla","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":498,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].is_xpu consistent with output[0].is_xpu","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":499,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":500,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":501,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":502,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].nbytes consistent with output[0].nbytes","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":503,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].ndim consistent with output[0].ndim","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":504,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].requires_grad consistent with output[0].requires_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":505,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].retains_grad consistent with output[0].retains_grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":506,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":507,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].shape.1 consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":508,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[0].shape.2 consistent with output[0].shape.2","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":509,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":510,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":511,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].grad consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":512,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].grad consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":513,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].grad_dtype consistent with output[0].dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":514,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].grad_dtype consistent with output[0].grad_dtype","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":515,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].grad_fn consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":516,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].grad_fn consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":517,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].itemsize consistent with output[0].itemsize","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":518,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].name consistent with output[0].grad","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":519,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].name consistent with output[0].name","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":520,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].ndim consistent with output[0].shape.1","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":521,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].shape.0 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":522,"relation":"ConsistentInputOutputRelation","description":"TransformerEncoderLayer.forward(): input[1].shape.1 consistent with output[0].shape.0","identity_clauses":[],"runs":{"reference":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"control":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"holdout":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true],"fault":[40,"complete_function_call",40,40,0,0,40,40,0,40,0,null,true,false,true,false,false,true]}} +{"index":523,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls Module._named_members()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",1,1,0,0,1,1,0,1,9,null,true,false,false,false,false,true],"holdout":[20,"parent_call",1,1,0,0,1,1,0,1,19,null,true,false,false,false,false,true],"fault":[20,"parent_call",1,1,0,0,1,1,0,1,19,null,true,false,false,false,false,true]}} +{"index":524,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls Module.named_modules()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":525,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls Module.named_parameters()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",1,1,0,0,1,1,0,1,9,null,true,false,false,false,false,true],"holdout":[20,"parent_call",1,1,0,0,1,1,0,1,19,null,true,false,false,false,false,true],"fault":[20,"parent_call",1,1,0,0,1,1,0,1,19,null,true,false,false,false,false,true]}} +{"index":526,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls _VariableFunctionsClass._foreach_mul_()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":527,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls _VariableFunctionsClass._foreach_norm()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":528,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls _VariableFunctionsClass.clamp()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":529,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls _VariableFunctionsClass.stack()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":530,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls _foreach_utils._device_has_foreach_support()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":531,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls _foreach_utils._get_foreach_kernels_supported_devices()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":532,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls _foreach_utils._group_tensors_by_device_and_dtype()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":533,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls _foreach_utils._has_foreach_support()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":534,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls clip_grad._clip_grads_with_norm_()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":535,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls clip_grad._get_total_norm()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":536,"relation":"APIContainRelation","description":"clip_grad.clip_grad_norm_() always calls torch.is_grad_enabled()","identity_clauses":[],"runs":{"reference":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"parent_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"parent_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":537,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with dtype=torch.float32","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":538,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with grad=None","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":539,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with grad_dtype=torch.float32","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":540,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with grad_fn=None","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":541,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_cpu=True","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":542,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_cuda=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":543,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_ipu=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":544,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_leaf=True","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":545,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_maia=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":546,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_meta=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":547,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_mkldnn=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":548,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_mps=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":549,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_mtia=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":550,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_nested=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":551,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_quantized=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":552,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_sparse=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":553,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_sparse_csr=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":554,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_vulkan=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":555,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_xla=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":556,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with is_xpu=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":557,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with itemsize=4","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":558,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with name=None","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":559,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with nbytes=4","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":560,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with ndim=0","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":561,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with requires_grad=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":562,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with retains_grad=False","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} +{"index":563,"relation":"ConsistentOutputRelation","description":"clip_grad.clip_grad_norm_() consistently returns tensors with shape=()","identity_clauses":[],"runs":{"reference":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"control":[10,"complete_function_call",10,10,0,0,10,10,0,10,0,null,true,false,true,false,false,true],"holdout":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true],"fault":[20,"complete_function_call",20,20,0,0,20,20,0,20,0,null,true,false,true,false,false,true]}} diff --git a/docs/measurements/data/fresh-run-invariant-validation.json b/docs/measurements/data/fresh-run-invariant-validation.json new file mode 100644 index 00000000..95340944 --- /dev/null +++ b/docs/measurements/data/fresh-run-invariant-validation.json @@ -0,0 +1,1894 @@ +{ + "study_date": "2026-10-25", + "harness_commit": "cad62cc98ef5fb7b313325443caa33f715ed141e", + "source_composition": { + "base": "99d16905296d83495570fdcafa91370b242d2528", + "trace_integrity": "7d6c683db92d83ffd6db9fc9c11fbc0ebdd2c5d4", + "torchscript": "9ac61d1140cf8679c24442a174ec12f4de069980", + "state_fix_hunks_from": "1b5aedffbeefd4cdd9d30036dbb0a833521526cf", + "changed_python_files": [ + "tests/test_dynamic_typename.py", + "tests/test_precondition_state_isolation.py", + "tests/test_torchscript_compat.py", + "tests/test_trace_integrity.py", + "traincheck/instrumentor/dumper.py", + "traincheck/instrumentor/torchscript_compat.py", + "traincheck/instrumentor/tracer.py", + "traincheck/invariant/precondition.py", + "traincheck/utils.py" + ], + "dynamic_typename_fix": "2f034ae5d0fab7e792b7c0a5bdc6a1acfeb8962c", + "source_sha256": { + "tests/test_utils.py": "cc63afa128223d1a7c518becf2c215ffd0ab91891a01b23e0bbeba2e9726e56c", + "tests/test_dynamic_policy.py": "ac82536add642da8179acd54b07d86cff81b5856d90fc41896d18a57334ccdec", + "tests/test_inference_sampling.py": "41a632102208e53091615d21714b7dc1d6840465fd550efa33caf8eeca3fe361", + "tests/test_torchscript_compat.py": "653be1032409579030ebd3ec5694cc3e43f34df8e4f5750a76a9ffc64dd0a2e2", + "tests/test_trace_integrity.py": "0d12f81b39c4772e9d6743605c4a627078b4a9f5e85b254d694ad999d3496f08", + "tests/test_loop_injection.py": "c9fda263c5b014f2a56030d070f72144e5aea3be50c431d2a67a910e5978e11d", + "tests/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "tests/test_display_names.py": "bf85fee31e1bb8384468df76bf30d4209aa8e14d0b9b822a2af9860ee6c624cd", + "tests/test_dynamic_typename.py": "fc86e92919f07a82f5f06e2d9ebcd52431ec2ea175ceb5d1c2417aab45924ba8", + "tests/test_call_index.py": "979c4b4489b5ca8c2468db7ee5d7b39bb9fe8b3c193f7b43f1627ada17c7b541", + "tests/test_example_reservoir.py": "4805845c77e5bdeaf48793dd8b8cc1ed42126e9c2ea5154c073e9fa2cc075b90", + "tests/test_violation_summary.py": "fa5bd46bbc5592867eb7d018a494b81aa5d59f451fd282a147ed56aefd7e3619", + "tests/test_precondition_state_isolation.py": "d56e2e0e3ca5d410ccef5cb1de350a21f37a6be679efd302b523aae1f7a1d919", + "tests/test_identity_cache.py": "733575f48f690cfc2913169bff2a8656787fe94f37e4658850e29d2f4656ac50", + "tests/test_context_index.py": "7702f13e695f430e2aa1e302e3719ce9d5e8bd74a2d65cac5b7fbfa2178159dc", + "tests/test_policy_injection.py": "9409b8f693da8b05e556f658486346422c6e9b853f1a991cad28572dfd977848", + "tests/test_variable_tracking_policy.py": "059178ac644d6d9305dd10ffcea75dc99e5fd5019b02f09f37a8a93728f5d989", + "tests/fixtures/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "tests/fixtures/legacy_trace_pandas.py": "342bef3c1de26d77cc8b1e34f94dc470f45d929e038e458ac00eff11eebc1860", + "traincheck/collect_trace.py": "c1db96c798af9689f8ee7f7b0b929d90694d49a234cb217beaf435d4b8d5cfb0", + "traincheck/checker_online.py": "2824c9ccd2f684c86884c68f2214a1a4be1d69c6098ce30b817d03f1fe1077d6", + "traincheck/infer_engine.py": "476a3ec99bef0ab823b23f94bc95842c20b6d9b7b4a57ab4a2efa2987f30f286", + "traincheck/__init__.py": "47fdcb090c4c8c5822769d642dc23a520e8d8ff3e4724583450723e05408164b", + "traincheck/utils.py": "854c11519e7aa6d0cbbfd79694d2ad4895fb1aaf72e9efbb0068048598b70750", + "traincheck/progress.py": "aab7d040324e283c8474aa82bc7550a777aec316131a26e1dc1b6a277333d1ae", + "traincheck/parallel.py": "6843319dd8c5c397e93f361a29fe564161e7307fbe4f0f8368090d252d4a7bc4", + "traincheck/inference_sampling.py": "ba566c3745ed3043ef27cbc626c21b358d1d01c95a26f1ebc9c8f56391d63681", + "traincheck/checker.py": "44dc20fc66ca82bb6b71fa02033aae3b7d489ae7afbc1611c44e857f399716b2", + "traincheck/reporting/__init__.py": "299180153357c4e6f48cf75191e915d70cfca5ae6e7b3ac4d45144066b93ddca", + "traincheck/reporting/checker_report.py": "0d59b1753fc348701b49a0eb74e4b1d849a47d8234005ccbbb26e09e3cd0c808", + "traincheck/developer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/developer/annotations.py": "0ff28d35e085b40e1e366499feec96c8eb1e13bf3100dcdd7ab304c7f2f6e963", + "traincheck/invariant/consistency_transient_vars.py": "a702cd1b6787e8be1904bef9119773c0e037a83017959c771020dbbbbe1549e2", + "traincheck/invariant/symbolic_value.py": "28627e2c2a47a146087a96510f07f7deb8463c46371e418afa3fdeff33c7e6cb", + "traincheck/invariant/var_periodic_change_relation.py": "8086df3b9cfe34229b8bd46b18dce22a4b746902e38d1ba4bd02bc6f6c2f2ef4", + "traincheck/invariant/consistency_relation.py": "bc5b14810ea8a5e0e5f3e6c2c6be24d137a444d8609216d00eafd155b4aa3917", + "traincheck/invariant/cover_relation.py": "0eb0e1c30ad5b34a15ce32c7b0d15e894f7a3b9b68bfcaae5b901f5dc6a092c6", + "traincheck/invariant/__init__.py": "773e9f767566ffd92658dc0e94107ac840672e446a41ba437a49cee9cc52197a", + "traincheck/invariant/DistinctArgumentRelation.py": "6e01540583668e30718ba90c341fd540cd9df53f20525a6cf502fcceee8ed7bc", + "traincheck/invariant/precondition.py": "dfd02e6c68a6c3f2d06d6a9beaa109dcf99ef099efeda75252fd8c559fdbb5da", + "traincheck/invariant/relation_pool.py": "8ac568cf1924162a791728ed5db8c41868a550f9e0cd5190fc9a19827f55d3e6", + "traincheck/invariant/lead_relation.py": "978a4db32b69dc1a90a84123aa93e52a3caaba97761321dbb844666cb47d4360", + "traincheck/invariant/base_cls.py": "31ceb584760d26e9923d2c7ce962fd6f093a3a6ba983ed177fcb4257c1da2466", + "traincheck/invariant/contain_relation.py": "b32f819db4d5a11a11318bf7e38c64148f4c9360137ac5f09475b480c34c16a5", + "traincheck/trace/trace.py": "eac49a1cb958ccaae57ac93e8aad1cbc616e7fcef23068ef5bb68bbc54890c00", + "traincheck/trace/trace_pandas.py": "41040a288284dc10041a4c06d8598219605732dd5dcd2ef080b9ed0799387f8a", + "traincheck/trace/__init__.py": "ec84edd0ff32906c20956166a51b85b3847e22e48432c6a31698080cafbcbc2b", + "traincheck/trace/types.py": "96c1b137f3d72fb05dcf806a4e7ea21fe453ae9cbc3e08352c55e95973fe96e0", + "traincheck/trace/trace_polars.py": "64136efcd60a851b0391e708108570b259c2288db6242564b636f69abd1b7b0e", + "traincheck/trace/utils.py": "6356156e4c5bbb6c111140302660f36ad1c126e5bc977bd55c81f097022102a3", + "traincheck/trace/trace_dict.py": "3aec874ef0ce2c34f6332dface604d5f79965976fd3d1c2b692f746f2af4e703", + "traincheck/config/config.py": "4c940a4d92f5de5fe8d3885e570078e479d9842d9626651fd60483e78f0160a9", + "traincheck/config/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/runner/runner.py": "89027b488ded8640fddb387dd80d98f059cf40be5f94863966b259acea1da7c7", + "traincheck/runner/__init__.py": "287ffa9cab82f01da0e82faea77589dc7d348710c862496458d2d60c364394ba", + "traincheck/script/num_invs_each_rel.py": "202f13d536929c52defa84cdce00dec2251ea09aabf3ae8ae57067c32d0b844d", + "traincheck/script/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/instrumentor/caches.py": "320eb1ffa1ce5f5b97fe4675a16d47e9591265779fdc29b94b72cdf5df93f168", + "traincheck/instrumentor/control.py": "3612a39b61e85fb77b40008b7fcd2f7569b246e8985ca64a109ea53733cd29ef", + "traincheck/instrumentor/__init__.py": "228b4f6a11cde8118cde9aba2f98b60a839f4331e7fa2b06c37cee99e79fcd50", + "traincheck/instrumentor/types.py": "258679aa8d0b40247d900ba2e26ab03b25cb4626b046ab6d724973226c7b5469", + "traincheck/instrumentor/dumper.py": "dd813a342b4531a6543995a23591f558db00d04142611a2d9656b1258833fac3", + "traincheck/instrumentor/source_file.py": "dad92c793efb41161d86250e89aa6a9636749564e567df8b9af92a477eb4858d", + "traincheck/instrumentor/variable.py": "b01219cdd74f011055e0bc2a645ed283dbc47e66a834d5555b814fa815a9eab8", + "traincheck/instrumentor/torchscript_compat.py": "0d0c348fa02d5f93bb74e874829e5fb4c57365b82ce973ae8ff1f21a42c2d64e", + "traincheck/instrumentor/replace_functions.py": "ca4f10b52083a4a5c251db194b7608406cbe36c85372dfdb9df17644f3d20b62", + "traincheck/instrumentor/tracer.py": "ad57320f0a3764c02c73cf4bfa623f1ef73b9835dfa6cb1365dc2988551de428", + "traincheck/instrumentor/proxy_wrapper/proxy_registry.py": "9dfecae65aee4185152d152a5c2fb0e1e1b63339efceab3087df59a5882410a8", + "traincheck/instrumentor/proxy_wrapper/subclass.py": "66c751aa8ff31ff9fbcda0a361be8187c50d6dfec0b339cd218955c5c6978a19", + "traincheck/instrumentor/proxy_wrapper/torch_proxy.py": "d6abd7f2eac207757fd90cbf55e179ae506b25996bed906c5a094564ff6faf2f", + "traincheck/instrumentor/proxy_wrapper/proxy_config.py": "afb2f8dd04ed10e628ff21339ec8ebef36357fd2a0cc084e99e99e37756cf16b", + "traincheck/instrumentor/proxy_wrapper/proxy.py": "fe241558f0591738391bc57d0666e3b11e04d8579eb220ce5de855abc309baf3", + "traincheck/instrumentor/proxy_wrapper/proxy_observer.py": "10e83570fbe1f72089e4a594a308f83e045f7d74bb9e49193012645ac906d97b", + "traincheck/instrumentor/proxy_wrapper/__init__.py": "7c71aaf70b978ee04e236497af6f803e4436049bc0acdf18c3edb48bfcece63d", + "traincheck/instrumentor/proxy_wrapper/dumper.py": "6907ac0a198443dd9138a37dc9b5de1232d3d7fb26412f8bba3fb08c85cd222d", + "traincheck/instrumentor/proxy_wrapper/proxy_methods.py": "6ae18928aa12e89b33102368c497aff048e3d6bed330eca2118fac051b8e3a97", + "traincheck/instrumentor/proxy_wrapper/hash.py": "e917bec817df1981c682d8b8645307432c6badb220b9f7c424b4089d9bf97bd7", + "traincheck/instrumentor/proxy_wrapper/proxy_handler.py": "4bab74647d35c64c5e9c702295dc4cb010302bf620401da0fad8539692b95ac7", + "traincheck/instrumentor/proxy_wrapper/utils.py": "5f058609e89a96a3545561ed82d83f7dd6a153bb0376608654c199691984dc14", + "traincheck/instrumentor/proxy_wrapper/proxy_basics.py": "f012e97fa51cf77be1806807ef6130b06c9cec6fde8d047d57d45b6ec9fcc6e5", + "traincheck/static_analyzer/config.py": "840460167bd6d26db429076cd4dc227e6c17c4e5fb157172041c48574f44edf7", + "traincheck/static_analyzer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/graph_generator/anutils.py": "9e62b7ce5fced45a033b76de3d408bdff2fd8561ffa28bc58de82e4105b51d39", + "traincheck/static_analyzer/graph_generator/analyzer.py": "035684eb7ea4edf8ee0dbf4c6b12b91b428cd6dd57be5fe658253c21c8c420b1", + "traincheck/static_analyzer/graph_generator/call_graph_parser.py": "69e2555a1595078a2d2bca174de32cd7d4b0f023c0a20b68a01ca00d1d3b8647", + "traincheck/static_analyzer/graph_generator/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/graph_generator/node.py": "b6a0cea44a28d33f28fee79b7f7073e69914e95565f7a8d62f9e5a5fd6b240cd", + "traincheck/static_analyzer/graph_generator/main.py": "a568488c50383a67a6ab8feecaf96e29b288861c1dce17481a1b5032d20748a4", + "traincheck/static_analyzer/func_level/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/onlinechecker/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/onlinechecker/streamhandler_filesystem.py": "b5a298bd06a9408e857ff6721578cb4ae10f227762ba3d66aec685e574cfa511", + "traincheck/onlinechecker/utils.py": "65ba5d8a84b748adf4267800e704df1f4662c0f483534289af20096bf9a78a00", + "traincheck/toolkit/analyze_trace.py": "48dfc81ef85d96678b5fe466b86ba80d892ecfc497387773fa7ab5995f22e822", + "traincheck/toolkit/analyze_invariant.py": "706cf8c06062b6df3f1f86691f588c91696cbef6c27698b2f0b290ad8f53ff4b", + "traincheck/toolkit/detect_anomaly_from_trace_diff.py": "2ffc0723dd3709ccab09cb9b58d76c7bb0558dda09b66c47184e335b815cedd3", + "traincheck/toolkit/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + }, + "runtime": { + "python": "3.11.4", + "torch": "2.16.0.dev20261012+cpu", + "pandas": "2.3.3", + "numpy": "2.1.0", + "device": "cpu", + "native_threads": 1 + }, + "protocol": { + "reference_seed": 42, + "reference_steps": 10, + "control_seed": 42, + "control_steps": 10, + "holdout_seed": 43, + "holdout_steps": 20, + "batch_size": 2, + "prepared_data_steps": 20, + "check_relation_first": false, + "inference_workers": 1, + "disabled_relations": [ + "FunctionLeadRelation", + "FunctionCoverRelation" + ], + "each_phase_fresh_OS_process": true, + "holdout_inputs_overlap_reference": true, + "API_instrumentation": true, + "proxy_scanning": false + }, + "tests": { + "local_counter_and_mutation": 13, + "matching_linux_counter_and_mutation": 13, + "matching_linux_composed_production_stack": 29 + }, + "archives_sha256": { + "harness-cad62cc-complete.tar.gz": "b9ed7f3401363e73d8b69c9b12f183429365df18c28a61c1969e3be90285cb5a", + "integration-v3-source.tar.gz": "4b19005ae6ecad19e48b531185a7552cdfd0d08b6d23c907460ac33adb6498c3", + "final-mnist-74b-diagnostic.tar.gz": "0612f10428e1f22ebc831e6e63f990a91c917aa6128044e43a86c3790ba61195", + "final-mnist-cad62cc-evidence.tar.gz": "6a9a6866c4ac78d6050527fd6dac1f7d9586427b2cd607cbe182e86bd8135e6d", + "final-transformer-cad62cc-evidence.tar.gz": "f3b5d6b4c9065362d3d59cfe2339363700137a2aec1cbe865217a3e5f6120acc" + }, + "workloads": { + "mnist": { + "comparison": { + "status": "study_completed", + "invariants_sha256": "696547d55cb658fd691fbb11cad3a4ea5d7313874a22931b1615b1152608b9f7", + "training_comparisons": { + "same_seed_fresh_control": { + "loss_trajectory_equal": true, + "initial_state_sha256_equal": true, + "final_state_sha256_equal": true, + "shapes_equal": true + }, + "holdout_instrumentation_comparison": { + "loss_trajectory_equal": true, + "initial_state_sha256_equal": true, + "final_state_sha256_equal": true, + "shapes_equal": true + } + }, + "benchmark_data_runtime_source_and_harness_matched": true, + "observer_matches_direct_checker": true, + "per_run_summary": { + "reference": { + "completed": 204, + "active_invariants": 204, + "dormant_invariants": 0, + "violated_invariants": 0, + "identity_bearing_invariants": 0, + "identity_bearing_active": 0 + }, + "control": { + "completed": 204, + "active_invariants": 204, + "dormant_invariants": 0, + "violated_invariants": 0, + "identity_bearing_invariants": 0, + "identity_bearing_active": 0 + }, + "holdout": { + "completed": 204, + "active_invariants": 204, + "dormant_invariants": 0, + "violated_invariants": 0, + "identity_bearing_invariants": 0, + "identity_bearing_active": 0 + }, + "fault": { + "completed": 204, + "active_invariants": 204, + "dormant_invariants": 0, + "violated_invariants": 3, + "identity_bearing_invariants": 0, + "identity_bearing_active": 0 + } + }, + "per_relation": { + "reference": { + "APIContainRelation": { + "invariants": 33, + "active": 33, + "dormant": 0, + "violations": 0, + "opportunities": 360, + "precondition_group_passes": 333, + "actual_decisions": 333, + "proxy_checks_skipped": 0, + "fully_validated_active": 33 + }, + "ConsistentOutputRelation": { + "invariants": 71, + "active": 71, + "dormant": 0, + "violations": 0, + "opportunities": 1420, + "precondition_group_passes": 1420, + "actual_decisions": 1420, + "proxy_checks_skipped": 0, + "fully_validated_active": 71 + }, + "DistinctArgumentRelation": { + "invariants": 3, + "active": 3, + "dormant": 0, + "violations": 0, + "opportunities": 30, + "precondition_group_passes": 3, + "actual_decisions": 30, + "proxy_checks_skipped": 0, + "fully_validated_active": 3 + }, + "ConsistentInputOutputRelation": { + "invariants": 97, + "active": 97, + "dormant": 0, + "violations": 0, + "opportunities": 1940, + "precondition_group_passes": 1940, + "actual_decisions": 1880, + "proxy_checks_skipped": 0, + "fully_validated_active": 97 + } + }, + "control": { + "APIContainRelation": { + "invariants": 33, + "active": 33, + "dormant": 0, + "violations": 0, + "opportunities": 360, + "precondition_group_passes": 333, + "actual_decisions": 333, + "proxy_checks_skipped": 0, + "fully_validated_active": 33 + }, + "ConsistentOutputRelation": { + "invariants": 71, + "active": 71, + "dormant": 0, + "violations": 0, + "opportunities": 1420, + "precondition_group_passes": 1420, + "actual_decisions": 1420, + "proxy_checks_skipped": 0, + "fully_validated_active": 71 + }, + "DistinctArgumentRelation": { + "invariants": 3, + "active": 3, + "dormant": 0, + "violations": 0, + "opportunities": 30, + "precondition_group_passes": 3, + "actual_decisions": 30, + "proxy_checks_skipped": 0, + "fully_validated_active": 3 + }, + "ConsistentInputOutputRelation": { + "invariants": 97, + "active": 97, + "dormant": 0, + "violations": 0, + "opportunities": 1940, + "precondition_group_passes": 1940, + "actual_decisions": 1880, + "proxy_checks_skipped": 0, + "fully_validated_active": 97 + } + }, + "holdout": { + "APIContainRelation": { + "invariants": 33, + "active": 33, + "dormant": 0, + "violations": 0, + "opportunities": 720, + "precondition_group_passes": 663, + "actual_decisions": 663, + "proxy_checks_skipped": 0, + "fully_validated_active": 33 + }, + "ConsistentOutputRelation": { + "invariants": 71, + "active": 71, + "dormant": 0, + "violations": 0, + "opportunities": 2840, + "precondition_group_passes": 2840, + "actual_decisions": 2840, + "proxy_checks_skipped": 0, + "fully_validated_active": 71 + }, + "DistinctArgumentRelation": { + "invariants": 3, + "active": 3, + "dormant": 0, + "violations": 0, + "opportunities": 60, + "precondition_group_passes": 3, + "actual_decisions": 60, + "proxy_checks_skipped": 0, + "fully_validated_active": 3 + }, + "ConsistentInputOutputRelation": { + "invariants": 97, + "active": 97, + "dormant": 0, + "violations": 0, + "opportunities": 3880, + "precondition_group_passes": 3880, + "actual_decisions": 3760, + "proxy_checks_skipped": 0, + "fully_validated_active": 97 + } + }, + "fault": { + "APIContainRelation": { + "invariants": 33, + "active": 33, + "dormant": 0, + "violations": 0, + "opportunities": 720, + "precondition_group_passes": 663, + "actual_decisions": 663, + "proxy_checks_skipped": 0, + "fully_validated_active": 33 + }, + "ConsistentOutputRelation": { + "invariants": 71, + "active": 71, + "dormant": 0, + "violations": 1, + "opportunities": 2840, + "precondition_group_passes": 2801, + "actual_decisions": 2801, + "proxy_checks_skipped": 0, + "fully_validated_active": 71 + }, + "DistinctArgumentRelation": { + "invariants": 3, + "active": 3, + "dormant": 0, + "violations": 0, + "opportunities": 60, + "precondition_group_passes": 3, + "actual_decisions": 60, + "proxy_checks_skipped": 0, + "fully_validated_active": 3 + }, + "ConsistentInputOutputRelation": { + "invariants": 97, + "active": 97, + "dormant": 0, + "violations": 2, + "opportunities": 3880, + "precondition_group_passes": 3802, + "actual_decisions": 3682, + "proxy_checks_skipped": 0, + "fully_validated_active": 97 + } + } + }, + "raw_ids": { + "reference": { + "process": [ + 1347106 + ], + "thread": [ + 140498110309440 + ] + }, + "control": { + "process": [ + 1347133 + ], + "thread": [ + 140011360134208 + ] + }, + "holdout": { + "process": [ + 1347155 + ], + "thread": [ + 140714782577728 + ] + } + }, + "fault": { + "kind": "post_hoc_trace_field_sensitivity_control_not_training_fault", + "invariant_index": 21, + "healthy_decisions": 40, + "fault_decisions": 1, + "fault_precondition_group_passes": 1, + "violation": true, + "edit": { + "file": "trace_API_1347155_140714782577728.log", + "zero_based_line": 15, + "path": [ + "return_values", + "torch.FloatTensor", + "dtype" + ], + "attribute": "dtype", + "before_value": "torch.float32", + "after_value": "torch.float64", + "raw_process_id": 1347155, + "raw_thread_id": 140714782577728 + } + }, + "limitations": [ + "One reference seed and one holdout seed per workload; no generalization estimate.", + "The first half of holdout inputs overlaps the reference inputs.", + "Offline checking; no online checker, distributed training, or runtime overhead claim.", + "Fault control is post-hoc trace sensitivity, not a reproduced training bug.", + "Source-exact checker stops at the first violation per invariant." + ] + }, + "model": { + "model_config": { + "architecture": "exact Net class" + }, + "parameter_count": 1199882, + "precision": "float32", + "extracted_symbols": [ + "Net" + ], + "benchmark_source_sha256": "4478c5de92610a1e03d61c356801d51ffca749f99dac00f234d6b6ea2bcb2aa7", + "extracted_definitions_sha256": "f839142a57ce449054e05ffa79d746e5282e2599e101810e6ccf6e7a9dd33ce5", + "adapter_sha256": "876f28f728ef924fa97afb7be7786c8001df9e549e1d724fc6ba60c7fa6bf2f9", + "instrumentation": { + "funcs_to_instr": null, + "models_to_track": [], + "sampling_interval": 1, + "scan_proxy_in_args": false, + "target": "torch", + "use_full_instr": false, + "warm_up_steps": 0 + }, + "instrumented_function_count": 20998 + }, + "data": { + "count": 40, + "files": [ + { + "path": "train-images-idx3-ubyte", + "sha256": "ba891046e6505d7aadcbbe25680a0738ad16aec93bde7f9b65e87a2fc25776db" + }, + { + "path": "train-labels-idx1-ubyte", + "sha256": "65a50cbbf4e906d70832878ad85ccda5333a97f0f4c3dd2ef09a8a9eef7101c5" + } + ], + "kind": "cached_mnist_training_prefix", + "normalization": "(uint8 / 255 - 0.1307) / 0.3081", + "ordering": "first samples in IDX order; no shuffle" + }, + "trace_rows": { + "reference": 1584, + "control": 1584, + "holdout": 3104, + "fault": 3104 + }, + "failed_inference_hypotheses": 20, + "artifact_sha256": { + "baseline-holdout/adaptation.json": "e513882d9150e0f71fd1389dc3a2c21901a3098c44475d207856fb0f0c0020d3", + "baseline-holdout/metrics.json": "b6fed469dd317b29f34b26a0be43035ce046483277b4b2ec0786deed72a4a445", + "baseline-holdout/phase-result.json": "57a7f3f7877aec96c5a7581b1bc3934e3aca82521719c35646b95033d56b0f34", + "check-control/coverage.json": "f07b558285dbc44eafccaea64f6f812f9b3ed3f52a8155b22e0c8fdc649a4a6b", + "check-control/phase-result.json": "3f19692915fe7560759742d1bc0e50d40a74f7c0eed628e36621e83b06dcdf3b", + "check-fault/coverage.json": "d3cc7954b4b07506c319bfb36ef8b60205392a02ceaaa8b4e5a17b2a8077097e", + "check-fault/phase-result.json": "d382c3a28c73aec5594cb5e998b58c1a32f858ba11b6126fd6479919acb3fbc4", + "check-holdout/coverage.json": "5dc166605d095aeaada421e5e201774f0c91d83e58cf097b1f5ac11f2cb33c2d", + "check-holdout/phase-result.json": "87586fb7106d5425b88dce9d696c661773b29bcf5daa123fefb57d17176f6cf7", + "check-holdout-direct/coverage.json": "29d316bf84d3de6dec6d2441c8a4f52e6a03a953e1f16182cf1255ae31a5fde0", + "check-holdout-direct/phase-result.json": "4c978a879f1af1e88595bbb7126ed8da69dc566598fe0a8958aa149d9cb24523", + "check-reference/coverage.json": "b30b981c7a88357ab883d3cde688e88d40c784204b474418d10d42a8643f99b8", + "check-reference/phase-result.json": "de96730dcb4d33a6760189b98824f0e4d5728bb9488984f7a2d69e36920c8d15", + "comparison.json": "e9feb79f1954ac3f5dc7a75e703a6bd00b29f4b59a6519d268f801e8a9a0697e", + "control/adaptation.json": "5abdf5a1f238cbd0e8a83ed88a908698e8b8cdb460a466f043f558950e7532af", + "control/metrics.json": "61e93a78e4612d6201decf311dde5b726d62405c5772f541c3ced16102226788", + "control/phase-result.json": "8c7a42816ea67cf150fe037715e14a64437f7519c803460183e8a26909f26c3c", + "fault/mutation.json": "c4b7db0e8d68453fb5cb89611573db5aec1773ebaca0ada67f866938e0da1a0a", + "holdout/adaptation.json": "2d4c4079c071a0500d8f08e734096b767c7da1998ebd667f4ace6b1c97d14f6d", + "holdout/metrics.json": "30e4e104d56b2a9bb5218e3b891dbab483362a435b1f3f3455fb3aba33bd6218", + "holdout/phase-result.json": "b00787f9a5b65f2b8b724ef6e38b71b126ec0bb0c19897806e8ae9fd24e4c754", + "inference/phase-result.json": "cec09cc23d492a1a5fa910ee69412d210b0cc47ca5b6dfe6ac878570f6043ad3", + "prepare-fault/phase-result.json": "46590662108731e2fac6096653c72c9e2dc6ddb51355c82bae69d5119993dd56", + "reference/adaptation.json": "8204e4744995631ee4982033a3949d1b59c99fa58cdf624031c250610dac8603", + "reference/metrics.json": "31a6c6e58a0ead22b57ad69584672e956491dc54872f14b8a20c08968df444b0", + "reference/phase-result.json": "d511dfe65964b24f114852e0ad8acfe36a016dc43aa4833e8c537f82fe23cb02", + "study-plan.json": "bd5d1b4ec4ffa8ef68813ef0f781de6b3cfa9790b323e4d98447cc8de501af87" + }, + "invariants_jsonl_sha256": "696547d55cb658fd691fbb11cad3a4ea5d7313874a22931b1615b1152608b9f7", + "trace_sha256": { + "control/traces/trace_API_1347133_140011360134208.log": "e68a120d1713eeec0fb916a7938ee8b2ee2088ff16f9e2c41e89f1291cbee8d2", + "fault/traces/trace_API_1347155_140714782577728.log": "953fd8ed2ae5c7b9b8489a83b114c1b6f63a1fb210fe328ce63c65e8507f00fa", + "holdout/traces/trace_API_1347155_140714782577728.log": "764533de616105fecaf8f5a5b7c2bb9bf84b9a14c3cffb33fa0f88a92d047c44", + "reference/traces/trace_API_1347106_140498110309440.log": "c4800bbf95b3ef0fa367e107ca621beaba9768dc6a6c07fcc4e9439246da20be" + }, + "training_metrics": { + "reference": { + "completed_steps": 10, + "initial_model_state_sha256": "877e9abeb783ae5ed703a9666a7276e921e0424b78bb2d50a41461221d90ddb0", + "final_model_state_sha256": "1b6002ccebfac00518fc53119b3166628a01d87a03a53b791b2759594281dc84", + "state_shapes_unchanged": true, + "success": true, + "losses": [ + { + "loss": 2.2311902046203613 + }, + { + "loss": 3.296325445175171 + }, + { + "loss": 2.6148438453674316 + }, + { + "loss": 2.3794608116149902 + }, + { + "loss": 2.291707992553711 + }, + { + "loss": 2.423546552658081 + }, + { + "loss": 2.289013385772705 + }, + { + "loss": 2.39404559135437 + }, + { + "loss": 2.370988368988037 + }, + { + "loss": 2.282020092010498 + } + ] + }, + "control": { + "completed_steps": 10, + "initial_model_state_sha256": "877e9abeb783ae5ed703a9666a7276e921e0424b78bb2d50a41461221d90ddb0", + "final_model_state_sha256": "1b6002ccebfac00518fc53119b3166628a01d87a03a53b791b2759594281dc84", + "state_shapes_unchanged": true, + "success": true, + "losses": [ + { + "loss": 2.2311902046203613 + }, + { + "loss": 3.296325445175171 + }, + { + "loss": 2.6148438453674316 + }, + { + "loss": 2.3794608116149902 + }, + { + "loss": 2.291707992553711 + }, + { + "loss": 2.423546552658081 + }, + { + "loss": 2.289013385772705 + }, + { + "loss": 2.39404559135437 + }, + { + "loss": 2.370988368988037 + }, + { + "loss": 2.282020092010498 + } + ] + }, + "holdout": { + "completed_steps": 20, + "initial_model_state_sha256": "5ce184958e1b3340cb58e0cc5f54150a9e7a3c8c84b6d5bce9a095f4c6f1256b", + "final_model_state_sha256": "5a1c83cc4671c529a5c7525303db31a3e9196de4619a49ef101a294c27686f3e", + "state_shapes_unchanged": true, + "success": true, + "losses": [ + { + "loss": 2.237604856491089 + }, + { + "loss": 3.1580514907836914 + }, + { + "loss": 2.471099853515625 + }, + { + "loss": 2.4436941146850586 + }, + { + "loss": 2.2179641723632812 + }, + { + "loss": 2.353369951248169 + }, + { + "loss": 2.487359046936035 + }, + { + "loss": 2.25258731842041 + }, + { + "loss": 2.375113010406494 + }, + { + "loss": 2.3781189918518066 + }, + { + "loss": 2.244692802429199 + }, + { + "loss": 2.16099214553833 + }, + { + "loss": 1.955580711364746 + }, + { + "loss": 1.7661290168762207 + }, + { + "loss": 2.8482470512390137 + }, + { + "loss": 2.6151835918426514 + }, + { + "loss": 2.337063789367676 + }, + { + "loss": 2.2885587215423584 + }, + { + "loss": 2.2511115074157715 + }, + { + "loss": 2.382375717163086 + } + ] + }, + "baseline-holdout": { + "completed_steps": 20, + "initial_model_state_sha256": "5ce184958e1b3340cb58e0cc5f54150a9e7a3c8c84b6d5bce9a095f4c6f1256b", + "final_model_state_sha256": "5a1c83cc4671c529a5c7525303db31a3e9196de4619a49ef101a294c27686f3e", + "state_shapes_unchanged": true, + "success": true, + "losses": [ + { + "loss": 2.237604856491089 + }, + { + "loss": 3.1580514907836914 + }, + { + "loss": 2.471099853515625 + }, + { + "loss": 2.4436941146850586 + }, + { + "loss": 2.2179641723632812 + }, + { + "loss": 2.353369951248169 + }, + { + "loss": 2.487359046936035 + }, + { + "loss": 2.25258731842041 + }, + { + "loss": 2.375113010406494 + }, + { + "loss": 2.3781189918518066 + }, + { + "loss": 2.244692802429199 + }, + { + "loss": 2.16099214553833 + }, + { + "loss": 1.955580711364746 + }, + { + "loss": 1.7661290168762207 + }, + { + "loss": 2.8482470512390137 + }, + { + "loss": 2.6151835918426514 + }, + { + "loss": 2.337063789367676 + }, + { + "loss": 2.2885587215423584 + }, + { + "loss": 2.2511115074157715 + }, + { + "loss": 2.382375717163086 + } + ] + } + }, + "coverage_columns": [ + "opportunities", + "opportunity_unit", + "precondition_group_attempts", + "precondition_group_passes", + "precondition_group_failures", + "precondition_group_errors", + "decisions_started", + "decisions_completed", + "decisions_not_completed", + "visited_calls", + "unvisited_calls_after_stop", + "unvisited_pairs_after_stop", + "active", + "dormant", + "check_passed", + "variable_precondition_requested", + "variable_check_skipped", + "fully_validated_active" + ], + "fault": { + "kind": "post_hoc_trace_field_sensitivity_control_not_training_fault", + "selection": "first actually evaluated healthy mutable shape rule, otherwise a float32/float64 dtype rule", + "attribute": "dtype", + "invariant_index": 21, + "invariant": { + "text_description": "Conv2d.forward() consistently returns tensors with dtype=torch.float32", + "relation": "ConsistentOutputRelation", + "params": [ + { + "param_type": "APIParam", + "api_full_name": "torch.nn.modules.conv.Conv2d.forward" + }, + { + "param_type": "VarTypeParam", + "var_type": "torch.Tensor", + "attr_name": "dtype", + "const_value": "torch.float32" + } + ], + "precondition": { + "pre_event": { + "inverted": false, + "preconditions": [ + { + "clauses": "Unconditional" + } + ] + } + }, + "num_positive_examples": 20, + "num_negative_examples": 0 + }, + "func_call_id": "9d110c7f11d845f9ad86b2a2e542b000_23960437373776510", + "edits": [ + { + "file": "trace_API_1347155_140714782577728.log", + "zero_based_line": 15, + "path": [ + "return_values", + "torch.FloatTensor", + "dtype" + ], + "attribute": "dtype", + "before_value": "torch.float32", + "after_value": "torch.float64", + "raw_process_id": 1347155, + "raw_thread_id": 140714782577728 + } + ], + "traces": { + "trace_API_1347155_140714782577728.log": { + "original_sha256": "764533de616105fecaf8f5a5b7c2bb9bf84b9a14c3cffb33fa0f88a92d047c44", + "fault_sha256": "953fd8ed2ae5c7b9b8489a83b114c1b6f63a1fb210fe328ce63c65e8507f00fa" + } + }, + "invariants_sha256": "696547d55cb658fd691fbb11cad3a4ea5d7313874a22931b1615b1152608b9f7", + "healthy_coverage_sha256": "5dc166605d095aeaada421e5e201774f0c91d83e58cf097b1f5ac11f2cb33c2d", + "precondition_inputs_unchanged": true, + "ids_unchanged": true, + "invariants_not_retrained_or_rewritten": true + }, + "per_invariant_ledger": { + "file": "fresh-run-invariant-validation-mnist.jsonl", + "sha256": "59a8cf816bd4474ed4505d23959d52a7d050ef01ea222471f8fc3f1020d27bbd", + "format": "One rule per line; run arrays use coverage_columns in this document; null means not applicable/unobserved, never zero." + } + }, + "transformer": { + "comparison": { + "status": "study_completed", + "invariants_sha256": "4d4afa40e1480809882761869195bf97e954cbc76bb7eff826125a91b8cd3996", + "training_comparisons": { + "same_seed_fresh_control": { + "loss_trajectory_equal": true, + "initial_state_sha256_equal": true, + "final_state_sha256_equal": true, + "shapes_equal": true + }, + "holdout_instrumentation_comparison": { + "loss_trajectory_equal": true, + "initial_state_sha256_equal": true, + "final_state_sha256_equal": true, + "shapes_equal": true + } + }, + "benchmark_data_runtime_source_and_harness_matched": true, + "observer_matches_direct_checker": true, + "per_run_summary": { + "reference": { + "completed": 564, + "active_invariants": 564, + "dormant_invariants": 0, + "violated_invariants": 0, + "identity_bearing_invariants": 0, + "identity_bearing_active": 0 + }, + "control": { + "completed": 564, + "active_invariants": 564, + "dormant_invariants": 0, + "violated_invariants": 2, + "identity_bearing_invariants": 0, + "identity_bearing_active": 0 + }, + "holdout": { + "completed": 564, + "active_invariants": 564, + "dormant_invariants": 0, + "violated_invariants": 2, + "identity_bearing_invariants": 0, + "identity_bearing_active": 0 + }, + "fault": { + "completed": 564, + "active_invariants": 564, + "dormant_invariants": 0, + "violated_invariants": 4, + "identity_bearing_invariants": 0, + "identity_bearing_active": 0 + } + }, + "per_relation": { + "reference": { + "APIContainRelation": { + "invariants": 70, + "active": 70, + "dormant": 0, + "violations": 0, + "opportunities": 1100, + "precondition_group_passes": 1100, + "actual_decisions": 1100, + "proxy_checks_skipped": 0, + "fully_validated_active": 70 + }, + "ConsistentOutputRelation": { + "invariants": 205, + "active": 205, + "dormant": 0, + "violations": 0, + "opportunities": 6010, + "precondition_group_passes": 6010, + "actual_decisions": 6010, + "proxy_checks_skipped": 0, + "fully_validated_active": 205 + }, + "ConsistentInputOutputRelation": { + "invariants": 285, + "active": 285, + "dormant": 0, + "violations": 0, + "opportunities": 8450, + "precondition_group_passes": 8450, + "actual_decisions": 8450, + "proxy_checks_skipped": 0, + "fully_validated_active": 285 + }, + "DistinctArgumentRelation": { + "invariants": 4, + "active": 4, + "dormant": 0, + "violations": 0, + "opportunities": 220, + "precondition_group_passes": 4, + "actual_decisions": 220, + "proxy_checks_skipped": 0, + "fully_validated_active": 4 + } + }, + "control": { + "APIContainRelation": { + "invariants": 70, + "active": 70, + "dormant": 0, + "violations": 2, + "opportunities": 1100, + "precondition_group_passes": 1082, + "actual_decisions": 1082, + "proxy_checks_skipped": 0, + "fully_validated_active": 70 + }, + "ConsistentOutputRelation": { + "invariants": 205, + "active": 205, + "dormant": 0, + "violations": 0, + "opportunities": 6010, + "precondition_group_passes": 6010, + "actual_decisions": 6010, + "proxy_checks_skipped": 0, + "fully_validated_active": 205 + }, + "ConsistentInputOutputRelation": { + "invariants": 285, + "active": 285, + "dormant": 0, + "violations": 0, + "opportunities": 8450, + "precondition_group_passes": 8450, + "actual_decisions": 8450, + "proxy_checks_skipped": 0, + "fully_validated_active": 285 + }, + "DistinctArgumentRelation": { + "invariants": 4, + "active": 4, + "dormant": 0, + "violations": 0, + "opportunities": 220, + "precondition_group_passes": 4, + "actual_decisions": 220, + "proxy_checks_skipped": 0, + "fully_validated_active": 4 + } + }, + "holdout": { + "APIContainRelation": { + "invariants": 70, + "active": 70, + "dormant": 0, + "violations": 2, + "opportunities": 2200, + "precondition_group_passes": 2162, + "actual_decisions": 2162, + "proxy_checks_skipped": 0, + "fully_validated_active": 70 + }, + "ConsistentOutputRelation": { + "invariants": 205, + "active": 205, + "dormant": 0, + "violations": 0, + "opportunities": 12020, + "precondition_group_passes": 12020, + "actual_decisions": 12020, + "proxy_checks_skipped": 0, + "fully_validated_active": 205 + }, + "ConsistentInputOutputRelation": { + "invariants": 285, + "active": 285, + "dormant": 0, + "violations": 0, + "opportunities": 16900, + "precondition_group_passes": 16900, + "actual_decisions": 16900, + "proxy_checks_skipped": 0, + "fully_validated_active": 285 + }, + "DistinctArgumentRelation": { + "invariants": 4, + "active": 4, + "dormant": 0, + "violations": 0, + "opportunities": 440, + "precondition_group_passes": 4, + "actual_decisions": 440, + "proxy_checks_skipped": 0, + "fully_validated_active": 4 + } + }, + "fault": { + "APIContainRelation": { + "invariants": 70, + "active": 70, + "dormant": 0, + "violations": 2, + "opportunities": 2200, + "precondition_group_passes": 2162, + "actual_decisions": 2162, + "proxy_checks_skipped": 0, + "fully_validated_active": 70 + }, + "ConsistentOutputRelation": { + "invariants": 205, + "active": 205, + "dormant": 0, + "violations": 1, + "opportunities": 12020, + "precondition_group_passes": 12001, + "actual_decisions": 12001, + "proxy_checks_skipped": 0, + "fully_validated_active": 205 + }, + "ConsistentInputOutputRelation": { + "invariants": 285, + "active": 285, + "dormant": 0, + "violations": 1, + "opportunities": 16900, + "precondition_group_passes": 16881, + "actual_decisions": 16881, + "proxy_checks_skipped": 0, + "fully_validated_active": 285 + }, + "DistinctArgumentRelation": { + "invariants": 4, + "active": 4, + "dormant": 0, + "violations": 0, + "opportunities": 440, + "precondition_group_passes": 4, + "actual_decisions": 440, + "proxy_checks_skipped": 0, + "fully_validated_active": 4 + } + } + }, + "raw_ids": { + "reference": { + "process": [ + 1347485 + ], + "thread": [ + 140490739766336 + ] + }, + "control": { + "process": [ + 1347544 + ], + "thread": [ + 140262258558016 + ] + }, + "holdout": { + "process": [ + 1347647 + ], + "thread": [ + 139745137321024 + ] + } + }, + "fault": { + "kind": "post_hoc_trace_field_sensitivity_control_not_training_fault", + "invariant_index": 84, + "healthy_decisions": 20, + "fault_decisions": 1, + "fault_precondition_group_passes": 1, + "violation": true, + "edit": { + "file": "trace_API_1347647_139745137321024.log", + "zero_based_line": 75, + "path": [ + "return_values", + "torch.FloatTensor", + "shape", + 0 + ], + "attribute": "shape", + "before_value": [ + 16, + 2, + 32 + ], + "after_value": [ + 17, + 2, + 32 + ], + "raw_process_id": 1347647, + "raw_thread_id": 139745137321024 + } + }, + "limitations": [ + "One reference seed and one holdout seed per workload; no generalization estimate.", + "The first half of holdout inputs overlaps the reference inputs.", + "Offline checking; no online checker, distributed training, or runtime overhead claim.", + "Fault control is post-hoc trace sensitivity, not a reproduced training bug.", + "Source-exact checker stops at the first violation per invariant." + ] + }, + "model": { + "model_config": { + "clip": 0.25, + "dropout_argument": 0.2, + "lr": 1.0, + "nhead": 2, + "nhid": 64, + "ninp": 32, + "nlayers": 2, + "ntoken": 293, + "sequence_length": 16 + }, + "parameter_count": 36197, + "precision": "float32", + "extracted_symbols": [ + "PositionalEncoding", + "TransformerModel" + ], + "benchmark_source_sha256": "e4451a33e9b16953004bc639d736d14f42a2b21f2939ccabe4cff10d3c40b74c", + "extracted_definitions_sha256": "1f2db95454690be6a91aaa141e3a5a246308f9abca11444c9113e4adfb78b95d", + "adapter_sha256": "876f28f728ef924fa97afb7be7786c8001df9e549e1d724fc6ba60c7fa6bf2f9", + "instrumentation": { + "funcs_to_instr": null, + "models_to_track": [], + "sampling_interval": 1, + "scan_proxy_in_args": false, + "target": "torch", + "use_full_instr": false, + "warm_up_steps": 0 + }, + "instrumented_function_count": 20998 + }, + "data": { + "adaptation": "Whitespace tokenization plus ; vocabulary and tensors are bounded to the chosen prefix, unlike the full benchmark corpus.", + "kind": "wikitext2_training_prefix_with_prefix_only_vocabulary", + "path": "train.txt", + "prefix_sha256": "457b31f0b69fdfbf48c919d7fa9a7b073c2b03b8adfa51de3543e109b0e01903", + "sha256": "9e9fa1ad55b1c2c95b08e37dd8e653f638fac2c6de904b79e813611eefbc985f", + "token_count": 642, + "vocabulary_size": 293 + }, + "trace_rows": { + "reference": 4446, + "control": 4446, + "holdout": 8886, + "fault": 8886 + }, + "failed_inference_hypotheses": 11, + "artifact_sha256": { + "baseline-holdout/adaptation.json": "d73ea9cff0d15217963372d4c6b18eba13e0eaa4e03d45aa627ff9c753597a32", + "baseline-holdout/metrics.json": "f9d111863e26ff2023405c41cf862513d8ea167464ad4f7c10989abb257d669d", + "baseline-holdout/phase-result.json": "02e5f2576d7223c059e159f624d6e5030f3d08c9bab8391ee2e9bd051d1af4ef", + "check-control/coverage.json": "2fcdad5051bf4679d3663a0aaf86048a236e971a80517f33890a7f56b17f64c0", + "check-control/phase-result.json": "58d12a56e1be8fb59638f8ab13cb6257958be989bda2e0028d900a28dc3e64c6", + "check-fault/coverage.json": "a89b68a24e46228872553b8ee6fed51984e172e52651583be2ac4f23330d6d6d", + "check-fault/phase-result.json": "e063ef35ebd7eea7accff6e046e5fc2ca7650e4c94e3c3c2af7f69f8c99b114e", + "check-holdout/coverage.json": "079382f905a5809e6d66e2f1fd4ca4ca65423b1056eb69bdea1fa93a488f00f0", + "check-holdout/phase-result.json": "4807c3c7edd8e49d87150c29a3bd8aeee71992d21e08718c68976da3967c6d89", + "check-holdout-direct/coverage.json": "c060b0f9ef7da9002198017625d56876950518c5f4072d39d75e3982cc5019dd", + "check-holdout-direct/phase-result.json": "86455b01be33c9798433c7be940fc8bad767789e95852a0014fdabdbce1b34a9", + "check-reference/coverage.json": "771e09f0692dee2230d8e073d515945ebc74292b61baa04d830facb049284af2", + "check-reference/phase-result.json": "fde3c128ccc040852965261555e0d65b543a9360268f46a443cc2c8382a06eb7", + "comparison.json": "ce41f5fa74255424fb7c87344fbf3ba540629004f6e37662e2b5509444fed201", + "control/adaptation.json": "f1f9d3cebb92dc9a53a1810fd7ac2b9784f826359010fe8669209f17a2d6afef", + "control/metrics.json": "c91ee13d398cad807b7a62841f47e82d37492d49c23b961dcdb28a68ca7503f9", + "control/phase-result.json": "4d8a83a3b84ed1cc11c745392b8c3cdd5d34b395a99a4fb732c7449582de372a", + "fault/mutation.json": "cadfcc8f26752d903f5d05170cb38ff320b66f431e46518b190c8e3f0d7cd444", + "holdout/adaptation.json": "d9162c4392cf9a64111ed97779344ceda32f74e02bcf4c1274b2e14e2d7ec6da", + "holdout/metrics.json": "17a4d6ff99936f05e49552b26569ae8309955ff7088d69c3130327bed2b61d6c", + "holdout/phase-result.json": "2bb9f6cc8ffda717bb89688697da7889ce5855965c9e7f3b5087000b4c06e8d8", + "inference/phase-result.json": "15c4fbd12e3cf9609dd9be61491fd8a9f04e5b9813f55eed0656766b1a538d7c", + "prepare-fault/phase-result.json": "9b90cb87a59c3e12fb99dd819276f8b85231c431d0566a18ba88559c51b30cfb", + "reference/adaptation.json": "6078c8beafec0759c2742020cf94899ded01b5ebcbfb633d34ed6bb982d79c5f", + "reference/metrics.json": "710ef8a517ffd905f03efb88a59e35f2c7353b20d512d75e48f331738cd87b57", + "reference/phase-result.json": "25adbbf0eb426c585ced6ef4605d2d227bcda0b5c7079b9ceafa4908d1a89780", + "study-plan.json": "9d065aae89605f2fe63b299f15164f07275980c775bfe993c7baa12a6a0a9534" + }, + "invariants_jsonl_sha256": "4d4afa40e1480809882761869195bf97e954cbc76bb7eff826125a91b8cd3996", + "trace_sha256": { + "control/traces/trace_API_1347544_140262258558016.log": "440ad458b125167c19146e5043ab355e236067dae7df623a8abd992e16b3df96", + "fault/traces/trace_API_1347647_139745137321024.log": "860ee3203f82a34b9e119cb975231ec57a6a98c548e81dae02809f422dc0dc55", + "holdout/traces/trace_API_1347647_139745137321024.log": "6f5985e8fb93c8e2783d28813457581e2f491592d62c0fa4c581ca2ce558f73d", + "reference/traces/trace_API_1347485_140490739766336.log": "4184c9abeacc8c3c49a583038f507a3b0c3fb7218733230f34530df54003f60a" + }, + "training_metrics": { + "reference": { + "completed_steps": 10, + "initial_model_state_sha256": "83911ab0a0000c4a6fd71aa5bed8ca44de3241c5cd123367e70181c6eee5903c", + "final_model_state_sha256": "5034413e68786dd722b7681035bd6efb1c7d64b3c28ab450e458f5d80853fd76", + "state_shapes_unchanged": true, + "success": true, + "losses": [ + { + "loss": 5.8035569190979 + }, + { + "loss": 5.7292656898498535 + }, + { + "loss": 5.618026256561279 + }, + { + "loss": 5.672450065612793 + }, + { + "loss": 5.592149257659912 + }, + { + "loss": 5.601678848266602 + }, + { + "loss": 5.614868640899658 + }, + { + "loss": 5.65100622177124 + }, + { + "loss": 5.449929237365723 + }, + { + "loss": 5.456549167633057 + } + ] + }, + "control": { + "completed_steps": 10, + "initial_model_state_sha256": "83911ab0a0000c4a6fd71aa5bed8ca44de3241c5cd123367e70181c6eee5903c", + "final_model_state_sha256": "5034413e68786dd722b7681035bd6efb1c7d64b3c28ab450e458f5d80853fd76", + "state_shapes_unchanged": true, + "success": true, + "losses": [ + { + "loss": 5.8035569190979 + }, + { + "loss": 5.7292656898498535 + }, + { + "loss": 5.618026256561279 + }, + { + "loss": 5.672450065612793 + }, + { + "loss": 5.592149257659912 + }, + { + "loss": 5.601678848266602 + }, + { + "loss": 5.614868640899658 + }, + { + "loss": 5.65100622177124 + }, + { + "loss": 5.449929237365723 + }, + { + "loss": 5.456549167633057 + } + ] + }, + "holdout": { + "completed_steps": 20, + "initial_model_state_sha256": "31d5c753005bcf762ac2c1af38c3e71693cde10d193d80a85729ec342e69a572", + "final_model_state_sha256": "7e7a7529ef3c7d351ca0b0b3beccaf39c8eb8df71f2c8e8f9f9f5d936f9431c5", + "state_shapes_unchanged": true, + "success": true, + "losses": [ + { + "loss": 5.669175624847412 + }, + { + "loss": 5.593283653259277 + }, + { + "loss": 5.664262771606445 + }, + { + "loss": 5.635302543640137 + }, + { + "loss": 5.59808874130249 + }, + { + "loss": 5.639740467071533 + }, + { + "loss": 5.552767753601074 + }, + { + "loss": 5.676934719085693 + }, + { + "loss": 5.419702529907227 + }, + { + "loss": 5.414367198944092 + }, + { + "loss": 5.355993747711182 + }, + { + "loss": 5.5031280517578125 + }, + { + "loss": 5.387535572052002 + }, + { + "loss": 5.472236633300781 + }, + { + "loss": 5.2465410232543945 + }, + { + "loss": 5.450842380523682 + }, + { + "loss": 5.42155647277832 + }, + { + "loss": 5.213186264038086 + }, + { + "loss": 5.500979423522949 + }, + { + "loss": 5.30380392074585 + } + ] + }, + "baseline-holdout": { + "completed_steps": 20, + "initial_model_state_sha256": "31d5c753005bcf762ac2c1af38c3e71693cde10d193d80a85729ec342e69a572", + "final_model_state_sha256": "7e7a7529ef3c7d351ca0b0b3beccaf39c8eb8df71f2c8e8f9f9f5d936f9431c5", + "state_shapes_unchanged": true, + "success": true, + "losses": [ + { + "loss": 5.669175624847412 + }, + { + "loss": 5.593283653259277 + }, + { + "loss": 5.664262771606445 + }, + { + "loss": 5.635302543640137 + }, + { + "loss": 5.59808874130249 + }, + { + "loss": 5.639740467071533 + }, + { + "loss": 5.552767753601074 + }, + { + "loss": 5.676934719085693 + }, + { + "loss": 5.419702529907227 + }, + { + "loss": 5.414367198944092 + }, + { + "loss": 5.355993747711182 + }, + { + "loss": 5.5031280517578125 + }, + { + "loss": 5.387535572052002 + }, + { + "loss": 5.472236633300781 + }, + { + "loss": 5.2465410232543945 + }, + { + "loss": 5.450842380523682 + }, + { + "loss": 5.42155647277832 + }, + { + "loss": 5.213186264038086 + }, + { + "loss": 5.500979423522949 + }, + { + "loss": 5.30380392074585 + } + ] + } + }, + "coverage_columns": [ + "opportunities", + "opportunity_unit", + "precondition_group_attempts", + "precondition_group_passes", + "precondition_group_failures", + "precondition_group_errors", + "decisions_started", + "decisions_completed", + "decisions_not_completed", + "visited_calls", + "unvisited_calls_after_stop", + "unvisited_pairs_after_stop", + "active", + "dormant", + "check_passed", + "variable_precondition_requested", + "variable_check_skipped", + "fully_validated_active" + ], + "fault": { + "kind": "post_hoc_trace_field_sensitivity_control_not_training_fault", + "selection": "first actually evaluated healthy mutable shape rule, otherwise a float32/float64 dtype rule", + "attribute": "shape", + "invariant_index": 84, + "invariant": { + "text_description": "Embedding.forward() consistently returns tensors with shape=(16, 2, 32)", + "relation": "ConsistentOutputRelation", + "params": [ + { + "param_type": "APIParam", + "api_full_name": "torch.nn.modules.sparse.Embedding.forward" + }, + { + "param_type": "VarTypeParam", + "var_type": "torch.Tensor", + "attr_name": "shape", + "const_value": [ + 16, + 2, + 32 + ] + } + ], + "precondition": { + "pre_event": { + "inverted": false, + "preconditions": [ + { + "clauses": "Unconditional" + } + ] + } + }, + "num_positive_examples": 10, + "num_negative_examples": 0 + }, + "func_call_id": "e4ac9bdf7ba245438bdc313f1b54c933_23960502411558452", + "edits": [ + { + "file": "trace_API_1347647_139745137321024.log", + "zero_based_line": 75, + "path": [ + "return_values", + "torch.FloatTensor", + "shape", + 0 + ], + "attribute": "shape", + "before_value": [ + 16, + 2, + 32 + ], + "after_value": [ + 17, + 2, + 32 + ], + "raw_process_id": 1347647, + "raw_thread_id": 139745137321024 + } + ], + "traces": { + "trace_API_1347647_139745137321024.log": { + "original_sha256": "6f5985e8fb93c8e2783d28813457581e2f491592d62c0fa4c581ca2ce558f73d", + "fault_sha256": "860ee3203f82a34b9e119cb975231ec57a6a98c548e81dae02809f422dc0dc55" + } + }, + "invariants_sha256": "4d4afa40e1480809882761869195bf97e954cbc76bb7eff826125a91b8cd3996", + "healthy_coverage_sha256": "079382f905a5809e6d66e2f1fd4ca4ca65423b1056eb69bdea1fa93a488f00f0", + "precondition_inputs_unchanged": true, + "ids_unchanged": true, + "invariants_not_retrained_or_rewritten": true + }, + "healthy_false_alarm_evidence": [ + { + "index": 523, + "rule": { + "text_description": "clip_grad.clip_grad_norm_() always calls Module._named_members()", + "relation": "APIContainRelation", + "params": [ + { + "param_type": "APIParam", + "api_full_name": "torch.nn.utils.clip_grad.clip_grad_norm_" + }, + { + "param_type": "APIParam", + "api_full_name": "torch.nn.modules.module.Module._named_members", + "arguments": { + "args": { + "prefix": { + "str": "" + }, + "recurse": { + "bool": true + }, + "remove_duplicate": { + "bool": true + }, + "self": { + "trace_impact_benchmark.TransformerModel": { + "batch_first": false, + "call_super_init": false, + "d_model": 32, + "dump_patches": false, + "model_type": "Transformer", + "nhead": 2, + "ninp": 32, + "_TRAINCHECK_src_mask_ID": 140489496626592, + "training": true + } + }, + "get_members_fn": { + "torch.nn.modules.module.Module.named_parameters..": {} + } + }, + "func_name": "torch.nn.modules.module.Module._named_members" + } + } + ], + "precondition": { + "parent_func_call_pre": { + "inverted": false, + "preconditions": [ + { + "clauses": "Unconditional" + } + ] + } + }, + "num_positive_examples": 10, + "num_negative_examples": 0 + }, + "evidence": { + "reference": { + "parent_call_id": "1ea3c64687494354b9700101b1e519a0_23960484884548567", + "child_call_id": "1ea3c64687494354b9700101b1e519a0_23960484884819012", + "same_process_and_thread": true, + "strictly_within_parent_times": true, + "self_argument_differences": {}, + "child_raw_args": { + "0": { + "trace_impact_benchmark.TransformerModel": { + "batch_first": false, + "call_super_init": false, + "d_model": 32, + "dump_patches": false, + "model_type": "Transformer", + "nhead": 2, + "ninp": 32, + "_TRAINCHECK_src_mask_ID": 140489496626592, + "training": true + } + }, + "1": { + "torch.nn.modules.module.Module.named_parameters..": {} + } + }, + "child_raw_kwargs": { + "prefix": { + "str": "" + }, + "recurse": { + "bool": true + }, + "remove_duplicate": { + "bool": true + } + } + }, + "control": { + "parent_call_id": "3490429521a1490fb8fdab3ab3bb7098_23960493769087879", + "child_call_id": "3490429521a1490fb8fdab3ab3bb7098_23960493769353813", + "same_process_and_thread": true, + "strictly_within_parent_times": true, + "self_argument_differences": { + "_TRAINCHECK_src_mask_ID": { + "expected": 140489496626592, + "actual": 140261014914224 + } + }, + "child_raw_args": { + "0": { + "trace_impact_benchmark.TransformerModel": { + "batch_first": false, + "call_super_init": false, + "d_model": 32, + "dump_patches": false, + "model_type": "Transformer", + "nhead": 2, + "ninp": 32, + "_TRAINCHECK_src_mask_ID": 140261014914224, + "training": true + } + }, + "1": { + "torch.nn.modules.module.Module.named_parameters..": {} + } + }, + "child_raw_kwargs": { + "prefix": { + "str": "" + }, + "recurse": { + "bool": true + }, + "remove_duplicate": { + "bool": true + } + } + }, + "holdout": { + "parent_call_id": "e4ac9bdf7ba245438bdc313f1b54c933_23960502637904174", + "child_call_id": "e4ac9bdf7ba245438bdc313f1b54c933_23960502638163694", + "same_process_and_thread": true, + "strictly_within_parent_times": true, + "self_argument_differences": { + "_TRAINCHECK_src_mask_ID": { + "expected": 140489496626592, + "actual": 139743891407568 + } + }, + "child_raw_args": { + "0": { + "trace_impact_benchmark.TransformerModel": { + "batch_first": false, + "call_super_init": false, + "d_model": 32, + "dump_patches": false, + "model_type": "Transformer", + "nhead": 2, + "ninp": 32, + "_TRAINCHECK_src_mask_ID": 139743891407568, + "training": true + } + }, + "1": { + "torch.nn.modules.module.Module.named_parameters..": {} + } + }, + "child_raw_kwargs": { + "prefix": { + "str": "" + }, + "recurse": { + "bool": true + }, + "remove_duplicate": { + "bool": true + } + } + } + } + }, + { + "index": 525, + "rule": { + "text_description": "clip_grad.clip_grad_norm_() always calls Module.named_parameters()", + "relation": "APIContainRelation", + "params": [ + { + "param_type": "APIParam", + "api_full_name": "torch.nn.utils.clip_grad.clip_grad_norm_" + }, + { + "param_type": "APIParam", + "api_full_name": "torch.nn.modules.module.Module.named_parameters", + "arguments": { + "args": { + "recurse": { + "bool": true + }, + "self": { + "trace_impact_benchmark.TransformerModel": { + "batch_first": false, + "call_super_init": false, + "d_model": 32, + "dump_patches": false, + "model_type": "Transformer", + "nhead": 2, + "ninp": 32, + "_TRAINCHECK_src_mask_ID": 140489496626592, + "training": true + } + } + }, + "func_name": "torch.nn.modules.module.Module.named_parameters" + } + } + ], + "precondition": { + "parent_func_call_pre": { + "inverted": false, + "preconditions": [ + { + "clauses": "Unconditional" + } + ] + } + }, + "num_positive_examples": 10, + "num_negative_examples": 0 + }, + "evidence": { + "reference": { + "parent_call_id": "1ea3c64687494354b9700101b1e519a0_23960484884548567", + "child_call_id": "1ea3c64687494354b9700101b1e519a0_23960484884631538", + "same_process_and_thread": true, + "strictly_within_parent_times": true, + "self_argument_differences": {}, + "child_raw_args": { + "0": { + "trace_impact_benchmark.TransformerModel": { + "batch_first": false, + "call_super_init": false, + "d_model": 32, + "dump_patches": false, + "model_type": "Transformer", + "nhead": 2, + "ninp": 32, + "_TRAINCHECK_src_mask_ID": 140489496626592, + "training": true + } + } + }, + "child_raw_kwargs": { + "recurse": { + "bool": true + } + } + }, + "control": { + "parent_call_id": "3490429521a1490fb8fdab3ab3bb7098_23960493769087879", + "child_call_id": "3490429521a1490fb8fdab3ab3bb7098_23960493769166145", + "same_process_and_thread": true, + "strictly_within_parent_times": true, + "self_argument_differences": { + "_TRAINCHECK_src_mask_ID": { + "expected": 140489496626592, + "actual": 140261014914224 + } + }, + "child_raw_args": { + "0": { + "trace_impact_benchmark.TransformerModel": { + "batch_first": false, + "call_super_init": false, + "d_model": 32, + "dump_patches": false, + "model_type": "Transformer", + "nhead": 2, + "ninp": 32, + "_TRAINCHECK_src_mask_ID": 140261014914224, + "training": true + } + } + }, + "child_raw_kwargs": { + "recurse": { + "bool": true + } + } + }, + "holdout": { + "parent_call_id": "e4ac9bdf7ba245438bdc313f1b54c933_23960502637904174", + "child_call_id": "e4ac9bdf7ba245438bdc313f1b54c933_23960502637977940", + "same_process_and_thread": true, + "strictly_within_parent_times": true, + "self_argument_differences": { + "_TRAINCHECK_src_mask_ID": { + "expected": 140489496626592, + "actual": 139743891407568 + } + }, + "child_raw_args": { + "0": { + "trace_impact_benchmark.TransformerModel": { + "batch_first": false, + "call_super_init": false, + "d_model": 32, + "dump_patches": false, + "model_type": "Transformer", + "nhead": 2, + "ninp": 32, + "_TRAINCHECK_src_mask_ID": 139743891407568, + "training": true + } + } + }, + "child_raw_kwargs": { + "recurse": { + "bool": true + } + } + } + } + } + ], + "per_invariant_ledger": { + "file": "fresh-run-invariant-validation-transformer.jsonl", + "sha256": "2e7ec6dbc35408bb1a39bff7be96943e00a64cf9d464c6091d54f275d06534f9", + "format": "One rule per line; run arrays use coverage_columns in this document; null means not applicable/unobserved, never zero." + }, + "literal_object_ID_argument_false_alarm_rules": 2 + } + }, + "limitations": [ + "One reference/control/holdout per workload; no population generalization estimate.", + "Fresh initialization and training RNG with longer horizon; not disjoint-dataset validation.", + "Offline checking; no online, distributed, performance, or inference-memory claim.", + "Faults alter recorded metadata only, not real training execution.", + "Active means at least one actual predicate decision; it does not mean every opportunity was evaluated.", + "Checker stops at first violation per rule; violation counts are failing invariants, not all bad events.", + "No raw process/thread clauses were inferred here; original TorchTitan literal-thread-ID rules remain untested.", + "The 74b3adf MNIST attempt stopped after healthy phases because its shape-only fault selector had no eligible rule; preserved separately." + ], + "metric_semantics": { + "identity_bearing_invariants": "Frozen checker counter means raw process/thread-ID PRECONDITION clauses only; excludes object IDs in API argument selectors.", + "identity_bearing_active": "Active rules among that same narrow PID/TID-precondition group.", + "fully_validated_active": "At least one completed decision and no requested proxy-check skip; does not mean all opportunities were evaluated." + } +} diff --git a/docs/measurements/data/fresh-validation-contracts-before-after.json b/docs/measurements/data/fresh-validation-contracts-before-after.json new file mode 100644 index 00000000..f5120925 --- /dev/null +++ b/docs/measurements/data/fresh-validation-contracts-before-after.json @@ -0,0 +1,35 @@ +[ + { + "fixture": "renamed_local_checker", + "before_status": "completed", + "before_records": [ + { + "func_call_id": null + } + ], + "after_status": "observer_error", + "after_records": [] + }, + { + "fixture": "malformed_local_checker", + "before_status": "completed", + "before_records": [ + { + "func_call_id": null + } + ], + "after_status": "observer_error", + "after_records": [] + }, + { + "fixture": "catching_local_checker", + "before_status": "completed", + "before_records": [ + { + "func_call_id": null + } + ], + "after_status": "observer_error", + "after_records": [] + } +] diff --git a/docs/measurements/data/fresh-validation-hardening-integration.json b/docs/measurements/data/fresh-validation-hardening-integration.json new file mode 100644 index 00000000..9b2147cf --- /dev/null +++ b/docs/measurements/data/fresh-validation-hardening-integration.json @@ -0,0 +1,186 @@ +{ + "scope": "Frozen saved-study observer verification; no new training or performance claim", + "harness_source_commit": "f3804c0386454ddd1f97bf17b7a1ab9287cd23d2", + "source_composition": { + "inference": "ce25c12c7fbd9983bb78f2c4ed89842f23499228", + "tracing": "b64023b0651278ab987d3276aaad9141e8bf4894", + "state_patch": "aec748ea46b35b6b43652985f00820afa813175c" + }, + "git_history": "No local merge of #144 into #148; updated production source validated in an isolated snapshot", + "schema_version": 2, + "comparison": "Every original per-invariant field, summary field, raw trace PID/TID, input hash, row count and checking mode preserved; additive schema fields allowed. JSON NaN tokens compare as retained NaN, without deleting/coercing fields. Only operational PID/runtime-source and harness-provenance fields are excluded from semantic equality. Runtime source path verified explicitly.", + "tests": { + "composed_production_linux": 140, + "harness_linux": 28, + "stronger_mypy_1_9_files": 9 + }, + "runtime": { + "python": "3.11.4", + "torch": "2.16.0.dev20261012+cpu", + "pandas": "2.3.3", + "numpy": "2.1.0" + }, + "per_rule_reports": 3072, + "runs": [ + { + "workload": "mnist", + "phase": "check-reference", + "equal_existing_semantic_fields": true, + "inputs_unchanged": true, + "summary": { + "completed": 204, + "active_invariants": 204, + "dormant_invariants": 0, + "identity_bearing_active": 0, + "violated_invariants": 0, + "identity_bearing_invariants": 0, + "coverage_unobserved_invariants": 0, + "check_outcome_unknown_invariants": 0, + "observed_violated_invariants": 0, + "observed_active_invariants": 204, + "observed_dormant_invariants": 0, + "observed_identity_bearing_active": 0 + } + }, + { + "workload": "mnist", + "phase": "check-control", + "equal_existing_semantic_fields": true, + "inputs_unchanged": true, + "summary": { + "completed": 204, + "active_invariants": 204, + "dormant_invariants": 0, + "identity_bearing_active": 0, + "violated_invariants": 0, + "identity_bearing_invariants": 0, + "coverage_unobserved_invariants": 0, + "check_outcome_unknown_invariants": 0, + "observed_violated_invariants": 0, + "observed_active_invariants": 204, + "observed_dormant_invariants": 0, + "observed_identity_bearing_active": 0 + } + }, + { + "workload": "mnist", + "phase": "check-holdout", + "equal_existing_semantic_fields": true, + "inputs_unchanged": true, + "summary": { + "completed": 204, + "active_invariants": 204, + "dormant_invariants": 0, + "identity_bearing_active": 0, + "violated_invariants": 0, + "identity_bearing_invariants": 0, + "coverage_unobserved_invariants": 0, + "check_outcome_unknown_invariants": 0, + "observed_violated_invariants": 0, + "observed_active_invariants": 204, + "observed_dormant_invariants": 0, + "observed_identity_bearing_active": 0 + } + }, + { + "workload": "mnist", + "phase": "check-fault", + "equal_existing_semantic_fields": true, + "inputs_unchanged": true, + "summary": { + "completed": 204, + "active_invariants": 204, + "dormant_invariants": 0, + "identity_bearing_active": 0, + "violated_invariants": 3, + "identity_bearing_invariants": 0, + "coverage_unobserved_invariants": 0, + "check_outcome_unknown_invariants": 0, + "observed_violated_invariants": 3, + "observed_active_invariants": 204, + "observed_dormant_invariants": 0, + "observed_identity_bearing_active": 0 + } + }, + { + "workload": "transformer", + "phase": "check-reference", + "equal_existing_semantic_fields": true, + "inputs_unchanged": true, + "summary": { + "completed": 564, + "active_invariants": 564, + "dormant_invariants": 0, + "identity_bearing_active": 0, + "violated_invariants": 0, + "identity_bearing_invariants": 0, + "coverage_unobserved_invariants": 0, + "check_outcome_unknown_invariants": 0, + "observed_violated_invariants": 0, + "observed_active_invariants": 564, + "observed_dormant_invariants": 0, + "observed_identity_bearing_active": 0 + } + }, + { + "workload": "transformer", + "phase": "check-control", + "equal_existing_semantic_fields": true, + "inputs_unchanged": true, + "summary": { + "completed": 564, + "active_invariants": 564, + "dormant_invariants": 0, + "identity_bearing_active": 0, + "violated_invariants": 2, + "identity_bearing_invariants": 0, + "coverage_unobserved_invariants": 0, + "check_outcome_unknown_invariants": 0, + "observed_violated_invariants": 2, + "observed_active_invariants": 564, + "observed_dormant_invariants": 0, + "observed_identity_bearing_active": 0 + } + }, + { + "workload": "transformer", + "phase": "check-holdout", + "equal_existing_semantic_fields": true, + "inputs_unchanged": true, + "summary": { + "completed": 564, + "active_invariants": 564, + "dormant_invariants": 0, + "identity_bearing_active": 0, + "violated_invariants": 2, + "identity_bearing_invariants": 0, + "coverage_unobserved_invariants": 0, + "check_outcome_unknown_invariants": 0, + "observed_violated_invariants": 2, + "observed_active_invariants": 564, + "observed_dormant_invariants": 0, + "observed_identity_bearing_active": 0 + } + }, + { + "workload": "transformer", + "phase": "check-fault", + "equal_existing_semantic_fields": true, + "inputs_unchanged": true, + "summary": { + "completed": 564, + "active_invariants": 564, + "dormant_invariants": 0, + "identity_bearing_active": 0, + "violated_invariants": 4, + "identity_bearing_invariants": 0, + "coverage_unobserved_invariants": 0, + "check_outcome_unknown_invariants": 0, + "observed_violated_invariants": 4, + "observed_active_invariants": 564, + "observed_dormant_invariants": 0, + "observed_identity_bearing_active": 0 + } + } + ] +} diff --git a/docs/measurements/fresh-run-invariant-validation.md b/docs/measurements/fresh-run-invariant-validation.md new file mode 100644 index 00000000..80b02f75 --- /dev/null +++ b/docs/measurements/fresh-run-invariant-validation.md @@ -0,0 +1,144 @@ +# Fresh-process invariant validation + +Fresh-process checking works for the tested MNIST rules, but exposes two object-ID +false alarms in the small Transformer rules. This is a reproducible offline +validation harness and a bounded result, not a generalization or performance claim. +No production code changes are included. + +| Workload | Learned rules | Reference active / violations | Fresh same-seed control active / violations | Longer changed-seed holdout active / violations | Dormant rules | Requested proxy checks skipped | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| MNIST, 1,199,882 parameters | 204 | 204 / 0 | 204 / 0 | 204 / 0 | 0 | 0 | +| Transformer, 36,197 parameters | 564 | 564 / 0 | 564 / 2 | 564 / 2 | 0 | 0 | + +Every active rule completed at least one actual relation decision. That does not +mean every candidate opportunity was evaluated: for example, MNIST holdout IO +rules have 3,880 opportunities, 3,880 precondition passes, and 3,760 comparisons +because missing values skip 120 comparisons. Rules stop on their first violation. +There were no checker errors or unsupported-coverage results. The direct checker +without observers produced identical CheckerResult structures on both holdouts. + +For both workloads, the same-seed fresh control's complete loss trajectory and +initial/final canonical parameter hashes exactly match the reference. The +instrumented 20-step holdout likewise exactly matches its uninstrumented baseline. +These comparisons apply to these CPU runs and are not a claim of universal +instrumentation transparency. + +The two Transformer alarms are unconditional APIContain rules asserting that +`clip_grad_norm_` contains `Module.named_parameters` / `Module._named_members` with +particular arguments. Both child calls are present in the failed parent's strict +time interval, on the same process/thread. Their recorded model arguments differ +only in `_TRAINCHECK_src_mask_ID`: the reference's memory-object ID was embedded +literally into both learned child selectors. Fresh processes have different IDs. +Thus both rules fail even on the same-seed control despite identical losses and +parameters. The evidence retains the frozen selectors and raw child arguments; +no IDs or learned rules were rewritten. This is an object-identity portability +failure, distinct from thread-ID precondition portability. + +The post-hoc single-field controls activate a passing frozen rule and violate it +at the edited call. MNIST changes one recorded output dtype from `torch.float32` +to `torch.float64`: one output rule plus two IO rules fail (3 new violations). +Transformer changes one output shape from `[16, 2, 32]` to `[17, 2, 32]`: two rules +newly fail, in addition to the 2 existing healthy alarms (4 total). These are +trace-metadata sensitivity controls, not injected training bugs. They establish +neither broad defect coverage nor the consistency of an alternative tensor execution. + +The [compact evidence](data/fresh-run-invariant-validation.json) records source +commits/hashes, data hashes, actual runtime, paired parameter hashes/losses, raw +process/thread IDs, fault edits, family totals, and explicit failed-attempt history. +The [MNIST ledger](data/fresh-run-invariant-validation-mnist.jsonl) and +[Transformer ledger](data/fresh-run-invariant-validation-transformer.jsonl) retain +every rule's opportunities, group-level precondition counts, completed decisions, +visited/unvisited counts, violations, and zeros for all four checked traces. +Their run arrays use the evidence JSON's `coverage_columns`; null means not +applicable/unobserved. `fully_validated_active` means an active rule without a +requested proxy-check skip; it does not promise all candidates were checked. + +Final runs use signed harness `cad62cc98ef5fb7b313325443caa33f715ed141e`, Python +3.11.4, PyTorch `2.16.0.dev20261012+cpu`, pandas 2.3.3, and NumPy 2.1.0. Thirteen +focused counter/mutation tests pass locally and on that Linux runtime; 29 composed +production-stack tests also pass there. Both complete study commands exited 0, +with all input/source immutability and comparison gates satisfied. The +`study_completed` status means those gates passed; it does not erase the two +healthy Transformer violations. + +The initial workloads are benchmark-derived MNIST and a small Transformer. +`benchmark_workload.py` extracts the named model definitions from a supplied +TrainCheck-Benchmarks checkout; it never executes the benchmark's top-level CLI +or downloads data. MNIST requires cached real IDX data. The Transformer uses the +checked-in WikiText prefix. See `tools/fresh_validation/ADAPTERS.md` for the original +adaptations. The fixed `data_steps=20` prepares the same corpus/vocabulary/model +shape for every run: reference/control train 10 steps with seed 42; holdout and its +uninstrumented baseline train 20 steps with seed 43. The first half of holdout inputs +overlaps reference inputs; this is not disjoint-dataset validation. + +`prepare_source.py` creates an isolated snapshot from signed source revisions: +identity/cache stack 99d1690, tracing integrity 7d6c683, TorchScript 9ac61d1, +precondition-state fix 1b5aedf (only its two production hunks), and dynamic type-name +fix 2f034ae. The existing deterministic sampling/reservoir changes are retained. +The harness PR contains no production-code change. Earlier failed setup and +historical-compatibility-patch pilots remain separate from this final recipe. + +Example from a checkout containing those commits, on Linux with an existing +compatible CPU PyTorch runtime and benchmark/data files: + +```sh +python tools/fresh_validation/prepare_source.py --repo "$PWD" --output /path/to/new-source +python tools/fresh_validation/run_study.py \ + --source /path/to/new-source --python /path/to/runtime/bin/python \ + --benchmark-root /path/to/TrainCheck-Benchmarks --data-root /path/to/MNIST-cache \ + --output /path/to/new-study --workload mnist +# Use --workload transformer and omit --data-root for the language-model adapter. +``` + +Each phase starts a new process, with native threads limited to one. A monitor +polls every 250 ms and stops the process group after a 600-second timeout, +8 GiB process-group RSS threshold, or 256 MiB API-trace threshold. These are +sampled monitor thresholds, not OS hard resource limits. Inference is +single-worker and disables FunctionLeadRelation/FunctionCoverRelation. These +bounds and observed phase times are operational controls, not performance data. + +Coverage calls the real `inv.check(trace, check_relation_first=False)`. It counts +precondition group verifications separately from actual relation decisions using +AST-located line observations of the installed checker. API opportunities are +calls; DistinctArgument opportunities are within-step argument pairs. The checker +stops at the first violation per invariant, so unvisited remainders are explicit. +A `triggered` flag or zero violations alone does not establish active checking. +Missing tensor values, dormant rules, checker errors, unsupported coverage, and +requested-but-skipped proxy-variable checks are distinguished. Direct checking +without observers must produce exactly the same CheckerResult structures. + +Raw process/thread IDs and frozen predicates are preserved. ID-bearing clause +kinds/values and activation are reported; their mere presence does not establish +that an ID caused dormancy. A workload with no ID-bearing rules leaves the +original TorchTitan rule set's portability unresolved. Neither new workload has +any process/thread-ID-bearing inferred clause. The frozen `identity_bearing_*` +counters measure only PID/TID precondition clauses, not object IDs in API arguments. In particular, these results do not +validate the original 836-rule corpus's three literal-thread-ID APIContain rules +or its fourth nonliteral thread-inequality rule. + +A post-hoc sensitivity control selects a healthy, actually evaluated output-shape +rule, or an output-dtype rule when no eligible shape rule exists. It changes one +recorded shape dimension or float32/float64 dtype value, retaining its JSON type, +IDs and precondition inputs. Every other semantic field and every other raw line +must remain identical. The frozen checker must activate and violate the selected +rule. This is trace-level detector-path validation, not a reproduced training bug +or a general defect-coverage result. Dtype mutation changes recorded metadata only; +the tensor type-name key remains unchanged, so it does not represent a complete +alternative tensor execution. The original 74b3adf MNIST run completed its healthy +phases but stopped at the shape-only selector; that diagnostic remains separate +from fresh runs with the revised selector. + +`compare_study.py` verifies source/input immutability, fresh producer PIDs, exact +same-seed control losses and canonical parameter hashes, instrumented-versus-plain +holdout losses/hashes, observer/direct equality, and the targeted fault response. +Its `study_completed` status certifies these evidence gates. It does not assert +that every invariant passed the healthy runs; those violations are retained for +analysis. Per-family reports distinguish all active decisions from rules whose +requested checks were fully executed. + +Focused counter tests: + +```sh +OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 \ + python -m unittest tests.test_fresh_validation_coverage tests.test_fresh_validation_fault -v +``` diff --git a/docs/measurements/fresh-validation-contract-hardening.md b/docs/measurements/fresh-validation-contract-hardening.md new file mode 100644 index 00000000..850bba25 --- /dev/null +++ b/docs/measurements/fresh-validation-contract-hardening.md @@ -0,0 +1,120 @@ +# Fresh validation: typing and report contracts + +This hardens the reusable harness introduced in PR #148. The comparison base is +`27e67aafc1a526ed8e69e0a8e59ff30b8ff827cb`. Frozen predicates, training workloads, +and the original measured results are unchanged by this pass. + +## Problem and change + +All 36 original helper functions had incomplete type signatures. Worker/CLI +inputs, model factories, phase results, mutation reports, checker observation +state, and study comparisons now have explicit contracts. The eight helper +modules now contain 61 fully annotated function definitions, including boundary +validators. The new contract tests are annotated too. + +JSON enters through `object` and is narrowed at required-field boundaries. +Missing fields, wrong types, out-of-range invariant indexes, incomplete phases, +and loss rows inconsistent with the claimed step count fail explicitly. The +single `cast` documents a checked string-keyed, read-only mapping boundary; +there are no new `Any` annotations or type-checker ignores. + +The observer now requires each family's exact identity field. API containment +requires `parent_func_call_id`; the three single-call consistency/threshold +families require `func_call_id`; distinct-argument decisions require both event +records and their string call IDs. It never substitutes another family's field +or `None` when checker source changes. The proxy-skip flag is also required when +that precondition is requested. + +`ObservationReport` uses optional detail fields while the mutable report is being +constructed. `finalize_report` enforces the conditional schema before returning +or aggregating it: completed checks require their checker payload and boolean +outcome; observed coverage requires counters, groups, decisions, source metadata, +and the family-specific call/pair fields. A captured observer failure remains +primary even if the checker catches it and later returns or raises a different +exception. The latter exception remains available as secondary error evidence. + +The benchmark adapter validates dynamically loaded model classes and initializer +callbacks. Its helper-import bootstrap appends the harness path, preserving the +separately composed TrainCheck source selected by `PYTHONPATH`. + +## Report compatibility + +Coverage output now declares `schema_version: 2`. Existing completed-check fields, +checker payloads, raw identities, and decision records keep their meaning. +`coverage_status` distinguishes `observed`, `disabled`, `unsupported`, and `error`. +`observer_error` is separate from `checker_error`; either causes a nonzero CLI +result rather than a successful validation. + +Aggregate active/dormant/identity-active totals are `null` when any coverage is +unobserved. `coverage_unobserved_invariants` and `observed_*` subset counts explain +that limit. Likewise, unknown checker outcomes produce a `null` violation total, +with `check_outcome_unknown_invariants` and `observed_violated_invariants` retained. +A direct checker run can have a known outcome while its coverage remains unknown. +These are deliberate schema corrections, not new performance measurements. + +## Evidence + +The [before/after fixtures](data/fresh-validation-contracts-before-after.json) run +both original and hardened observers against three controlled checker-source +changes: a renamed identity local, a `None` identity, and a checker that catches +the observer exception. Originally all three were labeled `completed` and emitted +one `func_call_id: null` decision. After hardening, all three report +`observer_error` and emit no fabricated decisions: 3/3 false completions become +0/3. A fourth regression covers a subsequent different checker exception. + +Validation on Python 3.10.16, Torch 2.2.2, and Pandas 2.2.3: + +- All 28 targeted tests pass: the 13 original checker/fault tests and 15 contract + tests, including observer/direct parity, dormant rules, early failure, + required identities, malformed reports, and composed-source import precedence. +- Pinned Mypy 1.9.0 checks all eight helper modules and the new test module with + `--check-untyped-defs --disallow-untyped-defs --disallow-any-generics + --warn-return-any --explicit-package-bases --follow-imports=silent + --ignore-missing-imports`. Its `--python-executable` points at the installed + TrainCheck runtime, so Torch annotations resolve. This is a stronger scoped + check, not a claim of repository-wide `--strict` compliance. +- Configured Ruff 0.3.3, Black 24.3.0, and isort 5.10.1 hooks pass on the changed + paths. The configured mypy hook does not select `tools/`; the explicit command + above covers that gap. `git diff --check` passes. + +Run the focused tests with: + +```sh +OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 \ +python -m unittest tests.test_fresh_validation_coverage \ + tests.test_fresh_validation_fault tests.test_fresh_validation_contracts -q +``` + +The source-builder recipe remains pinned to its original, reproducible composition. +Full saved-trace replay and the newly composed PR stack require separate integration +validation. This change makes no new training-overhead or generalization claim. + +## Updated production-stack integration + +An isolated snapshot combines signed inference `ce25c12`, tracing `b64023b`, +and only the #145 production state-isolation patch from `aec748e`, plus the exact +signed harness source `f3804c0`. It does not change this PR's Git history or its +reproducible original source-builder recipe. The combined production passes 140 +Linux regressions; all 28 harness regressions also pass against that source. +Stronger Mypy 1.9 checks pass all nine harness/test files against the updated +production interfaces with actual Torch type resolution. + +All eight frozen saved-study checker runs (reference, same-seed control, +holdout, and controlled fault for each workload) complete successfully with +unchanged sources/inputs and the expected production module path. Every original +semantic field in **3,072 per-rule reports** matches, including decisions, +coverage, raw trace IDs, checker payloads, and summary counts. Additive schema-2 +fields are allowed; meaningful fields and array order are never dropped. Retained +JSON NaN tokens compare as NaN rather than using Python's non-reflexive float +equality. Operational process IDs and changed harness/source provenance are +separate from semantic equality. + +| Workload | Active rules in each run | Reference violations | Control | Holdout | Fault | +|---|---:|---:|---:|---:|---:| +| MNIST | 204 | 0 | 0 | 0 | 3 | +| Transformer | 564 | 0 | 2 | 2 | 4 | + +The Transformer's two documented healthy false alarms remain visible. These are +saved-input checker replays, not new model training or a generalization claim. +[Compact integration evidence](data/fresh-validation-hardening-integration.json) +records exact source composition, runtime, counts, and comparison rules. diff --git a/docs/measurements/inference-hardening-validation-20261025.json b/docs/measurements/inference-hardening-validation-20261025.json new file mode 100644 index 00000000..85dc80ba --- /dev/null +++ b/docs/measurements/inference-hardening-validation-20261025.json @@ -0,0 +1,61 @@ +{ + "scope": "Engineering hardening verification; one bounded unprofiled replay, not a new before/after performance experiment", + "inference_source_commit": "ce25c12c7fbd9983bb78f2c4ed89842f23499228", + "common_correctness_patch_commit": "aec748ea46b35b6b43652985f00820afa813175c", + "patch_scope": "Only the two production pruning-state hunks, preserving deterministic sampling", + "runtime": { + "python": "3.11.4", + "torch": "2.16.0.dev20261012+cpu", + "pandas": "2.3.3", + "numpy": "2.1.0" + }, + "workload": { + "trace_rows": 573494, + "workers": 1, + "native_threads": 1, + "disabled_relations": [ + "FunctionLeadRelation", + "FunctionCoverRelation" + ] + }, + "success": true, + "source_unchanged": true, + "traces_unchanged": true, + "validation_run_wall_seconds": 76.79424141719937, + "parity": { + "source": "ce25c12 + aec748e production state patch", + "comparison": "Complete structural JSON multisets; object-key and file-line order only; duplicate multiplicity and array order retained", + "outputs": { + "invariants.jsonl": { + "before_records": 836, + "after_records": 836, + "equal": true, + "raw_bytes_equal": true, + "candidate_sha256": "67ae805da54231ecfcfba060280ca8d5a0b2e98b44523f06b7c64490e9e3a8f0" + }, + "invariants.jsonl.failed": { + "before_records": 104, + "after_records": 104, + "equal": true, + "raw_bytes_equal": true, + "candidate_sha256": "173364b2792421ae048975cc5e7c6a89230f74223f166af046f7a7c9c95d11b2" + } + }, + "contexts": { + "contexts-0.json": true, + "contexts-1.json": true + } + }, + "whole_production_checks": { + "mypy_1_9": "77 files pass", + "ruff_0_3_3": "pass", + "isort_5_10_1": "pass", + "black_24_3_0": "3 unchanged legacy files fail; original base has6 failures", + "strict_typing_limit": "Scoped to newly introduced modules/tools/tests; legacy modules retain documented pre-existing diagnostics", + "composition": { + "inference": "ce25c12c7fbd9983bb78f2c4ed89842f23499228", + "tracing": "b64023b0651278ab987d3276aaad9141e8bf4894", + "state_patch": "aec748ea46b35b6b43652985f00820afa813175c" + } + } +} diff --git a/docs/measurements/inference-identity-cache-20261025.json b/docs/measurements/inference-identity-cache-20261025.json new file mode 100644 index 00000000..690fa7fa --- /dev/null +++ b/docs/measurements/inference-identity-cache-20261025.json @@ -0,0 +1,134 @@ +{ + "measurement_date": "2026-10-25", + "scope": "construction plus repeated identity-checked get_meta_vars calls", + "settings": { + "rows": 573494, + "calls": 1000, + "repetitions": 3 + }, + "runtime": { + "python": "3.10.16", + "platform": "macOS-27.0.1-arm64-arm-64bit", + "host_class": "local Darwin arm64 workstation", + "logical_cpu_count": 12, + "pandas": "2.2.3", + "numpy": "1.26.4", + "native_thread_settings": { + "OMP_NUM_THREADS": "1", + "OPENBLAS_NUM_THREADS": "1", + "MKL_NUM_THREADS": "1" + }, + "candidate_source_sha256": "41040a288284dc10041a4c06d8598219605732dd5dcd2ef080b9ed0799387f8a", + "candidate_commit": "504e04d4de1a4168aabb5436f1b236edef1dadd7", + "candidate_source_matches_commit": true, + "legacy_methods_source_sha256": "22a7976d0fc8558b5a3dcc77e9e527572ec1f1e36b15d5e244a4a0b7ec6f44e7" + }, + "input": { + "sha256_pandas_row_hashes": "ce7af99430be7c9494ef8ee397049b77847897dd965c1082a3cda9e88a159f8b", + "query_sha256": "cd7c7c854fec82747a9704c95dc1e897d6f2dca5664b1b9167a8bd845f227384", + "dtypes": { + "time": "int64", + "process_id": "float64", + "thread_id": "float64" + }, + "dataframe_deep_bytes": 13763984, + "context_manager_count": 0, + "process_count": 8, + "thread_count": 16 + }, + "medians": { + "legacy": { + "construction_seconds": 8.187499770428985e-05, + "queries_seconds": 9.489749166001275, + "total_seconds": 9.489831040998979 + }, + "candidate": { + "construction_seconds": 5.3582996770273894e-05, + "queries_seconds": 0.01196087500284193, + "total_seconds": 0.01200404100382002 + } + }, + "trials": [ + { + "repetition": 1, + "position": 1, + "variant": "legacy", + "total_seconds": 9.2562226250011 + }, + { + "repetition": 1, + "position": 2, + "variant": "candidate", + "total_seconds": 0.01413512499857461 + }, + { + "repetition": 2, + "position": 1, + "variant": "candidate", + "total_seconds": 0.010777290997793898 + }, + { + "repetition": 2, + "position": 2, + "variant": "legacy", + "total_seconds": 9.489831040998979 + }, + { + "repetition": 3, + "position": 1, + "variant": "legacy", + "total_seconds": 9.548994832999597 + }, + { + "repetition": 3, + "position": 2, + "variant": "candidate", + "total_seconds": 0.01200404100382002 + } + ], + "correctness": { + "all_results_equal_expected": true, + "result_sha256": "481e48248fde7cbd3011bd60ecb24d4b0ccaa470a6e5a1986acf27bcc3fa9570", + "input_unchanged": true + }, + "validation": { + "local_combined_tests_passed": 43, + "matching_runtime_linux_identity_tests_passed": 11, + "independent_differential_comparisons": 78, + "independent_dtype_cases": 13, + "pickle_protocols": 6, + "spawn_worker_tasks_passed": 2, + "matching_linux_runtime": { + "python": "3.11.4", + "pytorch": "2.16.0.dev20261012+cpu", + "numpy": "2.1.0", + "pandas": "2.3.3" + } + }, + "full_inference": { + "status": "completed_unprofiled_pair", + "evidence": "inference-identity-cache-full-trace-20261025.json", + "baseline_wall_seconds": 1385.656381815672, + "candidate_wall_seconds": 98.55111291632056, + "wall_time_ratio": 14.060281419573906, + "invariant_records": 835, + "failed_hypothesis_records": 105, + "complete_json_multisets_equal": true, + "performance_comparison_valid": true, + "scope": "One sequential saved-trace CPU pair with six relations enabled; no repeated-run distribution or training-overhead claim.", + "profile_attempts": "Two earlier periodic-faulthandler plus cProfile attempts exited SIGSEGV without completed outputs and are excluded." + }, + "limitations": [ + "Synthetic identity-lookup microbenchmark, not completed inference.", + "No context-manager work or trace-file loading; input generation excluded.", + "No RSS/PSS, peak-memory, or multiprocessing memory-scaling measurement.", + "Cache stores O(P+T) values and returns fresh O(P+T) lists.", + "Stable identity values, dtypes, and row order required within one frame." + ], + "source_evidence_sha256": { + "benchmark-full.json": "7c3b84db5645226b4d841b929185edad33c075c7868ad9951607a57b2852febe", + "review-results.json": "19e4c7bf403a8a3732e8dbf0fda3ab4f6902b99c0e997dc490d9383ffc406376", + "combined-tests.log": "6334a444075d2bd539f965998354922ea2d940d74536d61790e39e840bcc345c", + "matching-runtime-linux-tests.log": "56b695709855bb8a17cd0fbf58e1bf3e7d17ed228719d5cc2350d3718f8e3604" + } +} diff --git a/docs/measurements/inference-identity-cache-full-trace-20261025.json b/docs/measurements/inference-identity-cache-full-trace-20261025.json new file mode 100644 index 00000000..99c1b289 --- /dev/null +++ b/docs/measurements/inference-identity-cache-full-trace-20261025.json @@ -0,0 +1,247 @@ +{ + "measurement_date": "2026-10-25", + "scope": "One baseline-first sequential unprofiled pair on the same saved training trace; no repeats, affinity, or exclusive host reservation.", + "workload": { + "model_parameters": 6163712, + "training_steps": 10, + "sequence_length": 256, + "batch_size": 1, + "seed": 42, + "events": 573494, + "raw_trace_bytes": 286335289, + "training_run": "20261012T163111Z-88c4d357", + "trace_files": { + "trace_API_1088885_23438542501568.log": { + "bytes": 82668375, + "sha256": "883d288d6955b12c49dc741c166bcca7d191c890d19565802dad60708979278d" + }, + "trace_API_1088885_23456244348224.log": { + "bytes": 203666914, + "sha256": "416b8ca45b4df361fc73052447573d7cb23d433d16c0947d74bbc18f7cc6c9a8" + } + } + }, + "settings": { + "backend": "pandas", + "workers": 1, + "native_threads": 1, + "python_hash_seed": 0, + "gpu": false, + "disabled_relations": [ + "FunctionLeadRelation", + "FunctionCoverRelation" + ], + "timeout_seconds": 1800, + "sampled_child_session_pss_limit_gib": 12, + "memory_sampling_seconds": 1, + "profile_enabled": false, + "periodic_traceback_dumps": false, + "eligible_api_count": 41 + }, + "runtime": { + "host": "labmach", + "python": "3.11.4", + "torch": "2.16.0.dev20261012+cpu", + "torch_git_version": "8f9045d72f71e7aa9e16b6d7a055ec5bc1f720b5", + "pandas": "2.3.3", + "numpy": "2.1.0", + "original_trace_runtime_difference": "Original training used Python 3.12.3 and a CUDA 13.0 Torch build at the same Torch source revision. Original eligible API name set was not logged; only its count of 41 is available." + }, + "sources": { + "baseline": "Frozen call-index snapshot underlying PR141; before publication-only typing/comment formatting changes.", + "candidate_commit": "504e04d4de1a4168aabb5436f1b236edef1dadd7", + "trace_pandas_sha256": { + "baseline": "8e4c3589371449b78430ed5039d0159d3d0435664b218f3a96c81935be0d75d5", + "candidate": "41040a288284dc10041a4c06d8598219605732dd5dcd2ef080b9ed0799387f8a" + }, + "provenance_manifest_sha256": { + "baseline": "7e8e267085ece31844a820752004af6a1c5bb293a3a2733990b039d72fcd872a", + "candidate": "b48d705bd9dccbdf7bce17e8704e7ae5554a7d6239cfd545d7b5ffc66a888b09" + }, + "candidate_production_delta": "Identity caching plus publication-only import/type-annotation changes, comments/docstrings and formatting; all other production behavior unchanged. Independent pruning-state fix PR145 excluded from both variants." + }, + "comparison": { + "complete_pair": true, + "scope": "one sequential unprofiled run per source; no replicate/median claim", + "harness_sha256": "2dd8d63a8bae80a944defee70be6bf33e9c9c8934e22e4b909ff222b4f7d6777", + "baseline": { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "f3872b21c9ed7103e1020ccb83ef54b41009f2ec06348ce1b7dc5893e545f291", + "distinct_records": 105, + "records": 105 + }, + "invariants": { + "canonical_multiset_sha256": "f4f5db6772b857074d113e1c6ea091d5c4ed2452e091607fa425efab4edeed65", + "distinct_records": 835, + "records": 835 + }, + "sampled_peak_concurrent_memory": { + "pss_bytes": 2283671552, + "rss_bytes": 2291527680, + "uss_bytes": 2280935424 + }, + "source_unchanged": true, + "stage_times": { + "marker_durations": { + "cross_trace_collection_seconds": 0.001, + "generation_and_pruning_seconds": 619.58, + "precondition_and_output_sort_seconds": 751.403 + }, + "markers": { + "collection_start": "2026-10-25T03:37:57.781000", + "generation_start": "2026-10-25T03:27:38.201000", + "inference_end": "2026-10-25T03:50:29.185000", + "precondition_start": "2026-10-25T03:37:57.782000" + }, + "note": "Log marker intervals; missing endpoints mean incomplete phases, not zero duration.", + "reported": { + "inference_seconds": 1371.0323550701141, + "load_seconds": 10.503035306930542 + } + }, + "stop_reason": null, + "success": true, + "traces_unchanged": true, + "wall_seconds": 1385.656381815672 + }, + "candidate": { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "f3872b21c9ed7103e1020ccb83ef54b41009f2ec06348ce1b7dc5893e545f291", + "distinct_records": 105, + "records": 105 + }, + "invariants": { + "canonical_multiset_sha256": "f4f5db6772b857074d113e1c6ea091d5c4ed2452e091607fa425efab4edeed65", + "distinct_records": 835, + "records": 835 + }, + "sampled_peak_concurrent_memory": { + "pss_bytes": 2286718976, + "rss_bytes": 2294530048, + "uss_bytes": 2283991040 + }, + "source_unchanged": true, + "stage_times": { + "marker_durations": { + "cross_trace_collection_seconds": 0.0, + "generation_and_pruning_seconds": 72.759, + "precondition_and_output_sort_seconds": 10.913 + }, + "markers": { + "collection_start": "2026-10-25T03:51:57.205000", + "generation_start": "2026-10-25T03:50:44.446000", + "inference_end": "2026-10-25T03:52:08.118000", + "precondition_start": "2026-10-25T03:51:57.205000" + }, + "note": "Log marker intervals; missing endpoints mean incomplete phases, not zero duration.", + "reported": { + "inference_seconds": 83.71693992614746, + "load_seconds": 10.698827981948853 + } + }, + "stop_reason": null, + "success": true, + "traces_unchanged": true, + "wall_seconds": 98.55111291632056 + }, + "invariants": { + "comparison": "structural_json_multiset", + "equal": true, + "reference": { + "records": 835, + "distinct_records": 835, + "canonical_multiset_sha256": "f4f5db6772b857074d113e1c6ea091d5c4ed2452e091607fa425efab4edeed65" + }, + "candidate": { + "records": 835, + "distinct_records": 835, + "canonical_multiset_sha256": "f4f5db6772b857074d113e1c6ea091d5c4ed2452e091607fa425efab4edeed65" + }, + "missing_records": 0, + "extra_records": 0, + "missing_examples": [], + "extra_examples": [] + }, + "failed_hypotheses": { + "comparison": "structural_json_multiset", + "equal": true, + "reference": { + "records": 105, + "distinct_records": 105, + "canonical_multiset_sha256": "f3872b21c9ed7103e1020ccb83ef54b41009f2ec06348ce1b7dc5893e545f291" + }, + "candidate": { + "records": 105, + "distinct_records": 105, + "canonical_multiset_sha256": "f3872b21c9ed7103e1020ccb83ef54b41009f2ec06348ce1b7dc5893e545f291" + }, + "missing_records": 0, + "extra_records": 0, + "missing_examples": [], + "extra_examples": [] + }, + "wall_time_ratio_baseline_over_candidate": 14.060281419573906 + }, + "context_index": { + "baseline": [ + { + "call": 0, + "contexts": 711, + "rows": 573494, + "seconds": 0.46406279876828194, + "states_sha256": "02a9f08ad19e890f92d26367b4a8c433c4953770d5359554bf496fc6f17e3b07" + }, + { + "call": 1, + "contexts": 711, + "rows": 573494, + "seconds": 0.4610072821378708, + "states_sha256": "02a9f08ad19e890f92d26367b4a8c433c4953770d5359554bf496fc6f17e3b07" + } + ], + "candidate": [ + { + "call": 0, + "contexts": 711, + "rows": 573494, + "seconds": 0.4815337397158146, + "states_sha256": "02a9f08ad19e890f92d26367b4a8c433c4953770d5359554bf496fc6f17e3b07" + }, + { + "call": 1, + "contexts": 711, + "rows": 573494, + "seconds": 0.46630632504820824, + "states_sha256": "02a9f08ad19e890f92d26367b4a8c433c4953770d5359554bf496fc6f17e3b07" + } + ] + }, + "verification": { + "complete_structural_json_multisets_equal": true, + "duplicate_multiplicity_and_array_order_preserved": true, + "json_object_key_order_and_record_line_order_ignored": true, + "both_full_and_stage_context_files_byte_identical": true, + "all_source_and_input_hashes_unchanged": true + }, + "timer_scope": { + "wall": "Child startup through exit and monitor polling, including imports and JSON serialization; initial/final integrity hashing excluded.", + "load": "Read/concatenate/TracePandas construction, including harness context serialization.", + "inference": "Engine construction, hypothesis generation/pruning, collection, final preconditions and sort; ends before invariant JSON serialization.", + "marker_intervals": "Millisecond log timestamp resolution; generation/pruning and final preconditions/sort are parts of inference, not additional wall time." + }, + "memory_scope": "Peak concurrent child-session samples, excluding monitor; PSS depends on shared pages. No memory improvement or multiprocessing scaling claim.", + "evidence_archive": { + "name": "matching-runtime-unprofiled-pair-evidence.tar.gz", + "bytes": 314914, + "sha256": "d78a815e1516131b28342c7d7e9251bc195f9708bab247c9f2d184558ec26f9e" + }, + "limitations": [ + "One sequential pair, no distribution or confidence interval.", + "Offline saved-trace CPU inference, not training, distributed scaling, online checking overhead, or new trace collection.", + "Six enabled relations; FunctionLeadRelation and FunctionCoverRelation disabled identically.", + "This preserves existing inference behavior; it does not certify inferred invariants as ground truth.", + "Earlier periodic-faulthandler plus cProfile attempts crashed and are excluded; this successful pair alone does not isolate the crash cause." + ] +} diff --git a/docs/measurements/inference-identity-cache.md b/docs/measurements/inference-identity-cache.md new file mode 100644 index 00000000..7cf62e09 --- /dev/null +++ b/docs/measurements/inference-identity-cache.md @@ -0,0 +1,119 @@ +# Cache process and thread identities during precondition inference + +`get_meta_vars` checks process and thread membership for each request. Previously, +each check repeated `dropna().unique().tolist()` over the full event column. +The change caches those exact pandas results lazily per trace and returns a fresh +list to each caller. Null filtering, scalar types, encounter order, and independent +process/thread membership remain unchanged. Frame replacement or length changes +invalidate both identity caches. + +## Complete saved-trace inference + +On labmach, one sequential unprofiled pair reduced total wall time from +**1,385.656 s to 98.551 s (14.06x)**. Both runs completed successfully. All **835 +invariant records and 105 saved failed-hypothesis records** match as complete +structural JSON multisets, preserving duplicate occurrences and array order. +Object key order and record line order are ignored. No records were missing or +extra. Both full/stage context files are byte-identical (711 states each), and +all source and input hashes remained unchanged. + +| Measurement | Baseline | Identity cache | +|---|---:|---:| +| Total child wall time | 1,385.656 s | 98.551 s | +| Trace loading/construction | 10.503 s | 10.699 s | +| Reported inference | 1,371.032 s | 83.717 s | +| Generation and pruning (part of inference) | 619.580 s | 72.759 s | +| Final preconditions and sort (part of inference) | 751.403 s | 10.913 s | +| Peak sampled child-session PSS | 2.127 GiB | 2.130 GiB | + +The workload is the two complete trace files from the saved 6,163,712-parameter, +10-step reference training run: 573,494 events and 286,335,289 raw bytes, sequence +length 256, batch size 1, seed 42. No training was rerun. Both variants use the +context/call indexes from PR #141; this comparison isolates the identity cache, +with only publication-time imports/type annotations, comments/docstrings, and +formatting also differing. +The independent pruning-state correctness fix in PR #145 is excluded from both +variants and evaluated separately. + +Both runs used pandas, one worker, native thread settings of one, no GPU, +`PYTHONHASHSEED=0`, and the same six enabled relations. `FunctionLeadRelation` +and `FunctionCoverRelation` were disabled. Both had a 1,800-second cutoff and a +12-GiB sampled child-session PSS limit. The runtime was Python 3.11.4, Torch +2.16.0.dev20261012+cpu, pandas 2.3.3, and NumPy 2.1.0. Torch's source revision +matches the original CUDA wheel, but the original training used Python 3.12.3 +and CUDA 13.0. The eligible API count is 41 in both logs; the original name set +was not recorded, so exact original-environment coverage is not established. + +This is one baseline-first pair without CPU affinity or exclusive host access; +it is not a repeated-run speedup estimate. Total wall time includes child startup, +imports, JSON serialization, and monitor polling (about one-second resolution), +but excludes initial/final integrity hashing. The load timer includes harness +context serialization. The inference timer ends before invariant JSON writes; +its two large phases above are log-marker intervals, not additional time. Memory +is sampled every second for the concurrent child session, excludes the monitor, +and depends on shared pages; no memory reduction is established. + +The result concerns offline inference over this saved trace and enabled relation +set. It does not establish training overhead, distributed scaling, or correctness +of the inferred invariants against ground truth. Earlier attempts combining +cProfile and periodic traceback dumps exited with SIGSEGV and are excluded. Both +were disabled for this timed pair; its success alone does not identify the crash +cause. + +[Full-trace measurements and provenance](inference-identity-cache-full-trace-20261025.json) +include phase times, output digests, trace/source hashes, runtime, and archive +identity. The retained evidence archive contains both complete output sets, +context files, stdout logs, memory samples, and invocation/provenance manifests. + +## Isolated measurement + +On a deterministic 573,494-row synthetic trace, the median time for construction +plus 1,000 `get_meta_vars` calls fell from **9.489831 s to 0.012004 s** across three +alternating repetitions. Each trial starts with a fresh trace and includes the +first cache fill. The baseline restores only the two original identity methods; +both variants use the same constructor and other methods. + +| Repetition | Legacy | Cache | +|---|---:|---:| +| 1 (legacy first) | 9.256223 s | 0.014135 s | +| 2 (cache first) | 9.489831 s | 0.010777 s | +| 3 (legacy first) | 9.548995 s | 0.012004 s | + +The shared input has eight process IDs, sixteen thread IDs, deterministic missing +values, and no context managers. All six ordered outputs match: 750 empty +dictionaries and 250 `None` values; input hashes remain unchanged. Input generation, +garbage collection, and result hashing are excluded. The run used a macOS arm64 +workstation, Python 3.10.16, pandas 2.2.3, NumPy 1.26.4, and native thread settings +of one. No CPU affinity or exclusive machine access was imposed. + +This isolates identity lookup. The separate complete inference comparison above +measures the saved training trace. + +## Correctness and contract + +All 43 local context/call/identity regression tests and all 11 identity tests in the +matching Linux runtime pass. Independent review adds 78 differential comparisons +across 13 dtype/value cases, six pickle protocols, and two spawned-worker tasks. +The checks cover nulls, integer precision, scalar types, encounter order, missing +columns, independent process/thread membership, fresh return lists, stage isolation, +frame/length invalidation, serialization, and actual precondition inference. + +Identity values, dtypes, and row order must remain stable within a frame. Same-size +in-place edits, column replacement, and reordering require a new `TracePandas`. +The cache stores O(P + T) values; returned lists and membership checks still cost +O(P + T). Its source reference is not a copied DataFrame, but can retain the old +frame until invalidation. No memory or multiprocessing memory-scaling improvement is claimed. + +From the checkout, reproduce the isolated measurement with: + +```sh +OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 \ +python -m tools.benchmarks.identity_cache --rows 573494 --calls 1000 --repetitions 3 + +python -m unittest tests.test_context_index tests.test_call_index tests.test_identity_cache +``` + +Use `--rows 1000 --calls 20` for a smoke test. The script needs no external trace; +it records the checkout commit and source hash and checks output equality. +[Compact evidence and provenance](inference-identity-cache-20261025.json) retain the +measured source commit, runtime, inputs, per-trial timings, and limitations. diff --git a/docs/measurements/inference-time-range-index-20261025.json b/docs/measurements/inference-time-range-index-20261025.json new file mode 100644 index 00000000..2ae789d6 --- /dev/null +++ b/docs/measurements/inference-time-range-index-20261025.json @@ -0,0 +1,872 @@ +{ + "scope": "Three alternating sequential unprofiled CPU saved-trace inference pairs; both variants include the common pruning-state correction.", + "date_utc": "2026-10-25", + "started_utc": "2026-10-25T10:44:10.532938+00:00", + "ended_utc": "2026-10-25T10:53:12.729963+00:00", + "source": { + "base_branch_commit": "99d16905296d83495570fdcafa91370b242d2528", + "candidate_production_commit": "24e7cb0f7a573ff6225a388eff189d9e4af5e8b5", + "candidate_benchmark_commit": "39152aded8b97ed83988e3292bb89c6e3adcf750", + "common_state_fix_commit": "1b5aedffbeefd4cdd9d30036dbb0a833521526cf", + "only_production_difference": [ + "traincheck/trace/trace_pandas.py" + ], + "production_sha256": { + "baseline": { + "traincheck/__init__.py": "47fdcb090c4c8c5822769d642dc23a520e8d8ff3e4724583450723e05408164b", + "traincheck/checker.py": "44dc20fc66ca82bb6b71fa02033aae3b7d489ae7afbc1611c44e857f399716b2", + "traincheck/checker_online.py": "2824c9ccd2f684c86884c68f2214a1a4be1d69c6098ce30b817d03f1fe1077d6", + "traincheck/collect_trace.py": "c1db96c798af9689f8ee7f7b0b929d90694d49a234cb217beaf435d4b8d5cfb0", + "traincheck/config/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/config/config.py": "4c940a4d92f5de5fe8d3885e570078e479d9842d9626651fd60483e78f0160a9", + "traincheck/developer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/developer/annotations.py": "0ff28d35e085b40e1e366499feec96c8eb1e13bf3100dcdd7ab304c7f2f6e963", + "traincheck/infer_engine.py": "476a3ec99bef0ab823b23f94bc95842c20b6d9b7b4a57ab4a2efa2987f30f286", + "traincheck/inference_sampling.py": "ba566c3745ed3043ef27cbc626c21b358d1d01c95a26f1ebc9c8f56391d63681", + "traincheck/instrumentor/__init__.py": "228b4f6a11cde8118cde9aba2f98b60a839f4331e7fa2b06c37cee99e79fcd50", + "traincheck/instrumentor/caches.py": "320eb1ffa1ce5f5b97fe4675a16d47e9591265779fdc29b94b72cdf5df93f168", + "traincheck/instrumentor/control.py": "3612a39b61e85fb77b40008b7fcd2f7569b246e8985ca64a109ea53733cd29ef", + "traincheck/instrumentor/dumper.py": "c78bda77cfe83be10b6b385b55563fa6ca72ecc116f8bcc1557544d2e5d2df98", + "traincheck/instrumentor/proxy_wrapper/__init__.py": "7c71aaf70b978ee04e236497af6f803e4436049bc0acdf18c3edb48bfcece63d", + "traincheck/instrumentor/proxy_wrapper/dumper.py": "6907ac0a198443dd9138a37dc9b5de1232d3d7fb26412f8bba3fb08c85cd222d", + "traincheck/instrumentor/proxy_wrapper/hash.py": "e917bec817df1981c682d8b8645307432c6badb220b9f7c424b4089d9bf97bd7", + "traincheck/instrumentor/proxy_wrapper/proxy.py": "fe241558f0591738391bc57d0666e3b11e04d8579eb220ce5de855abc309baf3", + "traincheck/instrumentor/proxy_wrapper/proxy_basics.py": "f012e97fa51cf77be1806807ef6130b06c9cec6fde8d047d57d45b6ec9fcc6e5", + "traincheck/instrumentor/proxy_wrapper/proxy_config.py": "afb2f8dd04ed10e628ff21339ec8ebef36357fd2a0cc084e99e99e37756cf16b", + "traincheck/instrumentor/proxy_wrapper/proxy_handler.py": "4bab74647d35c64c5e9c702295dc4cb010302bf620401da0fad8539692b95ac7", + "traincheck/instrumentor/proxy_wrapper/proxy_methods.py": "6ae18928aa12e89b33102368c497aff048e3d6bed330eca2118fac051b8e3a97", + "traincheck/instrumentor/proxy_wrapper/proxy_observer.py": "10e83570fbe1f72089e4a594a308f83e045f7d74bb9e49193012645ac906d97b", + "traincheck/instrumentor/proxy_wrapper/proxy_registry.py": "9dfecae65aee4185152d152a5c2fb0e1e1b63339efceab3087df59a5882410a8", + "traincheck/instrumentor/proxy_wrapper/subclass.py": "66c751aa8ff31ff9fbcda0a361be8187c50d6dfec0b339cd218955c5c6978a19", + "traincheck/instrumentor/proxy_wrapper/torch_proxy.py": "d6abd7f2eac207757fd90cbf55e179ae506b25996bed906c5a094564ff6faf2f", + "traincheck/instrumentor/proxy_wrapper/utils.py": "5f058609e89a96a3545561ed82d83f7dd6a153bb0376608654c199691984dc14", + "traincheck/instrumentor/replace_functions.py": "ca4f10b52083a4a5c251db194b7608406cbe36c85372dfdb9df17644f3d20b62", + "traincheck/instrumentor/source_file.py": "dad92c793efb41161d86250e89aa6a9636749564e567df8b9af92a477eb4858d", + "traincheck/instrumentor/tracer.py": "5476b155823b4287f9cb01e498f70b99921063a052512a9f0d5d5b76974850f4", + "traincheck/instrumentor/types.py": "258679aa8d0b40247d900ba2e26ab03b25cb4626b046ab6d724973226c7b5469", + "traincheck/instrumentor/variable.py": "b01219cdd74f011055e0bc2a645ed283dbc47e66a834d5555b814fa815a9eab8", + "traincheck/invariant/DistinctArgumentRelation.py": "6e01540583668e30718ba90c341fd540cd9df53f20525a6cf502fcceee8ed7bc", + "traincheck/invariant/__init__.py": "773e9f767566ffd92658dc0e94107ac840672e446a41ba437a49cee9cc52197a", + "traincheck/invariant/base_cls.py": "31ceb584760d26e9923d2c7ce962fd6f093a3a6ba983ed177fcb4257c1da2466", + "traincheck/invariant/consistency_relation.py": "bc5b14810ea8a5e0e5f3e6c2c6be24d137a444d8609216d00eafd155b4aa3917", + "traincheck/invariant/consistency_transient_vars.py": "a702cd1b6787e8be1904bef9119773c0e037a83017959c771020dbbbbe1549e2", + "traincheck/invariant/contain_relation.py": "b32f819db4d5a11a11318bf7e38c64148f4c9360137ac5f09475b480c34c16a5", + "traincheck/invariant/cover_relation.py": "0eb0e1c30ad5b34a15ce32c7b0d15e894f7a3b9b68bfcaae5b901f5dc6a092c6", + "traincheck/invariant/lead_relation.py": "978a4db32b69dc1a90a84123aa93e52a3caaba97761321dbb844666cb47d4360", + "traincheck/invariant/precondition.py": "c14c7580643780954eab5a4d955b0b5d6a6768a57cd8e5e48328ad0d8f2f8703", + "traincheck/invariant/relation_pool.py": "8ac568cf1924162a791728ed5db8c41868a550f9e0cd5190fc9a19827f55d3e6", + "traincheck/invariant/symbolic_value.py": "28627e2c2a47a146087a96510f07f7deb8463c46371e418afa3fdeff33c7e6cb", + "traincheck/invariant/var_periodic_change_relation.py": "8086df3b9cfe34229b8bd46b18dce22a4b746902e38d1ba4bd02bc6f6c2f2ef4", + "traincheck/onlinechecker/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/onlinechecker/streamhandler_filesystem.py": "b5a298bd06a9408e857ff6721578cb4ae10f227762ba3d66aec685e574cfa511", + "traincheck/onlinechecker/utils.py": "65ba5d8a84b748adf4267800e704df1f4662c0f483534289af20096bf9a78a00", + "traincheck/parallel.py": "6843319dd8c5c397e93f361a29fe564161e7307fbe4f0f8368090d252d4a7bc4", + "traincheck/progress.py": "aab7d040324e283c8474aa82bc7550a777aec316131a26e1dc1b6a277333d1ae", + "traincheck/reporting/__init__.py": "299180153357c4e6f48cf75191e915d70cfca5ae6e7b3ac4d45144066b93ddca", + "traincheck/reporting/checker_report.py": "0d59b1753fc348701b49a0eb74e4b1d849a47d8234005ccbbb26e09e3cd0c808", + "traincheck/runner/__init__.py": "287ffa9cab82f01da0e82faea77589dc7d348710c862496458d2d60c364394ba", + "traincheck/runner/runner.py": "89027b488ded8640fddb387dd80d98f059cf40be5f94863966b259acea1da7c7", + "traincheck/script/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/script/num_invs_each_rel.py": "202f13d536929c52defa84cdce00dec2251ea09aabf3ae8ae57067c32d0b844d", + "traincheck/static_analyzer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/config.py": "840460167bd6d26db429076cd4dc227e6c17c4e5fb157172041c48574f44edf7", + "traincheck/static_analyzer/func_level/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/graph_generator/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/graph_generator/analyzer.py": "035684eb7ea4edf8ee0dbf4c6b12b91b428cd6dd57be5fe658253c21c8c420b1", + "traincheck/static_analyzer/graph_generator/anutils.py": "9e62b7ce5fced45a033b76de3d408bdff2fd8561ffa28bc58de82e4105b51d39", + "traincheck/static_analyzer/graph_generator/call_graph_parser.py": "69e2555a1595078a2d2bca174de32cd7d4b0f023c0a20b68a01ca00d1d3b8647", + "traincheck/static_analyzer/graph_generator/main.py": "a568488c50383a67a6ab8feecaf96e29b288861c1dce17481a1b5032d20748a4", + "traincheck/static_analyzer/graph_generator/node.py": "b6a0cea44a28d33f28fee79b7f7073e69914e95565f7a8d62f9e5a5fd6b240cd", + "traincheck/toolkit/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/toolkit/analyze_invariant.py": "706cf8c06062b6df3f1f86691f588c91696cbef6c27698b2f0b290ad8f53ff4b", + "traincheck/toolkit/analyze_trace.py": "48dfc81ef85d96678b5fe466b86ba80d892ecfc497387773fa7ab5995f22e822", + "traincheck/toolkit/detect_anomaly_from_trace_diff.py": "2ffc0723dd3709ccab09cb9b58d76c7bb0558dda09b66c47184e335b815cedd3", + "traincheck/trace/__init__.py": "ec84edd0ff32906c20956166a51b85b3847e22e48432c6a31698080cafbcbc2b", + "traincheck/trace/trace.py": "eac49a1cb958ccaae57ac93e8aad1cbc616e7fcef23068ef5bb68bbc54890c00", + "traincheck/trace/trace_dict.py": "3aec874ef0ce2c34f6332dface604d5f79965976fd3d1c2b692f746f2af4e703", + "traincheck/trace/trace_pandas.py": "41040a288284dc10041a4c06d8598219605732dd5dcd2ef080b9ed0799387f8a", + "traincheck/trace/trace_polars.py": "64136efcd60a851b0391e708108570b259c2288db6242564b636f69abd1b7b0e", + "traincheck/trace/types.py": "96c1b137f3d72fb05dcf806a4e7ea21fe453ae9cbc3e08352c55e95973fe96e0", + "traincheck/trace/utils.py": "6356156e4c5bbb6c111140302660f36ad1c126e5bc977bd55c81f097022102a3", + "traincheck/utils.py": "c36764ec7b9d7224c82071abf2f83d5ee9952e6868774dff62d851da9d0270e1" + }, + "candidate": { + "traincheck/__init__.py": "47fdcb090c4c8c5822769d642dc23a520e8d8ff3e4724583450723e05408164b", + "traincheck/checker.py": "44dc20fc66ca82bb6b71fa02033aae3b7d489ae7afbc1611c44e857f399716b2", + "traincheck/checker_online.py": "2824c9ccd2f684c86884c68f2214a1a4be1d69c6098ce30b817d03f1fe1077d6", + "traincheck/collect_trace.py": "c1db96c798af9689f8ee7f7b0b929d90694d49a234cb217beaf435d4b8d5cfb0", + "traincheck/config/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/config/config.py": "4c940a4d92f5de5fe8d3885e570078e479d9842d9626651fd60483e78f0160a9", + "traincheck/developer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/developer/annotations.py": "0ff28d35e085b40e1e366499feec96c8eb1e13bf3100dcdd7ab304c7f2f6e963", + "traincheck/infer_engine.py": "476a3ec99bef0ab823b23f94bc95842c20b6d9b7b4a57ab4a2efa2987f30f286", + "traincheck/inference_sampling.py": "ba566c3745ed3043ef27cbc626c21b358d1d01c95a26f1ebc9c8f56391d63681", + "traincheck/instrumentor/__init__.py": "228b4f6a11cde8118cde9aba2f98b60a839f4331e7fa2b06c37cee99e79fcd50", + "traincheck/instrumentor/caches.py": "320eb1ffa1ce5f5b97fe4675a16d47e9591265779fdc29b94b72cdf5df93f168", + "traincheck/instrumentor/control.py": "3612a39b61e85fb77b40008b7fcd2f7569b246e8985ca64a109ea53733cd29ef", + "traincheck/instrumentor/dumper.py": "c78bda77cfe83be10b6b385b55563fa6ca72ecc116f8bcc1557544d2e5d2df98", + "traincheck/instrumentor/proxy_wrapper/__init__.py": "7c71aaf70b978ee04e236497af6f803e4436049bc0acdf18c3edb48bfcece63d", + "traincheck/instrumentor/proxy_wrapper/dumper.py": "6907ac0a198443dd9138a37dc9b5de1232d3d7fb26412f8bba3fb08c85cd222d", + "traincheck/instrumentor/proxy_wrapper/hash.py": "e917bec817df1981c682d8b8645307432c6badb220b9f7c424b4089d9bf97bd7", + "traincheck/instrumentor/proxy_wrapper/proxy.py": "fe241558f0591738391bc57d0666e3b11e04d8579eb220ce5de855abc309baf3", + "traincheck/instrumentor/proxy_wrapper/proxy_basics.py": "f012e97fa51cf77be1806807ef6130b06c9cec6fde8d047d57d45b6ec9fcc6e5", + "traincheck/instrumentor/proxy_wrapper/proxy_config.py": "afb2f8dd04ed10e628ff21339ec8ebef36357fd2a0cc084e99e99e37756cf16b", + "traincheck/instrumentor/proxy_wrapper/proxy_handler.py": "4bab74647d35c64c5e9c702295dc4cb010302bf620401da0fad8539692b95ac7", + "traincheck/instrumentor/proxy_wrapper/proxy_methods.py": "6ae18928aa12e89b33102368c497aff048e3d6bed330eca2118fac051b8e3a97", + "traincheck/instrumentor/proxy_wrapper/proxy_observer.py": "10e83570fbe1f72089e4a594a308f83e045f7d74bb9e49193012645ac906d97b", + "traincheck/instrumentor/proxy_wrapper/proxy_registry.py": "9dfecae65aee4185152d152a5c2fb0e1e1b63339efceab3087df59a5882410a8", + "traincheck/instrumentor/proxy_wrapper/subclass.py": "66c751aa8ff31ff9fbcda0a361be8187c50d6dfec0b339cd218955c5c6978a19", + "traincheck/instrumentor/proxy_wrapper/torch_proxy.py": "d6abd7f2eac207757fd90cbf55e179ae506b25996bed906c5a094564ff6faf2f", + "traincheck/instrumentor/proxy_wrapper/utils.py": "5f058609e89a96a3545561ed82d83f7dd6a153bb0376608654c199691984dc14", + "traincheck/instrumentor/replace_functions.py": "ca4f10b52083a4a5c251db194b7608406cbe36c85372dfdb9df17644f3d20b62", + "traincheck/instrumentor/source_file.py": "dad92c793efb41161d86250e89aa6a9636749564e567df8b9af92a477eb4858d", + "traincheck/instrumentor/tracer.py": "5476b155823b4287f9cb01e498f70b99921063a052512a9f0d5d5b76974850f4", + "traincheck/instrumentor/types.py": "258679aa8d0b40247d900ba2e26ab03b25cb4626b046ab6d724973226c7b5469", + "traincheck/instrumentor/variable.py": "b01219cdd74f011055e0bc2a645ed283dbc47e66a834d5555b814fa815a9eab8", + "traincheck/invariant/DistinctArgumentRelation.py": "6e01540583668e30718ba90c341fd540cd9df53f20525a6cf502fcceee8ed7bc", + "traincheck/invariant/__init__.py": "773e9f767566ffd92658dc0e94107ac840672e446a41ba437a49cee9cc52197a", + "traincheck/invariant/base_cls.py": "31ceb584760d26e9923d2c7ce962fd6f093a3a6ba983ed177fcb4257c1da2466", + "traincheck/invariant/consistency_relation.py": "bc5b14810ea8a5e0e5f3e6c2c6be24d137a444d8609216d00eafd155b4aa3917", + "traincheck/invariant/consistency_transient_vars.py": "a702cd1b6787e8be1904bef9119773c0e037a83017959c771020dbbbbe1549e2", + "traincheck/invariant/contain_relation.py": "b32f819db4d5a11a11318bf7e38c64148f4c9360137ac5f09475b480c34c16a5", + "traincheck/invariant/cover_relation.py": "0eb0e1c30ad5b34a15ce32c7b0d15e894f7a3b9b68bfcaae5b901f5dc6a092c6", + "traincheck/invariant/lead_relation.py": "978a4db32b69dc1a90a84123aa93e52a3caaba97761321dbb844666cb47d4360", + "traincheck/invariant/precondition.py": "c14c7580643780954eab5a4d955b0b5d6a6768a57cd8e5e48328ad0d8f2f8703", + "traincheck/invariant/relation_pool.py": "8ac568cf1924162a791728ed5db8c41868a550f9e0cd5190fc9a19827f55d3e6", + "traincheck/invariant/symbolic_value.py": "28627e2c2a47a146087a96510f07f7deb8463c46371e418afa3fdeff33c7e6cb", + "traincheck/invariant/var_periodic_change_relation.py": "8086df3b9cfe34229b8bd46b18dce22a4b746902e38d1ba4bd02bc6f6c2f2ef4", + "traincheck/onlinechecker/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/onlinechecker/streamhandler_filesystem.py": "b5a298bd06a9408e857ff6721578cb4ae10f227762ba3d66aec685e574cfa511", + "traincheck/onlinechecker/utils.py": "65ba5d8a84b748adf4267800e704df1f4662c0f483534289af20096bf9a78a00", + "traincheck/parallel.py": "6843319dd8c5c397e93f361a29fe564161e7307fbe4f0f8368090d252d4a7bc4", + "traincheck/progress.py": "aab7d040324e283c8474aa82bc7550a777aec316131a26e1dc1b6a277333d1ae", + "traincheck/reporting/__init__.py": "299180153357c4e6f48cf75191e915d70cfca5ae6e7b3ac4d45144066b93ddca", + "traincheck/reporting/checker_report.py": "0d59b1753fc348701b49a0eb74e4b1d849a47d8234005ccbbb26e09e3cd0c808", + "traincheck/runner/__init__.py": "287ffa9cab82f01da0e82faea77589dc7d348710c862496458d2d60c364394ba", + "traincheck/runner/runner.py": "89027b488ded8640fddb387dd80d98f059cf40be5f94863966b259acea1da7c7", + "traincheck/script/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/script/num_invs_each_rel.py": "202f13d536929c52defa84cdce00dec2251ea09aabf3ae8ae57067c32d0b844d", + "traincheck/static_analyzer/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/config.py": "840460167bd6d26db429076cd4dc227e6c17c4e5fb157172041c48574f44edf7", + "traincheck/static_analyzer/func_level/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/graph_generator/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/static_analyzer/graph_generator/analyzer.py": "035684eb7ea4edf8ee0dbf4c6b12b91b428cd6dd57be5fe658253c21c8c420b1", + "traincheck/static_analyzer/graph_generator/anutils.py": "9e62b7ce5fced45a033b76de3d408bdff2fd8561ffa28bc58de82e4105b51d39", + "traincheck/static_analyzer/graph_generator/call_graph_parser.py": "69e2555a1595078a2d2bca174de32cd7d4b0f023c0a20b68a01ca00d1d3b8647", + "traincheck/static_analyzer/graph_generator/main.py": "a568488c50383a67a6ab8feecaf96e29b288861c1dce17481a1b5032d20748a4", + "traincheck/static_analyzer/graph_generator/node.py": "b6a0cea44a28d33f28fee79b7f7073e69914e95565f7a8d62f9e5a5fd6b240cd", + "traincheck/toolkit/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "traincheck/toolkit/analyze_invariant.py": "706cf8c06062b6df3f1f86691f588c91696cbef6c27698b2f0b290ad8f53ff4b", + "traincheck/toolkit/analyze_trace.py": "48dfc81ef85d96678b5fe466b86ba80d892ecfc497387773fa7ab5995f22e822", + "traincheck/toolkit/detect_anomaly_from_trace_diff.py": "2ffc0723dd3709ccab09cb9b58d76c7bb0558dda09b66c47184e335b815cedd3", + "traincheck/trace/__init__.py": "ec84edd0ff32906c20956166a51b85b3847e22e48432c6a31698080cafbcbc2b", + "traincheck/trace/trace.py": "eac49a1cb958ccaae57ac93e8aad1cbc616e7fcef23068ef5bb68bbc54890c00", + "traincheck/trace/trace_dict.py": "3aec874ef0ce2c34f6332dface604d5f79965976fd3d1c2b692f746f2af4e703", + "traincheck/trace/trace_pandas.py": "819f7a60271da11522bb451920835e2651efb7117dec8cb08c3cc23d0c193464", + "traincheck/trace/trace_polars.py": "64136efcd60a851b0391e708108570b259c2288db6242564b636f69abd1b7b0e", + "traincheck/trace/types.py": "96c1b137f3d72fb05dcf806a4e7ea21fe453ae9cbc3e08352c55e95973fe96e0", + "traincheck/trace/utils.py": "6356156e4c5bbb6c111140302660f36ad1c126e5bc977bd55c81f097022102a3", + "traincheck/utils.py": "c36764ec7b9d7224c82071abf2f83d5ee9952e6868774dff62d851da9d0270e1" + } + }, + "all_source_files_unchanged_after_all_runs": true + }, + "runtime": { + "python": "3.11.4 (main, Jul 5 2023, 13:45:01) [GCC 11.2.0]", + "packages": { + "numpy": "2.1.0", + "pandas": "2.3.3", + "torch": "2.16.0.dev20261012+cpu" + }, + "torch_git_revision": "8f9045d72f71e7aa9e16b6d7a055ec5bc1f720b5", + "original_training_runtime_difference": "Original training used Python 3.12.3 and CUDA 13.0; inference uses Python 3.11.4 and CPU torch from the same source revision." + }, + "workload": { + "saved_model_parameters": 6163712, + "training_steps": 10, + "sequence_length": 256, + "batch_size": 1, + "seed": 42, + "trace_events": 573494, + "raw_bytes": 286335289, + "traces": { + "trace_API_1088885_23438542501568.log": { + "bytes": 82668375, + "sha256": "883d288d6955b12c49dc741c166bcca7d191c890d19565802dad60708979278d" + }, + "trace_API_1088885_23456244348224.log": { + "bytes": 203666914, + "sha256": "416b8ca45b4df361fc73052447573d7cb23d433d16c0947d74bbc18f7cc6c9a8" + } + }, + "all_traces_unchanged_after_all_runs": true, + "enabled_relation_count": 6, + "disabled_relations": [ + "FunctionLeadRelation", + "FunctionCoverRelation" + ], + "eligible_api_count": 41, + "api_coverage_limit": "Original name set was not recorded; matching count alone does not establish exact original-environment API coverage." + }, + "settings": { + "backend": "pandas", + "workers": 1, + "native_threads": 1, + "CUDA_VISIBLE_DEVICES": "", + "PYTHONHASHSEED": "0", + "timeout_seconds": 1800, + "sampled_child_session_pss_budget_gib": 12, + "memory_sample_interval_seconds": 1, + "cprofile": false, + "periodic_traceback_dumps": false, + "fatal_faulthandler": true, + "fresh_process_and_trace_per_run": true, + "lazy_index_build_included": true, + "cpu_affinity": null, + "exclusive_host": false, + "run_order": [ + [ + "baseline", + "candidate" + ], + [ + "candidate", + "baseline" + ], + [ + "baseline", + "candidate" + ] + ] + }, + "runs": [ + { + "pair": 1, + "order": 1, + "variant": "baseline", + "monitor_returncode": 0, + "result": { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "5e4933736061977b60f01396ef03e14a7549faf67fda9b70743c116a62de0798", + "distinct_records": 104, + "records": 104 + }, + "invariants": { + "canonical_multiset_sha256": "955ecefffb96cc5f1acf3b95aaf82de2d808e13d86371c4aee27f5a811b07bdb", + "distinct_records": 836, + "records": 836 + }, + "sampled_peak_concurrent_memory": { + "pss_bytes": 2327513088, + "rss_bytes": 2335973376, + "uss_bytes": 2324221952 + }, + "source_unchanged": true, + "stage_times": { + "marker_durations": { + "cross_trace_collection_seconds": 0.0, + "generation_and_pruning_seconds": 75.47, + "precondition_and_output_sort_seconds": 12.565 + }, + "markers": { + "collection_start": "2026-10-25T06:45:38.025000", + "generation_start": "2026-10-25T06:44:22.555000", + "inference_end": "2026-10-25T06:45:50.590000", + "precondition_start": "2026-10-25T06:45:38.025000" + }, + "note": "Log marker intervals; missing endpoints mean incomplete phases, not zero duration.", + "reported": { + "inference_seconds": 88.07923460006714, + "load_seconds": 10.64183259010315 + } + }, + "stop_reason": null, + "success": true, + "traces_unchanged": true, + "wall_seconds": 102.65519127622247 + }, + "all_outputs_equal_corrected_reference": true, + "background_load": { + "before": { + "loadavg_1_5_15_minutes": [ + 1.29, + 1.24, + 1.16 + ], + "host_cpu_percent_sample": 4.6, + "available_memory_bytes": 52341108736 + }, + "after": { + "loadavg_1_5_15_minutes": [ + 1.93, + 1.5, + 1.27 + ], + "host_cpu_percent_sample": 3.5, + "available_memory_bytes": 52397568000 + } + } + }, + { + "pair": 1, + "order": 2, + "variant": "candidate", + "monitor_returncode": 0, + "result": { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "5e4933736061977b60f01396ef03e14a7549faf67fda9b70743c116a62de0798", + "distinct_records": 104, + "records": 104 + }, + "invariants": { + "canonical_multiset_sha256": "955ecefffb96cc5f1acf3b95aaf82de2d808e13d86371c4aee27f5a811b07bdb", + "distinct_records": 836, + "records": 836 + }, + "sampled_peak_concurrent_memory": { + "pss_bytes": 2328130560, + "rss_bytes": 2336595968, + "uss_bytes": 2324832256 + }, + "source_unchanged": true, + "stage_times": { + "marker_durations": { + "cross_trace_collection_seconds": 0.0, + "generation_and_pruning_seconds": 49.225, + "precondition_and_output_sort_seconds": 12.592 + }, + "markers": { + "collection_start": "2026-10-25T06:46:55.427000", + "generation_start": "2026-10-25T06:46:06.202000", + "inference_end": "2026-10-25T06:47:08.019000", + "precondition_start": "2026-10-25T06:46:55.427000" + }, + "note": "Log marker intervals; missing endpoints mean incomplete phases, not zero duration.", + "reported": { + "inference_seconds": 61.86267161369324, + "load_seconds": 10.646003484725952 + } + }, + "stop_reason": null, + "success": true, + "traces_unchanged": true, + "wall_seconds": 76.7478895187378 + }, + "all_outputs_equal_corrected_reference": true, + "background_load": { + "before": { + "loadavg_1_5_15_minutes": [ + 1.93, + 1.5, + 1.27 + ], + "host_cpu_percent_sample": 3.3, + "available_memory_bytes": 52395216896 + }, + "after": { + "loadavg_1_5_15_minutes": [ + 2.14, + 1.64, + 1.34 + ], + "host_cpu_percent_sample": 3.3, + "available_memory_bytes": 52372901888 + } + } + }, + { + "pair": 2, + "order": 1, + "variant": "candidate", + "monitor_returncode": 0, + "result": { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "5e4933736061977b60f01396ef03e14a7549faf67fda9b70743c116a62de0798", + "distinct_records": 104, + "records": 104 + }, + "invariants": { + "canonical_multiset_sha256": "955ecefffb96cc5f1acf3b95aaf82de2d808e13d86371c4aee27f5a811b07bdb", + "distinct_records": 836, + "records": 836 + }, + "sampled_peak_concurrent_memory": { + "pss_bytes": 2328266752, + "rss_bytes": 2336739328, + "uss_bytes": 2324983808 + }, + "source_unchanged": true, + "stage_times": { + "marker_durations": { + "cross_trace_collection_seconds": 0.001, + "generation_and_pruning_seconds": 49.337, + "precondition_and_output_sort_seconds": 12.469 + }, + "markers": { + "collection_start": "2026-10-25T06:48:13.227000", + "generation_start": "2026-10-25T06:47:23.890000", + "inference_end": "2026-10-25T06:48:25.697000", + "precondition_start": "2026-10-25T06:48:13.228000" + }, + "note": "Log marker intervals; missing endpoints mean incomplete phases, not zero duration.", + "reported": { + "inference_seconds": 61.853256940841675, + "load_seconds": 10.614852905273438 + } + }, + "stop_reason": null, + "success": true, + "traces_unchanged": true, + "wall_seconds": 76.78805723041296 + }, + "all_outputs_equal_corrected_reference": true, + "background_load": { + "before": { + "loadavg_1_5_15_minutes": [ + 2.14, + 1.64, + 1.34 + ], + "host_cpu_percent_sample": 3.3, + "available_memory_bytes": 52375351296 + }, + "after": { + "loadavg_1_5_15_minutes": [ + 2.0, + 1.73, + 1.4 + ], + "host_cpu_percent_sample": 3.7, + "available_memory_bytes": 52377493504 + } + } + }, + { + "pair": 2, + "order": 2, + "variant": "baseline", + "monitor_returncode": 0, + "result": { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "5e4933736061977b60f01396ef03e14a7549faf67fda9b70743c116a62de0798", + "distinct_records": 104, + "records": 104 + }, + "invariants": { + "canonical_multiset_sha256": "955ecefffb96cc5f1acf3b95aaf82de2d808e13d86371c4aee27f5a811b07bdb", + "distinct_records": 836, + "records": 836 + }, + "sampled_peak_concurrent_memory": { + "pss_bytes": 2286166016, + "rss_bytes": 2294603776, + "uss_bytes": 2282872832 + }, + "source_unchanged": true, + "stage_times": { + "marker_durations": { + "cross_trace_collection_seconds": 0.0, + "generation_and_pruning_seconds": 73.854, + "precondition_and_output_sort_seconds": 12.484 + }, + "markers": { + "collection_start": "2026-10-25T06:49:55.626000", + "generation_start": "2026-10-25T06:48:41.772000", + "inference_end": "2026-10-25T06:50:08.110000", + "precondition_start": "2026-10-25T06:49:55.626000" + }, + "note": "Log marker intervals; missing endpoints mean incomplete phases, not zero duration.", + "reported": { + "inference_seconds": 86.3845362663269, + "load_seconds": 10.706615447998047 + } + }, + "stop_reason": null, + "success": true, + "traces_unchanged": true, + "wall_seconds": 101.79567445069551 + }, + "all_outputs_equal_corrected_reference": true, + "background_load": { + "before": { + "loadavg_1_5_15_minutes": [ + 2.0, + 1.73, + 1.4 + ], + "host_cpu_percent_sample": 3.6, + "available_memory_bytes": 52377260032 + }, + "after": { + "loadavg_1_5_15_minutes": [ + 2.19, + 1.88, + 1.49 + ], + "host_cpu_percent_sample": 3.6, + "available_memory_bytes": 52357947392 + } + } + }, + { + "pair": 3, + "order": 1, + "variant": "baseline", + "monitor_returncode": 0, + "result": { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "5e4933736061977b60f01396ef03e14a7549faf67fda9b70743c116a62de0798", + "distinct_records": 104, + "records": 104 + }, + "invariants": { + "canonical_multiset_sha256": "955ecefffb96cc5f1acf3b95aaf82de2d808e13d86371c4aee27f5a811b07bdb", + "distinct_records": 836, + "records": 836 + }, + "sampled_peak_concurrent_memory": { + "pss_bytes": 2286102528, + "rss_bytes": 2294554624, + "uss_bytes": 2282823680 + }, + "source_unchanged": true, + "stage_times": { + "marker_durations": { + "cross_trace_collection_seconds": 0.0, + "generation_and_pruning_seconds": 73.811, + "precondition_and_output_sort_seconds": 12.432 + }, + "markers": { + "collection_start": "2026-10-25T06:51:38.197000", + "generation_start": "2026-10-25T06:50:24.386000", + "inference_end": "2026-10-25T06:51:50.629000", + "precondition_start": "2026-10-25T06:51:38.197000" + }, + "note": "Log marker intervals; missing endpoints mean incomplete phases, not zero duration.", + "reported": { + "inference_seconds": 86.28747224807739, + "load_seconds": 10.534409046173096 + } + }, + "stop_reason": null, + "success": true, + "traces_unchanged": true, + "wall_seconds": 101.61026244610548 + }, + "all_outputs_equal_corrected_reference": true, + "background_load": { + "before": { + "loadavg_1_5_15_minutes": [ + 2.19, + 1.88, + 1.49 + ], + "host_cpu_percent_sample": 3.4, + "available_memory_bytes": 52357136384 + }, + "after": { + "loadavg_1_5_15_minutes": [ + 2.25, + 1.98, + 1.57 + ], + "host_cpu_percent_sample": 3.3, + "available_memory_bytes": 52370046976 + } + } + }, + { + "pair": 3, + "order": 2, + "variant": "candidate", + "monitor_returncode": 0, + "result": { + "exit_code": 0, + "failed_hypotheses": { + "canonical_multiset_sha256": "5e4933736061977b60f01396ef03e14a7549faf67fda9b70743c116a62de0798", + "distinct_records": 104, + "records": 104 + }, + "invariants": { + "canonical_multiset_sha256": "955ecefffb96cc5f1acf3b95aaf82de2d808e13d86371c4aee27f5a811b07bdb", + "distinct_records": 836, + "records": 836 + }, + "sampled_peak_concurrent_memory": { + "pss_bytes": 2328313856, + "rss_bytes": 2336751616, + "uss_bytes": 2325057536 + }, + "source_unchanged": true, + "stage_times": { + "marker_durations": { + "cross_trace_collection_seconds": 0.001, + "generation_and_pruning_seconds": 49.004, + "precondition_and_output_sort_seconds": 12.445 + }, + "markers": { + "collection_start": "2026-10-25T06:52:56.197000", + "generation_start": "2026-10-25T06:52:07.193000", + "inference_end": "2026-10-25T06:53:08.643000", + "precondition_start": "2026-10-25T06:52:56.198000" + }, + "note": "Log marker intervals; missing endpoints mean incomplete phases, not zero duration.", + "reported": { + "inference_seconds": 61.49606490135193, + "load_seconds": 10.764497995376587 + } + }, + "stop_reason": null, + "success": true, + "traces_unchanged": true, + "wall_seconds": 76.6580608561635 + }, + "all_outputs_equal_corrected_reference": true, + "background_load": { + "before": { + "loadavg_1_5_15_minutes": [ + 2.25, + 1.98, + 1.57 + ], + "host_cpu_percent_sample": 3.6, + "available_memory_bytes": 52370046976 + }, + "after": { + "loadavg_1_5_15_minutes": [ + 2.26, + 2.03, + 1.62 + ], + "host_cpu_percent_sample": 3.5, + "available_memory_bytes": 52363665408 + } + } + } + ], + "medians": { + "baseline": { + "wall_seconds": 101.79567445069551, + "wall_min_seconds": 101.61026244610548, + "wall_max_seconds": 102.65519127622247, + "peak_pss_bytes": 2286166016, + "load_seconds": 10.64183259010315, + "inference_seconds": 86.3845362663269, + "generation_and_pruning_seconds": 73.854, + "precondition_and_output_sort_seconds": 12.484 + }, + "candidate": { + "wall_seconds": 76.7478895187378, + "wall_min_seconds": 76.6580608561635, + "wall_max_seconds": 76.78805723041296, + "peak_pss_bytes": 2328266752, + "load_seconds": 10.646003484725952, + "inference_seconds": 61.853256940841675, + "generation_and_pruning_seconds": 49.225, + "precondition_and_output_sort_seconds": 12.469 + } + }, + "ratio_of_median_wall_times": 1.3263644784113884, + "median_wall_reduction_percent": 24.605942312499295, + "median_peak_pss_increase_bytes": 42100736, + "equivalence": { + "all_outputs_equal_corrected_reference": true, + "comparison": "Complete structural JSON multisets; duplicate multiplicity and array order preserved, object key and record line order ignored.", + "outputs": { + "invariants": { + "records": 836, + "distinct_records": 836, + "canonical_multiset_sha256": "955ecefffb96cc5f1acf3b95aaf82de2d808e13d86371c4aee27f5a811b07bdb" + }, + "failed_hypotheses": { + "records": 104, + "distinct_records": 104, + "canonical_multiset_sha256": "5e4933736061977b60f01396ef03e14a7549faf67fda9b70743c116a62de0798" + } + }, + "all_six_output_files_also_byte_identical": true, + "raw_output_sha256": { + "invariants.jsonl": "67ae805da54231ecfcfba060280ca8d5a0b2e98b44523f06b7c64490e9e3a8f0", + "invariants.jsonl.failed": "173364b2792421ae048975cc5e7c6a89230f74223f166af046f7a7c9c95d11b2" + }, + "all_full_and_stage_context_files_equal_reference": true, + "context_sha256": { + "contexts-0.json": "bea2a909d42bc621be2fea66ad7de4e7e871b7f40903f546934e3d7e4d9c1207", + "contexts-1.json": "bea2a909d42bc621be2fea66ad7de4e7e871b7f40903f546934e3d7e4d9c1207" + } + }, + "fast_path_probe": { + "rows": 573494, + "dtypes": { + "type": "object", + "time": "int64", + "process_id": "int64", + "thread_id": "int64", + "func_call_id": "object" + }, + "sampled_queries": 24, + "all_sampled_queries_hit": true, + "indexed_rows": 573494, + "identity_groups": 2, + "limit": "Sampled real-call queries in a separate probe, not a measured hit count for every inference query." + }, + "validation": { + "local_combined_tests": 61, + "matching_runtime_combined_tests": 67, + "matching_runtime_tests_note": "Includes six common pruning-state tests. First packaging attempt lacked tests.fixtures; completed test-support-only rerun passed without production changes.", + "matching_runtime_test_log_sha256": "84cfeba80e7c2032b2e7a28baebc4c7bd2fc2e778f27bf6eb16362b4f857b5d3", + "independent_differential_audit": { + "parent_commit": "99d16905296d83495570fdcafa91370b242d2528", + "parent_source_sha256": "41040a288284dc10041a4c06d8598219605732dd5dcd2ef080b9ed0799387f8a", + "candidate_source_sha256": "819f7a60271da11522bb451920835e2651efb7117dec8cb08c3cc23d0c193464", + "pandas": "2.3.3", + "numpy": "2.1.0", + "cases": 144, + "comparisons": 6480, + "differences": [] + }, + "independent_evidence_audit": { + "passed": true, + "archive_files_verified": 295, + "all_outputs_and_contexts_equal_corrected_reference": true, + "only_signed_production_delta_confirmed": true, + "raw_trace_hash_verification_scope": "Before/after hashes computed remotely and reviewed; raw input is not in evidence archive and was not rehashed locally." + } + }, + "synthetic_benchmark": { + "scope": "synthetic function-call window queries, including construction and lazy build; not full inference or peak memory", + "settings": { + "rows": 200000, + "queries": 1000, + "repetitions": 3 + }, + "runtime": { + "python": "3.11.4", + "platform": "Linux-6.1.0-1027-oem-x86_64-with-glibc2.35", + "pandas": "2.3.3", + "numpy": "2.1.0", + "native_threads": { + "OMP_NUM_THREADS": "1", + "OPENBLAS_NUM_THREADS": "1", + "MKL_NUM_THREADS": "1" + } + }, + "source_sha256": "819f7a60271da11522bb451920835e2651efb7117dec8cb08c3cc23d0c193464", + "input_sha256": "4d2a33e4e4c692eaa5339751b3db19fbb8a2335cb1c9ebd26ba073fbbf407c1f", + "input_unchanged": true, + "all_ordered_results_equal": true, + "measurements": [ + { + "repetition": 1, + "order": 1, + "variant": "legacy", + "construction_seconds": 0.04303494095802307, + "first_query_seconds": 0.006073214113712311, + "remaining_queries_seconds": 2.613212414085865, + "total_seconds": 2.6623205691576004, + "result_sha256": "30844ebafc1edb92116075982549646f9c626aab9f63a9bbfc5a3e21771439eb" + }, + { + "repetition": 1, + "order": 2, + "variant": "candidate", + "construction_seconds": 0.0409860722720623, + "first_query_seconds": 0.07652264088392258, + "remaining_queries_seconds": 0.21955957636237144, + "total_seconds": 0.3370682895183563, + "result_sha256": "30844ebafc1edb92116075982549646f9c626aab9f63a9bbfc5a3e21771439eb", + "indexed_rows": 200000, + "identity_groups": 8 + }, + { + "repetition": 2, + "order": 1, + "variant": "candidate", + "construction_seconds": 0.044426146894693375, + "first_query_seconds": 0.07581401616334915, + "remaining_queries_seconds": 0.21997691318392754, + "total_seconds": 0.34021707624197006, + "result_sha256": "30844ebafc1edb92116075982549646f9c626aab9f63a9bbfc5a3e21771439eb", + "indexed_rows": 200000, + "identity_groups": 8 + }, + { + "repetition": 2, + "order": 2, + "variant": "legacy", + "construction_seconds": 0.04476165026426315, + "first_query_seconds": 0.003517802804708481, + "remaining_queries_seconds": 2.71007227152586, + "total_seconds": 2.7583517245948315, + "result_sha256": "30844ebafc1edb92116075982549646f9c626aab9f63a9bbfc5a3e21771439eb" + }, + { + "repetition": 3, + "order": 1, + "variant": "legacy", + "construction_seconds": 0.04182035103440285, + "first_query_seconds": 0.0035678334534168243, + "remaining_queries_seconds": 2.596189245581627, + "total_seconds": 2.6415774300694466, + "result_sha256": "30844ebafc1edb92116075982549646f9c626aab9f63a9bbfc5a3e21771439eb" + }, + { + "repetition": 3, + "order": 2, + "variant": "candidate", + "construction_seconds": 0.04158874601125717, + "first_query_seconds": 0.0771775059401989, + "remaining_queries_seconds": 0.22500132396817207, + "total_seconds": 0.34376757591962814, + "result_sha256": "30844ebafc1edb92116075982549646f9c626aab9f63a9bbfc5a3e21771439eb", + "indexed_rows": 200000, + "identity_groups": 8 + } + ], + "medians": { + "legacy": { + "construction_seconds": 0.04303494095802307, + "first_query_seconds": 0.0035678334534168243, + "remaining_queries_seconds": 2.613212414085865, + "total_seconds": 2.6623205691576004 + }, + "candidate": { + "construction_seconds": 0.04158874601125717, + "first_query_seconds": 0.07652264088392258, + "remaining_queries_seconds": 0.21997691318392754, + "total_seconds": 0.34021707624197006 + } + }, + "median_total_ratio": 7.825358440456698 + }, + "provenance": { + "evidence_archive_name": "timing-evidence.tar.gz", + "evidence_archive_bytes": 1705821, + "evidence_archive_sha256": "7f45cee3ca5bf8f4b8b1fa2f29ee54234fafd1fe81b57fd0badb7cb38fa81c36", + "evidence_archive_access": "Retained study artifact; raw trace/output data are not published in this repository.", + "harness_sha256": "2dd8d63a8bae80a944defee70be6bf33e9c9c8934e22e4b909ff222b4f7d6777", + "helper_sha256": "1fc8409ec68d23da26223424a41dcaeff252ede84723b4256fd7dc5797f8e994", + "controller_sha256": "416db6554b02299ccc44ef73ed4eba7735bfe925a0e68a708701ae0f6baf1271", + "paired_runs_json_sha256": "5304104c25281c369b00ff217fb2989a0be712dc9f4cfe08ac4d54c2b3af9b27", + "snapshot_manifest_sha256": "2f85ca6e20061565b6b6cf02ad2760f3c8abb70978ce76eb5d1ed69cd032d750", + "final_integrity_validation_sha256": "0bf446f2fe7387e4fdcc9e30f0c4e9012162251b340158e844d8e0712f1a2913", + "independent_audit_json_sha256": "6ca49fd92ede07e153c81c373d3242fa49c379204e3dc2e8fc3d1497057cfdd8", + "fast_path_probe_json_sha256": "dbc0be5aacdab32979a46e21a11c55fa6f4bcdf49a25c4d369113925a22006c7", + "process_check_utc": "2026-10-25T10:54:52.654222+00:00", + "remaining_study_processes": [] + }, + "limits": [ + "Three pairs on one saved trace and six enabled relations, with unrelated background load and no exclusive host or CPU affinity.", + "Measures offline inference, not training overhead or distributed scalability; output equality is not independent invariant ground truth.", + "Wall time includes child startup/imports/output writes and monitor polling, excludes integrity hashing; load includes harness context serialization.", + "Reported inference ends before invariant JSON writes; marker durations are subphases, not additive extra time.", + "One-second sampled concurrent child-session PSS excludes the monitor and depends on shared pages; the 12-GiB budget is a monitor threshold, not an OS hard limit.", + "The per-frame index stores O(N) values/positions and adds first-query build cost; same-size in-place edits to cached fields or row order require a new TracePandas.", + "This incremental speedup must not be multiplied by the previous single-pair identity-cache measurement as an independently measured combined speedup." + ] +} diff --git a/docs/measurements/inference-time-range-index.md b/docs/measurements/inference-time-range-index.md new file mode 100644 index 00000000..301fa5f7 --- /dev/null +++ b/docs/measurements/inference-time-range-index.md @@ -0,0 +1,193 @@ +# Index function-call time windows + +Repeated calls to `query_func_call_events_within_time` previously filtered every +trace row by type, time, process, and thread before grouping the selected calls. +The new lazy index uses binary search within each process/thread pair, restores +the selected rows to their original order, and retains the existing pandas +`groupby`, record construction, and assertions. A conservative fast path handles +ordinary int64 identities with int64 or float64 time; all unsupported dtypes, +query scalar types, and comparison cases use the original pandas mask. + +## Complete saved-trace inference + +Three alternating sequential pairs on labmach reduced median total wall time from +**101.796 s to 76.748 s (1.326x, 24.61% less time)**. Both variants include the +identity cache and the independent pruning-state correction from PR #145. The +only production difference between the measured snapshots is the time-window +index in `trace_pandas.py`. + +| Pair | Execution order | Baseline wall | Candidate wall | Baseline peak PSS | Candidate peak PSS | +|---|---|---:|---:|---:|---:| +| 1 | Baseline, candidate | 102.655 s | 76.748 s | 2.167665 GiB | 2.168241 GiB | +| 2 | Candidate, baseline | 101.796 s | 76.788 s | 2.129158 GiB | 2.168367 GiB | +| 3 | Baseline, candidate | 101.610 s | 76.658 s | 2.129099 GiB | 2.168411 GiB | + +All six runs completed and produced the same **836 invariant records and 104 +saved failed-hypothesis records** as the corrected reference. Complete structural +JSON multisets match, preserving duplicate multiplicity and array order. The raw +output files also match byte for byte across the six runs. Full/stage context +files match the corrected reference byte for byte, and all source/input hashes +remained unchanged. Independent review verified all 295 archived files, source +snapshots, complete outputs, contexts, and calculations. Trace integrity checks +ran before and after inference on the remote host; the raw 286-MB input is not +in the downloaded archive and was not independently rehashed locally. This is +an incremental measurement over the corrected cached +baseline; it should not be multiplied by the earlier single-pair identity-cache +speedup as an independently measured combined result. + +| Median measurement | Baseline | Time-window index | +|---|---:|---:| +| Total child wall time | 101.796 s | 76.748 s | +| Trace loading/construction | 10.642 s | 10.646 s | +| Reported inference | 86.385 s | 61.853 s | +| Generation and pruning (part of inference) | 73.854 s | 49.225 s | +| Final preconditions and sort (part of inference) | 12.484 s | 12.469 s | +| Peak sampled child-session PSS | 2.129 GiB | 2.168 GiB | + +The median sampled memory peak increased by **40.15 MiB**. The first baseline +peak was higher than the other two and close to its paired candidate peak; +per-run values above retain that variation. Each trial uses a fresh process and +trace, including the first lazy index construction. The index stores +O(N) timestamp/position values and requires temporary build storage. It is a +runtime/memory tradeoff, and its first-query cost matters for short-lived traces. + +The input is both complete trace files from the saved 6,163,712-parameter, +10-step reference run: 573,494 events and 286,335,289 raw bytes, sequence length +256, batch size 1, seed 42. No training was rerun. A separate probe confirmed that +all 24 sampled real-call queries hit the new fast path, with int64 time/PID/TID +columns and built-in integer bounds/identities. This is sampled eligibility +validation, not a hit count for every query in the timed inference runs. + +Both variants use pandas, one worker, native thread settings of one, no GPU, +`PYTHONHASHSEED=0`, and six enabled relations; `FunctionLeadRelation` and +`FunctionCoverRelation` remain disabled. Every run has a 1,800-second cutoff and +12-GiB sampled child-session PSS budget. The runtime is Python 3.11.4, +Torch 2.16.0.dev20261012+cpu, pandas 2.3.3, and NumPy 2.1.0. Torch's source revision +matches the original CUDA wheel, but training used Python 3.12.3 and CUDA 13.0. +The eligible API count is 41; the original name set was not retained, so matching +the count does not establish exact original-environment API coverage. + +The host was shared, with no CPU affinity or exclusive reservation. Other owned +experiment jobs were paused during the six timed runs; unrelated background +work was left untouched. Before-run one-minute load averages ranged from 1.29 +to 2.25, and sampled whole-host CPU utilization from 3.3% to 4.6%. These snapshots +do not bound interference during each run. The result covers one saved workload +and relation set, not training overhead, distributed scaling, or invariant +ground-truth correctness. + +Total wall time includes child startup, imports, output writes, and monitor +polling (about one-second resolution), but excludes integrity hashing. The load +timer includes harness context serialization. The inference timer ends before +invariant JSON writes. Its two phases in the table are log-marker intervals, +not additional time. Memory is sampled once per second for the concurrent child +session, excludes the monitor, and depends on shared pages. The memory budget is +a monitor threshold, not an OS hard limit. cProfile and periodic traceback dumps +were disabled; fatal-error traceback handling remained enabled. + +[Compact measurements and provenance](inference-time-range-index-20261025.json) +retain every run's timing, memory peak, background-load snapshots, source/input +hashes, output digests, and archive identity. Raw traces and output records are +retained as study artifacts rather than published in this repository. + +## Correctness and cache contract + +The 18 new differential tests compare the indexed path with the unchanged pandas +fallback. Together with context/call/identity tests, 61 tests pass locally. The +matching Linux runtime passes those tests plus six common pruning-state tests +(67 total). An independent oracle extracted from the exact parent method checks +144 dtype/offset cases and 6,480 queries with no differences in each of two +pandas/NumPy environments: 2.2.3/1.26.4 and 2.3.3/2.1.0. + +The tests cover strict boundaries, tied/reversed bounds, signed integer extremes, +mixed integer/float precision above 2**53, NaNs/infinities/signed zero, malformed +bounds and scalar subclasses, null/extension/categorical dtypes, empty frames, +unsorted rows and duplicate labels, categorical/null call IDs, incomplete or +exception records, fresh returned objects, and cache invalidation. Empty +selections still run the original grouping logic, including its assertions. + +Time, event type, process/thread identity values and dtypes, and row order must +remain stable within a cached frame. Replacement or length changes invalidate +the cache; same-size in-place changes require a new `TracePandas`. The cache holds +a source-frame reference rather than copying the DataFrame, but can retain an old +frame until invalidation. Existing pandas group ordering, category/dropna +behavior, within-group row order, assertion messages, and fresh record/event +objects remain unchanged. + +## Reproduction + +The portable synthetic benchmark needs no saved trace: + +```sh +OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 \ +python -m tools.benchmarks.time_range_index --rows 200000 --queries 1000 --repetitions 3 + +python -m unittest tests.test_context_index tests.test_call_index \ + tests.test_identity_cache tests.test_time_range_index +``` + +On the matching Linux runtime, its median construction-plus-query time fell from +**2.662321 s to 0.340217 s (7.825x)**. Ordered outputs match in all six trials, and +the input remains unchanged. Candidate first-query median was **0.076523 s** +versus **0.003568 s** for legacy filtering, exposing the lazy build cost. Input +generation, garbage collection, and result hashing are excluded; fresh record +construction is included. This synthetic result is separate from full inference. + +For the saved-trace measurement, reconstruct the baseline from commit +`99d16905296d83495570fdcafa91370b242d2528` plus only the pruning-state correction +hunks from `1b5aedffbeefd4cdd9d30036dbb0a833521526cf`. The candidate adds the production +change at `24e7cb0f7a573ff6225a388eff189d9e4af5e8b5`; the benchmark was checkpointed +at `39152aded8b97ed83988e3292bb89c6e3adcf750`. Match the recorded per-file hashes. +With the retained `inference_replay_fatal_only.py` and `inference_scaling.py` +harness files, invoke each fresh variant as follows, using the two trace files +identified by digest in the evidence JSON: + +```sh +"$INFERENCE_PYTHON" "$STUDY/inference_replay_fatal_only.py" \ + --mode infer --source "$SOURCE_SNAPSHOT" --traces "$TRACE_A" "$TRACE_B" \ + --output "$NEW_OUTPUT_DIRECTORY" --timeout 1800 --memory-limit-gib 12 +``` + +Run baseline/candidate, candidate/baseline, baseline/candidate sequentially, with +no `--profile` flag. The harness fixes workers/native threads/hash seed and CPU +mode, verifies source/input hashes, and saves phase/memory/context evidence. The +controller compares complete outputs. `run_pairs.py`, exact invocations, both +immutable source snapshots, full outputs, and the verified harness/helper hashes are +retained in the evidence archive. Reproducing this full-trace result requires +access to the saved trace and retained harness; the synthetic benchmark and +regression tests are self-contained in the checkout. + +## Engineering hardening verification + +The inference stack now types task payloads and RNG scopes, validates required +worker settings before any worker/global mutation, and types each benchmark's +inputs and measurement output. Identity APIs describe their actual schemaless +pandas scalar values instead of promising integers. The time index uses numeric +containers only after the existing conservative dtype checks. Legacy context +comparison retains one documented pandas scalar `Any` boundary; no values are +coerced to satisfy typing and the cache stability contract is unchanged. + +All 97 affected inference tests pass, and the configured Ruff/Mypy/Black/isort +hooks pass for changed files. Pinned Mypy 1.9 also passes the sampling and worker +modules, three benchmark tools, and worker-contract tests with +`--check-untyped-defs --disallow-untyped-defs --disallow-any-generics +--follow-imports=silent --ignore-missing-imports`. New worker-contract tests +produce 18 failures before runtime validation and pass afterward; missing, +unknown, and malformed fields are rejected before state changes. + +A new bounded Linux replay of signed inference head `ce25c12`, with the same +separate #145 production patch applied, completed on the original 573,494-event +trace. **All 836 invariants, 104 failed-hypothesis records, and both context files +are byte-for-byte identical to the corrected reference.** Source and input hashes +remain unchanged. This one verification run took 76.794 seconds; it is not a new +paired performance experiment, and the earlier performance claims above remain +tied to their original measured revisions. + +The separately composed production source (inference, tracing, and pruning-state +fixes) passes whole-source Mypy on 77 files, Ruff, and isort. Whole-source Black +still reports three unchanged legacy files (`config/config.py`, +`instrumentor/proxy_wrapper/hash.py`, and `instrumentor/proxy_wrapper/subclass.py`); +the original base reports six. Scoped strict checks do not certify fully strict +typing of every legacy body. + +[Compact hardening evidence](inference-hardening-validation-20261025.json) +records exact composition, runtime, output hashes, and limits. diff --git a/docs/measurements/precondition-state-isolation-20261025.json b/docs/measurements/precondition-state-isolation-20261025.json new file mode 100644 index 00000000..25924a80 --- /dev/null +++ b/docs/measurements/precondition-state-isolation-20261025.json @@ -0,0 +1,130 @@ +{ + "measurement_date": "2026-10-25", + "base_commit": "9eb4f320d3db34d372145261704b805b88e39bda", + "base_source_sha256": "b32c83db3546567b3a5bc4d8b6a589a862c39b5fb59bd284b885d72e47d47a68", + "fixed_source_sha256": "af6a2d33b7aff2f09260144dd73a828910357e520be259995c5a2d529b719f7a", + "fixture": { + "common_record": { + "time": 0, + "process_id": 1, + "thread_id": 2, + "function": "fixture.step" + }, + "A": { + "positive": { + "meta_vars.phase": "train", + "meta_vars.signal": true + }, + "negative": { + "meta_vars.phase": "train", + "meta_vars.signal": false + } + }, + "B": { + "positive": { + "meta_vars.phase": "train" + }, + "negative": { + "meta_vars.phase": "eval" + } + }, + "expected_A_clause": { + "prop_name": "meta_vars.signal", + "type": "constant", + "values": [ + true + ] + }, + "expected_B_clause": { + "prop_name": "meta_vars.phase", + "type": "constant", + "values": [ + "train" + ] + } + }, + "fresh_process_sequences": [ + { + "order": [ + "B" + ], + "before_counts": [ + 1 + ], + "after_counts": [ + 1 + ], + "python_random_state_unchanged_before_and_after": true + }, + { + "order": [ + "A", + "B" + ], + "before_counts": [ + 1, + 0 + ], + "after_counts": [ + 1, + 1 + ], + "python_random_state_unchanged_before_and_after": true + }, + { + "order": [ + "B", + "A" + ], + "before_counts": [ + 1, + 1 + ], + "after_counts": [ + 1, + 1 + ], + "python_random_state_unchanged_before_and_after": true + }, + { + "order": [ + "B", + "B" + ], + "before_counts": [ + 1, + 1 + ], + "after_counts": [ + 1, + 1 + ], + "python_random_state_unchanged_before_and_after": true + } + ], + "regression_tests": { + "total": 6, + "before_failures": 5, + "before_passes": 1, + "after_passes": 6 + }, + "coverage": [ + "independent and reversed order", + "repeated independent solves", + "explicit set updated and reused", + "recursive sibling sharing with distinct root sets", + "fresh versus warm process histories" + ], + "limits": [ + "No multi-worker-count scheduling matrix measured.", + "No global inference ordering or scheduling determinism claim.", + "No unchanged large-trace output claim; fixing leakage may recover additional preconditions and change inference outputs.", + "Independent correctness fix; not part of the identity-cache performance comparison." + ], + "evidence_sha256": { + "before.json": "cb4fe5ac7cdb5ebe1544dc61ec06ed18de2a435cba2fad2d33fbbfb4ac022119", + "after.json": "62beead0bab542c5b7961588ed99d8c0977696169caacbca77d845c0b74e1383", + "tests-before.log": "970f9696404be2409515ab16d0a44d452e3a82d4d85acfb42844ef3b8f23b0d5", + "tests-after.log": "bebcffb3ddb091ef39bac8cfea9085bbe09e97363ba67c01acd50c3821e1096d" + } +} diff --git a/docs/measurements/precondition-state-isolation-replay-20261025.json b/docs/measurements/precondition-state-isolation-replay-20261025.json new file mode 100644 index 00000000..9b85db11 --- /dev/null +++ b/docs/measurements/precondition-state-isolation-replay-20261025.json @@ -0,0 +1,296 @@ +{ + "measurement_date": "2026-10-25", + "scope": "One completed configured replay pair; observed output differences only, not ground truth or a speed comparison.", + "source_composition": { + "cache_base_commit": "504e04d4de1a4168aabb5436f1b236edef1dadd7", + "state_fix_commit": "1b5aedffbeefd4cdd9d30036dbb0a833521526cf", + "application": "Only the None-default and fresh-set initialization hunks applied to cache source; prior RNG isolation retained. Standalone PR145 source/base unchanged.", + "independent_audit": { + "production_python_files_compared": 76, + "production_changed": [ + "traincheck/invariant/precondition.py" + ], + "exactly_none_default_and_fresh_set_initialization": true, + "prior_local_rng_fix_preserved": true, + "all_combined_python_source_hashes_match_manifest": true, + "new_regression_tests_match_PR145": true, + "base_archive_sha256": "c1c6e2a0dc8af8664e4e3f9a421837bee8c966a5bbe5233ab89ed68491058884", + "combined_archive_sha256": "2e22746a5f2b002c267269a9680873dcbe4f673d9988c7faefd4399e1323454a", + "combined_precondition_source_sha256": "c14c7580643780954eab5a4d955b0b5d6a6768a57cd8e5e48328ad0d8f2f8703", + "standalone_PR145_source_sha256": "af6a2d33b7aff2f09260144dd73a828910357e520be259995c5a2d529b719f7a" + } + }, + "configuration": { + "trace_rows": 573494, + "rank_count": 1, + "trace_file_count": 2, + "workers": 1, + "native_threads": 1, + "profile_enabled": false, + "periodic_traceback_dumps": false, + "disabled_relations": [ + "FunctionLeadRelation", + "FunctionCoverRelation" + ] + }, + "runtime": { + "native_threads": 1, + "packages": { + "numpy": "2.1.0", + "pandas": "2.3.3", + "torch": "2.16.0.dev20261012+cpu" + }, + "profile_enabled": false, + "python": "3.11.4 (main, Jul 5 2023, 13:45:01) [GCC 11.2.0]", + "workers": 1 + }, + "focused_tests": { + "passed": 17, + "identity_cache_tests": 11, + "state_isolation_tests": 6, + "source_unchanged": true, + "log_sha256": "1453d313f13a7cc4c3e43491f54cd604ecc30449c80422a137f2f8252c8a6d52" + }, + "outputs": { + "cache_only": { + "emitted": { + "canonical_multiset_sha256": "f4f5db6772b857074d113e1c6ea091d5c4ed2452e091607fa425efab4edeed65", + "distinct_records": 835, + "records": 835 + }, + "failed": { + "canonical_multiset_sha256": "f3872b21c9ed7103e1020ccb83ef54b41009f2ec06348ce1b7dc5893e545f291", + "distinct_records": 105, + "records": 105 + } + }, + "cache_plus_state_isolation": { + "emitted": { + "canonical_multiset_sha256": "955ecefffb96cc5f1acf3b95aaf82de2d808e13d86371c4aee27f5a811b07bdb", + "distinct_records": 836, + "records": 836 + }, + "failed": { + "canonical_multiset_sha256": "5e4933736061977b60f01396ef03e14a7549faf67fda9b70743c116a62de0798", + "distinct_records": 104, + "records": 104 + } + } + }, + "hypothesis_identity_method": "all invariant fields except precondition, using outer FailedHypothesis example counts for failures; dictionary keys canonicalized, array order preserved", + "hypothesis_identity_counts": { + "after": 940, + "before": 940, + "only_after": 0, + "only_before": 0, + "shared": 940 + }, + "status_transitions": { + "emitted_to_emitted": 835, + "failed_to_emitted": 1, + "failed_to_failed": 104 + }, + "existing_emitted_preconditions": { + "different_even_after_order_normalization": 2, + "only_clause_disjunction_or_constant_value_order_changed": 0, + "strictly_identical": 833 + }, + "strict_structural_multisets": { + "emitted": { + "added_records": 3, + "equal": false, + "removed_records": 2, + "unchanged_records": 833 + }, + "failed": { + "added_records": 0, + "equal": false, + "removed_records": 1, + "unchanged_records": 104 + } + }, + "changed_records": [ + { + "relation": "APIContainRelation", + "apis": [ + "torch.cuda.random.get_rng_state", + "torch.cuda.current_device" + ], + "hypothesis_core_sha256": "670eebef31bf6194128618f1094c29bc334ad493c5f689f442174209287e4f5f", + "before_status": "emitted", + "after_status": "emitted", + "before_clause_count": 2, + "after_clause_count": 3, + "existing_clauses_preserved": true, + "new_constraint_targets": [ + { + "field": "thread_id", + "path": "None", + "type": "constant" + } + ], + "contains_concrete_trace_thread_id": true + }, + { + "relation": "APIContainRelation", + "apis": [ + "torch.nn.modules.normalization.RMSNorm.forward", + "torch.utils.checkpoint._internal_assert" + ], + "hypothesis_core_sha256": "10cbe781752237a942f768e17cae45ee45e56ebfab3c0c2c2ee13bdf8144d2ea", + "before_status": "emitted", + "after_status": "emitted", + "before_clause_count": 1, + "after_clause_count": 7, + "existing_clauses_preserved": true, + "new_constraint_targets": [ + { + "field": "meta_vars.context_managers.torch.amp.autocast_mode.autocast", + "path": [ + "enabled" + ], + "type": "constant" + }, + { + "field": "meta_vars.context_managers.torch.amp.autocast_mode.autocast", + "path": "None", + "type": "exist" + }, + { + "field": "meta_vars.context_managers.torch.amp.autocast_mode.autocast", + "path": [ + "cache_enabled" + ], + "type": "constant" + }, + { + "field": "meta_vars.context_managers.torch.amp.autocast_mode.autocast", + "path": [ + "device_type" + ], + "type": "constant" + }, + { + "field": "meta_vars.context_managers.torch.amp.autocast_mode.autocast", + "path": [ + "dtype" + ], + "type": "constant" + }, + { + "field": "thread_id", + "path": "None", + "type": "constant" + } + ], + "contains_concrete_trace_thread_id": true + }, + { + "relation": "APIContainRelation", + "apis": [ + "torch.nn.modules.linear.Linear.forward", + "torch.utils.checkpoint._internal_assert" + ], + "hypothesis_core_sha256": "bad3f84294dde635e3ea302d3ee6e21cd85658685621629c45e9cda388199a6c", + "before_status": "failed", + "after_status": "emitted", + "before_clause_count": null, + "after_clause_count": 6, + "existing_clauses_preserved": false, + "new_constraint_targets": [ + { + "field": "meta_vars.context_managers.torch.amp.autocast_mode.autocast", + "path": [ + "enabled" + ], + "type": "constant" + }, + { + "field": "meta_vars.context_managers.torch.amp.autocast_mode.autocast", + "path": "None", + "type": "exist" + }, + { + "field": "meta_vars.context_managers.torch.amp.autocast_mode.autocast", + "path": [ + "cache_enabled" + ], + "type": "constant" + }, + { + "field": "meta_vars.context_managers.torch.amp.autocast_mode.autocast", + "path": [ + "dtype" + ], + "type": "constant" + }, + { + "field": "meta_vars.context_managers.torch.amp.autocast_mode.autocast", + "path": [ + "device_type" + ], + "type": "constant" + }, + { + "field": "thread_id", + "path": "None", + "type": "constant" + } + ], + "contains_concrete_trace_thread_id": true + } + ], + "portability_observation": "All three changed/new predicates include a concrete trace-specific thread_id constraint; literal IDs are omitted. Applicability to future checking runs is unvalidated.", + "validation": { + "both_runs_exit_zero": true, + "both_sources_and_traces_unchanged": true, + "same_trace_hashes": true, + "same_harness_and_runtime": true, + "context_rows_states_and_order_hashes_unchanged": true, + "run_source_hashes_match_audited_manifest": true + }, + "input_traces": [ + { + "bytes": 82668375, + "sha256": "883d288d6955b12c49dc741c166bcca7d191c890d19565802dad60708979278d" + }, + { + "bytes": 203666914, + "sha256": "416b8ca45b4df361fc73052447573d7cb23d433d16c0947d74bbc18f7cc6c9a8" + } + ], + "harness_sha256": "2dd8d63a8bae80a944defee70be6bf33e9c9c8934e22e4b909ff222b4f7d6777", + "evidence_sha256": { + "cache_only": { + "invariants.jsonl": "313f9d56d25a0a015fde67135db558f0df891a07d825e425aa47f5cc9a9d11ab", + "invariants.jsonl.failed": "3f3e1b8ecf3193dde9ef727b07a95fb8d26d6feb67faa78807cac3f93f45f8d1", + "invocation.json": "4b650c6a1725f5294b10fa6122198486a4b6c0e9fae5a1530baa16b1a000a5e9", + "provenance.json": "b48d705bd9dccbdf7bce17e8704e7ae5554a7d6239cfd545d7b5ffc66a888b09", + "result.json": "663d9f99601e1725355796a7d62a6c269c0390f4ca40d7f1b2030ef38f39a85c", + "runtime.json": "b5e7bc216ea115fa35f940dc3b1b97604e23098e70ff4145523847b0e236e8c3" + }, + "cache_plus_state_isolation": { + "invariants.jsonl": "67ae805da54231ecfcfba060280ca8d5a0b2e98b44523f06b7c64490e9e3a8f0", + "invariants.jsonl.failed": "173364b2792421ae048975cc5e7c6a89230f74223f166af046f7a7c9c95d11b2", + "invocation.json": "5cdd44c2575906509285c987aa33710362c79ebdae71b10f7334f7df3b6cbbd3", + "provenance.json": "040a61bd84007f029d636449da38b3df2ce967260948c43d361a61628d7f8623", + "result.json": "732dda3e56e54b1a9d50a198522d5144c37142caa014ed1287fd0446a5f38001", + "runtime.json": "0f3b11a9f79ae2d827d003c17b57881276caafa2d0232a750751ceadeef5e9b3" + }, + "independent_reviews": { + "combined-source-review.json": "84e490ad710ccea93d300d4923f3625f0fa677212b3aef3b5ab20d35e9f49bfc", + "combined-output-review.json": "4f1810a5cc627a49556e913b950fc53ef65f0fb32ee67567288405754e2336e1", + "combined-structure-details.json": "9b6056356655c02bb361ad2b496c5543323b984f088822c4faf7c222d80128a8", + "compare_replay_outputs.py": "2749417cde5ed5f65314dc3ec4b83350f7cda32532fc9044637fea788e358367" + }, + "source_manifest": "33569e886e5ec1f5a4e5b985744652ad7b62a331256775fe2a87b6a0e3fa19e9" + }, + "limitations": [ + "One configured single-worker replay pair, not a worker-count or scheduling matrix.", + "FunctionLeadRelation and FunctionCoverRelation were disabled.", + "Structural comparisons are not ground-truth invariant validation.", + "Order normalization is limited to disjunctions, conjunctions, and constant-value membership; it is not general logical-equivalence proving.", + "Concrete thread-ID predicates may not apply across future runs; no generalization claim.", + "No speed or memory improvement claim." + ] +} diff --git a/docs/measurements/precondition-state-isolation.md b/docs/measurements/precondition-state-isolation.md new file mode 100644 index 00000000..500393ac --- /dev/null +++ b/docs/measurements/precondition-state-isolation.md @@ -0,0 +1,118 @@ +# Isolate pruning state between independent precondition solves + +`find_precondition_from_single_group` used a mutable default set for +`_pruned_clauses`. A clause rejected as uninformative for one solve remained in +that set and was removed from later independent solves in the same process. +This can suppress a valid precondition based on prior process history, even +without sampling or RNG changes. + +The fix uses a `None` default and allocates a fresh set only when no set is +provided. Recursive calls still receive the same explicit set, and callers that +supply a set retain its update/reuse behavior. No recursion or pruning rule changes. + +## Minimal reproduction + +Each synthetic example has one record with the same process/thread/function. +Solve A has phase `train` in both positive and negative examples; its signal is +`True` for positives and `False` for negatives. A correctly learns +`signal == True` and prunes `phase == train` as irrelevant to A. + +Independent solve B has positive phase `train` and negative phase `eval`. +It needs precisely the phase clause that A pruned. Before the fix, running B +after A loses that clause and returns no precondition. After the fix, B returns +`phase == train`. + +| Sequence in a fresh process | Before: preconditions per call | After | +|---|---|---| +| B | 1 | 1 | +| A, B | 1, **0** | 1, **1** | +| B, A | 1, 1 | 1, 1 | +| B, B | 1, 1 | 1, 1 | + +All examples use the actual `TracePandas`, clause construction, pruning, and +verification code. Python RNG state stays unchanged in every sequence. These are +fresh-process versus warm-process histories, not a multi-worker-count matrix. +Because workers retain Python module state across tasks, this defect can make a +solve depend on which earlier tasks ran in that process, independently of RNG. + +## Observed changes in a complete configured replay + +A separate correctness replay applied only this PR's two production hunks to the +identity-cache candidate (`504e04d`), preserving that candidate's earlier RNG fix. +The standalone source and base of this PR are unchanged. Independent inspection +of all 76 production Python files confirmed that only `precondition.py` changed, +exactly as intended. All 17 combined tests passed (11 cache tests and six state +isolation tests). + +Both unprofiled runs completed on the same 573,494-row, single-rank trace, with +the same Linux runtime (Python 3.11.4, PyTorch 2.16.0.dev20261012+cpu, pandas 2.3.3, +NumPy 2.1.0), one worker, and one native thread. `FunctionLeadRelation` and +`FunctionCoverRelation` were disabled. Input/source hashes and ordered context +states remained unchanged. + +| Output | Cache only | Cache + state isolation | +|---|---:|---:| +| Emitted invariants | 835 | 836 | +| Failed hypotheses | 105 | 104 | + +All 940 hypothesis identities match after excluding preconditions and accounting +for the failed-record wrapper's example-count fields. One formerly failed +`APIContainRelation` is now emitted; none moved from emitted to failed. Of the +835 previously emitted records, 833 remain structurally identical and **two gain +additional constraints**. These two differences persist after normalizing clause, +disjunction, and constant-value order; they are not ordering noise. + +| API containment predicate | Observed change | +|---|---| +| `Linear.forward` calls `checkpoint._internal_assert` | Previously failed; now emitted with six autocast/context/thread clauses. | +| `RMSNorm.forward` calls `checkpoint._internal_assert` | Retains its step clause and gains six autocast/context/thread clauses (one to seven). | +| `get_rng_state` calls `current_device` | Retains both CUDA-device clauses and gains a thread clause (two to three). | + +The two existing predicates become more restrictive through added conjuncts. +All three changed/new predicates include concrete trace-specific thread IDs. +Their applicability to future checking runs is unvalidated; these results do not +establish generalization or ground-truth correctness. This single replay pair is +not a speed or memory comparison. [Compact replay evidence](precondition-state-isolation-replay-20261025.json) +records source composition, checks, output hashes, and the observed structures +without publishing literal thread IDs or raw traces. + +## Validation and scope + +Six focused regression tests had five failures before the fix and all pass +afterward. They cover independent/reversed/repeated solves, fresh versus warm +processes, caller-supplied sets, and recursion. The recursive fixture splits two +positive groups into two valid preconditions; siblings share one set, while a +new root solve receives a different set. Repository-pinned lint, type, and format +hooks pass. + +```sh +OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MKL_NUM_THREADS=1 \ +python -m unittest tests.test_precondition_state_isolation -v +``` + +This is an independent correctness fix. Recovering previously suppressed clauses +can add preconditions or otherwise change inference outputs. No unchanged +large-trace output, global order independence, scheduling determinism, or +performance improvement is claimed. The identity-cache performance comparison +uses its own unchanged candidates. + +[Compact reproduction evidence](precondition-state-isolation-20261025.json) +records the fixture, before/after counts, source hashes, test results, and limits. + +## Engineering contract checks + +The regression fixture now has explicit record, example, solve, and recursion +observer types. The observer requires the actual recursive call parameters rather +than accepting arbitrary keyword arguments, and fresh-process comparisons check +the entire expected serialized precondition. The production fix is unchanged. + +Pinned Mypy 1.9 reports **14 errors before and zero after** on the fixture with +`--check-untyped-defs --disallow-untyped-defs --disallow-any-generics`, +`--ignore-missing-imports`, and `--follow-imports=silent`. The last option limits +diagnostics to this fixture; it does not certify strict typing of imported legacy +modules. All six regressions pass on Python 3.10.16 / PyTorch 2.2.2 / pandas 2.2.3. +The configured hooks also pass on the production function and fixture; the hooks +exclude tests from most checks, so the explicit fixture check is necessary. + +These checks add no new performance measurement. The earlier replay results above +remain tied to their recorded source revisions. diff --git a/docs/measurements/trace-pandas-indexes-20261025.json b/docs/measurements/trace-pandas-indexes-20261025.json new file mode 100644 index 00000000..94cf3a94 --- /dev/null +++ b/docs/measurements/trace-pandas-indexes-20261025.json @@ -0,0 +1,319 @@ +{ + "measurement_date": "2026-10-25", + "source": { + "repository": "https://github.com/OrderLab/TrainCheck", + "base_commit": "9eb4f320d3db34d372145261704b805b88e39bda", + "unchanged_prerequisites": [ + "deterministic inference sampling", + "example reservoir cap fixes" + ], + "trace_pandas_sha256": { + "legacy": "e1bb62eae653f95c5498dc921d991af013e8b5717398f94cfbc718812bb600ad", + "context_index": "13d5d1273595f8183d0511ae288a998cb57e3ea66dc232e4bfec503739de825c", + "context_and_call_index": "8e4c3589371449b78430ed5039d0159d3d0435664b218f3a96c81935be0d75d5" + }, + "pr_copies": { + "variants": { + "context_index": { + "measured_source_sha256": "13d5d1273595f8183d0511ae288a998cb57e3ea66dc232e4bfec503739de825c", + "pr_source_sha256": "52c296917bba877b1e87aebb54cfee1e05813e2ae32f85744d5ddfe27383d5e0", + "equivalent_ast_after_normalizing_only_documented_typing_change": true + }, + "context_and_call_index": { + "measured_source_sha256": "8e4c3589371449b78430ed5039d0159d3d0435664b218f3a96c81935be0d75d5", + "pr_source_sha256": "58aebdba9955834f4d6b152527e8ba2f9166f6e62f43a098c48b115b0bbc48f5", + "equivalent_ast_after_normalizing_only_documented_typing_change": true + } + }, + "changes_from_measured_sources": [ + "Both PR copies widen the inherited call-event cache annotation to include IncompleteFuncCallEvent, matching the base Trace contract, and import that existing type.", + "Final PR copy adds narrow E721 suppressions to preserve intentional exact-string checks; only comments/parentheses changed in lookup logic.", + "Existing explicit None-or-cache annotations are retained for the new lazy index." + ], + "no_frozen_source_edits": true, + "local_validation_tools": { + "ruff": "0.15.7", + "mypy": "1.19.1", + "black": "26.3.1", + "isort": "6.0.1" + }, + "pinned_repository_hooks_need_final_worktree_verification": true + } + }, + "saved_trace": { + "events": 573494, + "model_parameters": 6163712, + "training_steps": 10, + "archive_sha256": "62022da5a997c16ffe49aa44486a495b651c8758199f29732a216c7cf257559d", + "raw_inputs_unchanged": true, + "collection_pytorch": "2.16.0.dev20261012+cu130" + }, + "runtime": { + "platform": "Linux", + "python": "3.10.16", + "pandas": "2.3.3", + "numpy": "1.26.4", + "torch": "2.2.2+cu121", + "workers": 1, + "native_threads": 1 + }, + "preparation": { + "runs": { + "legacy": { + "context_index_seconds": 159.43910144269466, + "load_seconds": 90.64373006299138, + "rows": 573494, + "stage_rows": { + "training": 573494 + }, + "stage_split_seconds": 80.21087664365768, + "total_seconds": 170.85460681840777, + "sampled_peak_pss_bytes": 2409772032, + "context_states": [ + { + "call": 0, + "contexts": 711, + "rows": 573494, + "seconds": 79.72032228112221, + "states_sha256": "a5ddb221a3679a50c476776b19aaa3cbc5277174775c8559eb0dcef0a28e2dd0" + }, + { + "call": 1, + "contexts": 711, + "rows": 573494, + "seconds": 79.71877916157246, + "states_sha256": "a5ddb221a3679a50c476776b19aaa3cbc5277174775c8559eb0dcef0a28e2dd0" + } + ] + }, + "context_index": { + "context_index_seconds": 0.8803581297397614, + "load_seconds": 10.837436098605394, + "rows": 573494, + "stage_rows": { + "training": 573494 + }, + "stage_split_seconds": 0.8764883801341057, + "total_seconds": 11.713924750685692, + "sampled_peak_pss_bytes": 2411297792, + "context_states": [ + { + "call": 0, + "contexts": 711, + "rows": 573494, + "seconds": 0.4527508243918419, + "states_sha256": "a5ddb221a3679a50c476776b19aaa3cbc5277174775c8559eb0dcef0a28e2dd0" + }, + { + "call": 1, + "contexts": 711, + "rows": 573494, + "seconds": 0.42760730534791946, + "states_sha256": "a5ddb221a3679a50c476776b19aaa3cbc5277174775c8559eb0dcef0a28e2dd0" + } + ] + } + }, + "comparison": { + "same_number_of_calls": true, + "calls": [ + { + "call": 0, + "rows": 573494, + "contexts": 711, + "equal": true + }, + { + "call": 1, + "rows": 573494, + "contexts": 711, + "equal": true + } + ], + "all_equal": true + }, + "paired_repetitions": 1, + "timing_scope": "Load/init plus stage splitting, excluding process startup; includes context-state serialization. Context-index timers exclude serialization. PSS peaks are sampled." + }, + "large_trace_diagnostic": { + "runs": { + "optimized-infer-profile": { + "wall_seconds": 546.4971170388162, + "exit_code": 1, + "stop_reason": null, + "source_unchanged": true, + "traces_unchanged": true, + "error": "ValueError: Too many positional arguments for function torch._prims_common.is_contiguous, expecting 1 (['a']) but got 2", + "lookups": [ + { + "function": "get_pre_func_call_record", + "primitive_calls": 6891, + "calls": 6891, + "self_seconds": 0.087277701, + "cumulative_seconds": 187.91447304200003 + }, + { + "function": "get_post_func_call_record", + "primitive_calls": 13498, + "calls": 13498, + "self_seconds": 0.18074669200000001, + "cumulative_seconds": 289.47232359500003 + } + ] + }, + "call-index-infer-profile": { + "wall_seconds": 71.98188185319304, + "exit_code": 1, + "stop_reason": null, + "source_unchanged": true, + "traces_unchanged": true, + "error": "ValueError: Too many positional arguments for function torch._prims_common.is_contiguous, expecting 1 (['a']) but got 2", + "lookups": [ + { + "function": "get_pre_func_call_record", + "primitive_calls": 6891, + "calls": 6891, + "self_seconds": 0.020590995, + "cumulative_seconds": 1.034124108 + }, + { + "function": "get_post_func_call_record", + "primitive_calls": 13498, + "calls": 13498, + "self_seconds": 0.038829666000000006, + "cumulative_seconds": 3.888496781 + } + ] + } + }, + "same_error": true, + "completed_inference": false + }, + "diagnostic_scope": { + "completed_inference": false, + "profiled": true, + "disabled_relations": [ + "FunctionLeadRelation", + "FunctionCoverRelation" + ], + "same_error_is_framework_version_mismatch": true, + "eligible_containment_apis_in_replay": 35, + "eligible_containment_apis_in_collection_environment_log": 41, + "time_to_error_is_not_completed_inference_throughput": true + }, + "correctness": { + "differential_tests": 32, + "context_randomized_traces": 80, + "independent_extra_call_query_comparisons": 640, + "small_replay_run_count": 9, + "all_full_json_comparisons_equal": true, + "comparison": "Structural JSON multisets; object-key and file-line order ignored, multiplicity and array order retained.", + "runs": [ + { + "case": "mnist-fixed-seed42", + "exit_code": 0, + "workers": 1, + "valid_outputs": true, + "source": "base" + }, + { + "case": "mnist-fixed-seed42", + "exit_code": 0, + "workers": 1, + "valid_outputs": true, + "source": "optimized" + }, + { + "case": "transformer-fixed-seed42", + "exit_code": 0, + "workers": 1, + "valid_outputs": true, + "source": "optimized" + }, + { + "case": "transformer-fixed-seed42", + "exit_code": 0, + "workers": 1, + "valid_outputs": true, + "source": "base" + }, + { + "case": "dcgan-fixed-seed42", + "exit_code": 0, + "workers": 1, + "valid_outputs": true, + "source": "base" + }, + { + "case": "dcgan-fixed-seed42", + "exit_code": 0, + "workers": 1, + "valid_outputs": true, + "source": "optimized" + }, + { + "case": "mnist-fixed-seed42", + "exit_code": 0, + "workers": 1, + "valid_outputs": true, + "source": "context_and_call_index" + }, + { + "case": "transformer-fixed-seed42", + "exit_code": 0, + "workers": 1, + "valid_outputs": true, + "source": "context_and_call_index" + }, + { + "case": "dcgan-fixed-seed42", + "exit_code": 0, + "workers": 1, + "valid_outputs": true, + "source": "context_and_call_index" + } + ], + "outputs_by_workload": { + "mnist-fixed-seed42": { + "invariants": { + "canonical_multiset_sha256": "5691e461a1fba5bf5314b18a62bf72f82fc95e56f71b26eca87218e53f321e84", + "distinct_records": 193, + "records": 193 + }, + "failed_hypotheses": { + "canonical_multiset_sha256": "8e39cf710d943898ca007478953584d363db135ce3cb458619e71836ae179820", + "distinct_records": 21, + "records": 21 + } + }, + "transformer-fixed-seed42": { + "invariants": { + "canonical_multiset_sha256": "21332736553d2fe20c1fdb1e2c852597950e3b799bc399d47007a2b1f7a71b09", + "distinct_records": 518, + "records": 518 + }, + "failed_hypotheses": { + "canonical_multiset_sha256": "b3424febf9358e01c3e079362e17bc68a939a933e14f1cfab954884482073bd7", + "distinct_records": 10, + "records": 10 + } + }, + "dcgan-fixed-seed42": { + "invariants": { + "canonical_multiset_sha256": "596f82fcfb946a7fe2fe138fa9e61302bafb93465fdc529317ccf93d2da1022a", + "distinct_records": 512, + "records": 512 + }, + "failed_hypotheses": { + "canonical_multiset_sha256": "dfba6a7ed565ce66163b9eb98921ad05857649e37f78075b48fdf9341b912204", + "distinct_records": 86, + "records": 86 + } + } + } + }, + "limitations": [ + "Event row order, call IDs, and event types must remain stable after initialization. Frame replacement or a length change rebuilds the call index; same-size in-place changes or reordering are unsupported.", + "The large trace uses a different PyTorch version; both diagnostic variants end at the same signature error. No complete large-trace inference or online-checker overhead result is claimed." + ] +} diff --git a/docs/measurements/trace-pandas-indexes.md b/docs/measurements/trace-pandas-indexes.md new file mode 100644 index 00000000..03e5d3b0 --- /dev/null +++ b/docs/measurements/trace-pandas-indexes.md @@ -0,0 +1,38 @@ +# TracePandas indexing measurements + +The context index removes two full-frame scans per context instance. The call-ID index removes repeated full-frame scans for pre/post lookups, including lookups made before containment sampling. The optimizations preserve selection coverage and record-validation behavior. + +The adjacent `trace-pandas-indexes-20261025.json` records sanitized measurements, source/input hashes, environment versions, state equality, profiler call counts, and nine successful small-workload replay results. The complete saved Llama trace contains 573,494 events from ten training steps of a 6,163,712-parameter model. + +| Measurement | Before | After | +|---|---:|---:| +| Preparation, original → context index | 170.855 s | 11.714 s | +| Context-index work within preparation | 159.439 s | 0.880 s | +| Pre-record profiling, context → combined | 187.914 s / 6,891 calls | 1.034 s / 6,891 calls | +| Post-record profiling, context → combined | 289.472 s / 13,498 calls | 3.888 s / 13,498 calls | +| Diagnostic wall time to the same error | 546.50 s | 71.98 s | + +Preparation is one paired measurement on the same Linux host using one native thread. Total preparation includes state serialization and excludes process startup; context-index timers exclude serialization. Both preparation variants produce the same 711 ordered states in the full trace and stage trace. The profiler's cumulative times overlap across callers and must not be added indiscriminately. + +The large-trace diagnostics both fail at `torch._prims_common.is_contiguous` argument binding: replay PyTorch 2.2.2 expects one argument while the PyTorch 2.16.0.dev20261012 trace contains two. These results demonstrate reduced search cost, not completed large-trace inference throughput or online-checker overhead. The replay environment also resolves fewer eligible containment APIs than the collection environment (35 versus 41), so a final reference run needs compatible framework dependencies. + +Nine small-workload runs complete successfully: original/context/combined variants for MNIST, small Transformer, and DCGAN. Full invariant and failed-hypothesis JSON match under structural multiset comparison, retaining multiplicity and array order. Per-workload counts are 193/21, 518/10, and 512/86. + +## Reproduce local correctness and synthetic measurements + +From a checkout with TrainCheck dependencies installed: + +```sh +OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 python -m unittest tests.test_context_index tests.test_call_index +OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 python -m tools.benchmarks.trace_pandas_indexes --output measurements.json +``` + +The synthetic benchmark constructs its inputs and needs no external trace or cluster. It alternates variant order, checks exact results, and reports every repetition and the median. Its context case times construction plus state serialization; its call case times construction and all lookups, including the first lazy-index build. Its sizes/repetition count are configurable; these synthetic timings are separate from the saved-trace measurements above. + +The oracle fixture contains only the exact original context/pre/post/stage methods from commit `9eb4f320d3db34d372145261704b805b88e39bda`; supporting types and the constructor come from the package under test. Provenance includes the original file and method hashes. The fixture's `TracePandas` binding ensures stage splitting creates legacy instances, and the tests assert this explicitly. + +The published source also corrects the inherited call-event cache annotation to include `IncompleteFuncCallEvent`; narrow lint comments preserve exact-string guards. These typing/comment changes do not alter lookup behavior. + +## Cache contract + +Event rows, their order, call IDs, and event types must remain stable after initialization. Frame replacement or a length change rebuilds the call index. Same-size in-place edits or row reordering can leave stale positions and require a new `TracePandas` instance. The tested inference path does not perform those mutations. This change does not introduce general cache invalidation for arbitrary dataframe mutations. diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/fixtures/legacy_trace_pandas.provenance.json b/tests/fixtures/legacy_trace_pandas.provenance.json new file mode 100644 index 00000000..00498a4d --- /dev/null +++ b/tests/fixtures/legacy_trace_pandas.provenance.json @@ -0,0 +1,13 @@ +{ + "repository": "https://github.com/OrderLab/TrainCheck", + "commit": "9eb4f320d3db34d372145261704b805b88e39bda", + "source_path": "traincheck/trace/trace_pandas.py", + "source_sha256": "e1bb62eae653f95c5498dc921d991af013e8b5717398f94cfbc718812bb600ad", + "method_sha256": { + "_index_context_manager_meta_vars": "6b7e5646260ee0ce7377c7cf19edb4774bdf69825e15cf10ff3c29cd4555c13c", + "get_pre_func_call_record": "bf89fcd11b98e08a3029ee6ff3b2a4b6df980aa49bfe6c13bcdbafcb10e8d74c", + "get_post_func_call_record": "a36cd5bc217f8e85f0f30775c24480f0060b6f3496e8fb2a2cb45a4214d46c07", + "get_traces_for_stage": "583c5adfd122d86027f427beea7c7823783e14b37f078d01740f55cdd72f04f3" + }, + "hash_convention": "Original indented method source including its final newline. No source transformations." +} diff --git a/tests/fixtures/legacy_trace_pandas.py b/tests/fixtures/legacy_trace_pandas.py new file mode 100644 index 00000000..c7256089 --- /dev/null +++ b/tests/fixtures/legacy_trace_pandas.py @@ -0,0 +1,244 @@ +"""Frozen legacy query methods used as a differential oracle. + +Source: OrderLab/TrainCheck commit 9eb4f320d3db34d372145261704b805b88e39bda, +traincheck/trace/trace_pandas.py (SHA256 + e1bb62eae653f95c5498dc921d991af013e8b5717398f94cfbc718812bb600ad). +Only four exact original methods are retained. The production constructor and +supporting types are shared so tests isolate these query/index changes. Defining +this class as TracePandas intentionally makes the original stage factory create +legacy instances too. See legacy_trace_pandas.provenance.json for method hashes. +""" + +import logging + +import pandas as pd + +from traincheck.instrumentor.tracer import TraceLineType +from traincheck.instrumentor.types import PTID +from traincheck.trace.trace_pandas import STAGE_KEY +from traincheck.trace.trace_pandas import TracePandas as ProductionTracePandas +from traincheck.trace.types import ContextManagerState, Liveness +from traincheck.trace.utils import ( + BindedFuncInput, + bind_args_kwargs_to_signature, + load_signature_from_class_method_name, +) +from traincheck.utils import safe_isnan + +logger = logging.getLogger(__name__) + + +class TracePandas(ProductionTracePandas): + def _index_context_manager_meta_vars(self): + """Identify context manager entry and exit events, and add them to the meta_vars.""" + # 1. Find all trace records that are related to __enter__ and __exit__ functions + if "function" not in self.events.columns: + self.context_manager_states = None + return + + if ( + hasattr(self, "context_manager_states") + and self.context_manager_states is not None + ): + return self.context_manager_states + + all_context_managers: dict[PTID, list[ContextManagerState]] = {} + + context_manager_names = self.events[ + (self.events["function"].str.contains("__enter__", na=False)) + & ( + ~self.events["function"].str.contains( + "torch.autograd.grad_mode", na=False + ) + ) # HACK: ignore all autograd related context managers for now as it doesn't have an __init__ event + & ( + ~self.events["function"].str.contains( + "torch.autograd.profiler.record_function", na=False + ) + ) + ]["function"].unique() + + context_manager_init_pre_records = [] + for context_manager_name in context_manager_names: + context_manager_name = context_manager_name.removesuffix(".__enter__") + context_manager_init_pre_records.extend( + self.events[ + (self.events["function"] == f"{context_manager_name}.__init__") + & (self.events["type"] == TraceLineType.FUNC_CALL_PRE) + ].to_dict(orient="records") + ) + + for context_manager_init_pre_record in context_manager_init_pre_records: + init_pre_record = context_manager_init_pre_record + process_id = init_pre_record["process_id"] + thread_id = init_pre_record["thread_id"] + + context_manager_name = init_pre_record["function"].removesuffix(".__init__") + + # find nearest enter and exit events with the same obj_id + obj_id = init_pre_record["obj_id"] + try: + enter_post_record = ( + self.events[ + (self.events["type"] == TraceLineType.FUNC_CALL_POST) + & ( + self.events["function"] + == f"{context_manager_name}.__enter__" + ) + & (self.events["obj_id"] == obj_id) + & (self.events["time"] > init_pre_record["time"]) + & (self.events["process_id"] == process_id) + & (self.events["thread_id"] == thread_id) + ] + .iloc[0] + .to_dict() + ) + + exit_pre_record = ( + self.events[ + (self.events["type"] == TraceLineType.FUNC_CALL_PRE) + & ( + self.events["function"] + == f"{context_manager_name}.__exit__" + ) + & (self.events["obj_id"] == obj_id) + & (self.events["time"] > enter_post_record["time"]) + & (self.events["process_id"] == process_id) + & (self.events["thread_id"] == thread_id) + ] + .iloc[0] + .to_dict() + ) + except IndexError: + # sometimes this happens. an enter might not have a corresponding exit, vice versa. e.g. torch.serialization._opener.__enter__ and torch.serialization._open_zipfile_writer_file.__exit__ + logger.warning( + f"Context manager {context_manager_name} not used properly. Skipping." + ) + continue + + assert ( + enter_post_record["time"] < exit_pre_record["time"] + ), f"enter not before exit: {enter_post_record.to_dict()} {exit_pre_record.to_dict()}" + + start_time, end_time = ( + enter_post_record["time"], + exit_pre_record["time"], + ) + + args = init_pre_record["args"] + kwargs = init_pre_record["kwargs"] + if not safe_isnan(args): + signature = load_signature_from_class_method_name( + init_pre_record["function"] + ) + + binded_args_and_kwargs = bind_args_kwargs_to_signature( + args, kwargs, signature + ) + else: + # create an empty BindedFuncInput if args is NaN, as it indicates + # that we did not record the args and kwargs for this function call + binded_args_and_kwargs = BindedFuncInput({}) + + ptid = PTID(process_id, thread_id) + if ptid not in all_context_managers: + all_context_managers[ptid] = [] + + logger.debug(f"Adding context manager: {context_manager_name} {ptid}") + all_context_managers[ptid].append( + ContextManagerState( + name=init_pre_record["function"].removesuffix(".__init__"), + ptid=ptid, + liveness=Liveness(start_time, end_time), + input=binded_args_and_kwargs, + ) + ) + + logger.info(f"Found {len(all_context_managers)} context managers.") + + self.context_manager_states = all_context_managers + + def get_pre_func_call_record(self, func_call_id: str) -> dict: + """Get the pre call record of a function given its func_call_id.""" + # TODO: check if this is correct + pre_records = self.events[ + (self.events["func_call_id"] == func_call_id) + & (self.events["type"] == TraceLineType.FUNC_CALL_PRE) + ] + + assert ( + len(pre_records) == 1 + ), f"{len(pre_records)} pre call events found for {func_call_id}, expected 1" + + pre_record = pre_records.iloc[0].to_dict() + + return pre_record + + def get_post_func_call_record(self, func_call_id: str) -> dict | None: + """Get the post call record of a function given its func_call_id. + Returns None if the post call event is not found and the pre-call event is the outermost function call. (see the doc of `_rm_incomplete_trailing_func_calls`) + """ + # TODO: check if this is correct + post_records = self.events[ + (self.events["func_call_id"] == func_call_id) + & ( + self.events["type"].isin( + [ + TraceLineType.FUNC_CALL_POST, + TraceLineType.FUNC_CALL_POST_EXCEPTION, + ] + ) + ) + ] + + assert ( + len(post_records) <= 1 + ), f"{post_records.shape[0]} post call events found for {func_call_id}, expected 1" + + if len(post_records) == 0: + logger.warning(f"No post call event found for {func_call_id}") + # check if the pre-call event is the outermost function call + pre_record = self.get_pre_func_call_record(func_call_id) + + outermost_func_call_pre = ( + self.events[ + (self.events["type"] == TraceLineType.FUNC_CALL_PRE) + & (self.events["process_id"] == pre_record["process_id"]) + & (self.events["thread_id"] == pre_record["thread_id"]) + ] + .iloc[0] + .to_dict() + ) + + if pre_record["func_call_id"] == outermost_func_call_pre["func_call_id"]: + return None + else: + raise ValueError( + f"No post call event found for {func_call_id}, but it is not the outermost function call." + ) + + post_record = post_records.iloc[ + 0 + ].to_dict() # order of events doesn't matter as the result here is a single row + + return post_record + + def get_traces_for_stage(self) -> dict[str, "TracePandas"]: # type: ignore + """Get the traces split by stages.""" + + if not self.is_stage_annotated(): + raise ValueError("Trace is not annotated with stages.") + + if self.stage_traces: + return self.stage_traces + + traces = {} + for stage in self.events[STAGE_KEY].unique(): + traces[stage] = TracePandas( + self.events[self.events[STAGE_KEY] == stage], + truncate_incomplete_func_calls=False, + ) + + self.stage_traces = traces + + return traces diff --git a/tests/test_call_index.py b/tests/test_call_index.py new file mode 100644 index 00000000..acf8f030 --- /dev/null +++ b/tests/test_call_index.py @@ -0,0 +1,251 @@ +"""Call-record index differential tests against frozen legacy query methods.""" + +import copy +import logging +import unittest + +import numpy as np +import pandas as pd + +import traincheck.trace.trace_pandas as NEW +from tests.fixtures import legacy_trace_pandas as OLD + +PRE = OLD.TraceLineType.FUNC_CALL_PRE +POST = OLD.TraceLineType.FUNC_CALL_POST +EXCEPTION = OLD.TraceLineType.FUNC_CALL_POST_EXCEPTION + + +def record(call_id, when, kind, *, pid=1, tid=2, stage="train", **extra): + return { + "func_call_id": call_id, + "function": "fixtures.work", + "time": when, + "type": kind, + "process_id": pid, + "thread_id": tid, + "args": {"0": {"int": 7}}, + "kwargs": {}, + "return_values": {"int": 9}, + "exception": "fixture exception", + "meta_vars.stage": stage, + **extra, + } + + +def pair(call_id="call", start=0, **extra): + return [ + record(call_id, start, PRE, **extra), + record(call_id, start + 1, POST, **extra), + ] + + +def make(module, frame, truncate=False): + if not isinstance(frame, pd.DataFrame): + frame = pd.DataFrame(copy.deepcopy(frame)) + else: + frame = frame.copy(deep=True) + return module.TracePandas(frame, truncate_incomplete_func_calls=truncate) + + +def outcome(trace, method, call_id): + try: + return ("ok", getattr(trace, method)(call_id)) + except Exception as exc: + return (type(exc).__name__, str(exc)) + + +class StringAlias: + def __eq__(self, other): + return other == "alias" + + def __repr__(self): + return "StringAlias()" + + +class CallIndexTests(unittest.TestCase): + def setUp(self): + previous = logging.root.manager.disable + logging.disable(logging.WARNING) + self.addCleanup(logging.disable, previous) + + def equivalent(self, rows, call_ids, *, truncate=False): + old, new = [make(module, rows, truncate) for module in [OLD, NEW]] + for call_id in call_ids: + for method in ["get_pre_func_call_record", "get_post_func_call_record"]: + with self.subTest(call_id=repr(call_id), method=method): + expected = outcome(old, method, call_id) + actual = outcome(new, method, call_id) + self.assertEqual(expected, actual) + return old, new + + def test_exact_pre_post_records_and_row_order(self): + rows = pair("z", 10) + pair("a", 0, pid=9, tid=8) + _, new = self.equivalent(rows, ["a", "z", "missing"]) + self.assertEqual(new.get_pre_func_call_record("z"), rows[0]) + self.assertEqual(new.get_post_func_call_record("a"), rows[3]) + self.assertIs(type(new.get_pre_func_call_record("z")["time"]), int) + + def test_exception_post_record(self): + rows = [record("call", 0, PRE), record("call", 1, EXCEPTION)] + old, new = self.equivalent(rows, ["call"]) + self.assertEqual( + old.query_func_call_event("call").__dict__, + new.query_func_call_event("call").__dict__, + ) + + def test_duplicate_pre_count_assertions(self): + rows = [record("bad", 0, PRE), record("bad", 1, PRE), record("bad", 2, POST)] + old, new = self.equivalent(rows, ["bad"]) + self.assertEqual( + outcome(new, "get_pre_func_call_record", "bad"), + ("AssertionError", "2 pre call events found for bad, expected 1"), + ) + + def test_duplicate_post_and_exception_count_assertions(self): + rows = [ + record("bad", 0, PRE), + record("bad", 1, POST), + record("bad", 2, EXCEPTION), + ] + self.equivalent(rows, ["bad"]) + + def test_unqueried_duplicates_do_not_raise_early(self): + rows = pair("good") + [record("bad", 4, PRE), record("bad", 5, PRE)] + _, new = self.equivalent(rows, ["good"]) + self.assertIsNotNone(new._func_call_positions) + self.assertEqual(len(new._func_call_positions["bad"][0]), 2) + + def test_missing_outermost_and_nonoutermost_posts_per_thread(self): + rows = [ + record("outer", 100, PRE), + record("inner", 0, PRE), + record("other_thread", 2, PRE, tid=3), + ] + old, new = self.equivalent(rows, ["outer", "inner", "other_thread", "absent"]) + self.assertIsNone(new.get_post_func_call_record("outer")) + self.assertIsNone(new.get_post_func_call_record("other_thread")) + self.assertEqual( + outcome(new, "get_post_func_call_record", "inner"), + ( + "ValueError", + "No post call event found for inner, but it is not the outermost function call.", + ), + ) + + def test_duplicate_dataframe_labels_use_positions(self): + rows = pair("a") + pair("b", 4) + self.equivalent(pd.DataFrame(rows, index=[17, 17, 17, 17]), ["a", "b"]) + + def test_null_ids_never_use_dictionary_equality(self): + rows = pair("good") + for value in [None, float("nan"), pd.NA]: + rows += pair(value, 5) + _, new = self.equivalent(rows, ["good", None, float("nan"), pd.NA]) + self.assertIsNotNone(new._func_call_positions) + + def test_numeric_ids_and_custom_equality_fall_back(self): + for rows, ids in [ + (pair(7), [7, "7"]), + (pair(StringAlias()), ["alias"]), + (pair(np.str_("numpy")), ["numpy"]), + (pair("good") + pair(7, 5), ["good", 7]), + ]: + old, new = self.equivalent(rows, ids) + self.assertIsNone(new._func_call_positions) + + def test_pandas_string_and_unsupported_categorical_dtypes(self): + for dtype in ["string", "category"]: + frame = pd.DataFrame(pair("a") + pair("b", 4)) + for column in ["func_call_id", "type"]: + frame[column] = frame[column].astype(dtype) + _, new = self.equivalent(frame, ["a", "b"]) + if dtype == "category": + self.assertIsNone(new._func_call_positions) + else: + self.assertIsNotNone(new._func_call_positions) + + def test_null_event_types_are_ignored(self): + rows = pair("good") + rows += [record("extra", 3, value) for value in [None, float("nan"), pd.NA]] + self.equivalent(rows, ["good", "extra"]) + + def test_stage_split_indexes_are_independent(self): + rows = pair("init", stage="init") + pair("train", 5, stage="train") + rows += [ + record("cross", 10, PRE, stage="init"), + record("cross", 11, POST, stage="train"), + ] + old, new = self.equivalent(rows, ["init", "train", "cross"]) + old_stages = old.get_traces_for_stage() + new_stages = new.get_traces_for_stage() + for stage in old_stages: + self.assertIs(type(old_stages[stage]), OLD.TracePandas) + self.assertIs(type(new_stages[stage]), NEW.TracePandas) + for call_id in ["init", "train", "cross"]: + for method in ["get_pre_func_call_record", "get_post_func_call_record"]: + self.assertEqual( + outcome(old_stages[stage], method, call_id), + outcome(new_stages[stage], method, call_id), + ) + self.assertIsNot( + new._func_call_positions, new_stages["train"]._func_call_positions + ) + self.assertIsNot( + new_stages["init"]._func_call_positions, + new_stages["train"]._func_call_positions, + ) + + def test_top_level_return_dicts_are_fresh(self): + for module in [OLD, NEW]: + trace = make(module, pair()) + for method in ["get_pre_func_call_record", "get_post_func_call_record"]: + first = getattr(trace, method)("call") + second = getattr(trace, method)("call") + self.assertIsNot(first, second) + first["function"] = "mutated" + first["new_key"] = True + self.assertEqual(getattr(trace, method)("call"), second) + + def test_index_is_lazy_reused_and_rebuilt_on_frame_replacement(self): + old, new = [make(module, pair()) for module in [OLD, NEW]] + self.assertIsNone(new._func_call_positions) + new.get_pre_func_call_record("call") + cached = new._func_call_positions + new.get_post_func_call_record("call") + self.assertIs(new._func_call_positions, cached) + for trace in [old, new]: + trace.events = pd.DataFrame(pair("replacement", 9)) + for call_id in ["call", "replacement"]: + for method in ["get_pre_func_call_record", "get_post_func_call_record"]: + self.assertEqual( + outcome(old, method, call_id), outcome(new, method, call_id) + ) + self.assertIsNot(new._func_call_positions, cached) + + def test_in_place_row_count_change_rebuilds(self): + old, new = [make(module, pair()) for module in [OLD, NEW]] + new.get_pre_func_call_record("call") + cached = new._func_call_positions + for trace in [old, new]: + trace.events.loc[len(trace.events)] = record("call", 4, PRE) + self.assertEqual( + outcome(old, "get_pre_func_call_record", "call"), + outcome(new, "get_pre_func_call_record", "call"), + ) + self.assertIsNot(new._func_call_positions, cached) + + def test_index_built_after_incomplete_trailing_call_truncation(self): + rows = [ + record("outer", 0, PRE), + record("inner", 1, PRE), + record("inner", 2, POST), + record("outer", 10, POST), + record("trailing", 11, PRE), + ] + old, new = self.equivalent(rows, ["outer", "inner", "trailing"], truncate=True) + self.assertEqual(len(new.events), 4) + self.assertIs(new._func_call_positions_source, new.events) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/test_context_index.py b/tests/test_context_index.py new file mode 100644 index 00000000..a6390d8e --- /dev/null +++ b/tests/test_context_index.py @@ -0,0 +1,287 @@ +"""Differential context-index tests against frozen legacy query methods.""" + +import inspect +import logging +import random +import unittest +from unittest.mock import patch + +import numpy as np +import pandas as pd + +import traincheck.trace.trace_pandas as NEW +from tests.fixtures import legacy_trace_pandas as OLD + +PRE = OLD.TraceLineType.FUNC_CALL_PRE +POST = OLD.TraceLineType.FUNC_CALL_POST + + +def event( + time, + method, + *, + name="fixtures.Ctx", + obj=1, + pid=10, + tid=20, + kind=None, + args=float("nan"), + kwargs=None, + stage="train", +): + return { + "time": time, + "function": f"{name}.{method}", + "type": kind or (POST if method == "__enter__" else PRE), + "obj_id": obj, + "process_id": pid, + "thread_id": tid, + "args": args, + "kwargs": kwargs or {}, + "meta_vars.stage": stage, + } + + +def cycle(start, **kwargs): + return [ + event(start, "__init__", **kwargs), + event(start + 1, "__enter__", **kwargs), + event(start + 2, "__exit__", **kwargs), + ] + + +def run(module, frame): + trace = module.TracePandas( + frame.copy(deep=True), truncate_incomplete_func_calls=False + ) + return trace, encode(trace) + + +def encode(trace): + states = trace.context_manager_states + if states is None: + return None + return [ + (tuple(key), [state.to_dict() for state in value]) + for key, value in states.items() + ] + + +class ContextIndexTests(unittest.TestCase): + def setUp(self): + previous = logging.root.manager.disable + logging.disable(logging.WARNING) + self.addCleanup(logging.disable, previous) + + def equivalent(self, rows, **frame_options): + frame = ( + rows + if isinstance(rows, pd.DataFrame) + else pd.DataFrame(rows, **frame_options) + ) + original, expected = run(OLD, frame) + optimized, actual = run(NEW, frame) + self.assertEqual(expected, actual) + # Native scalar types remain suitable for the same JSON serialization. + for _, states in actual or []: + for state in states: + self.assertNotIsInstance(state["start_time"], np.generic) + self.assertNotIsInstance(state["end_time"], np.generic) + return original, optimized + + def test_nested_reused_ids_processes_threads_and_class_order(self): + rows = cycle(0, name="fixtures.B", obj=1) + rows += cycle(5, name="fixtures.A", obj=1) + rows += cycle(10, name="fixtures.B", obj=1) + rows += cycle(1, name="fixtures.B", obj=2) + rows += cycle(0, name="fixtures.B", obj=1, pid=11) + rows += cycle(0, name="fixtures.B", obj=1, tid=21) + old, new = self.equivalent(rows) + self.assertEqual( + [s.name for s in new.context_manager_states[OLD.PTID(10, 20)]], + ["fixtures.B", "fixtures.B", "fixtures.B", "fixtures.A"], + ) + for when in [0, 1, 2, 6, 11, 12, 100]: + self.assertEqual( + [s.to_dict() for s in old.query_active_context_managers(when, 10, 20)], + [s.to_dict() for s in new.query_active_context_managers(when, 10, 20)], + ) + + def test_first_row_not_nearest_timestamp(self): + rows = [ + event(0, "__init__"), + event(8, "__enter__"), + event(2, "__enter__"), + event(15, "__exit__"), + event(9, "__exit__"), + ] + _, new = self.equivalent(rows) + state = next(iter(new.context_manager_states.values()))[0] + self.assertEqual((state.liveness.start_time, state.liveness.end_time), (8, 15)) + + def test_equal_timestamps_strictly_later(self): + rows = [ + event(5, "__init__"), + event(5, "__enter__"), + event(6, "__enter__"), + event(6, "__exit__"), + event(7, "__exit__"), + ] + _, new = self.equivalent(rows) + self.assertEqual( + next(iter(new.context_manager_states.values()))[0].liveness.start_time, 6 + ) + + def test_null_keys_do_not_match_each_other(self): + for field in ["obj_id", "process_id", "thread_id"]: + for missing in [None, float("nan"), pd.NA]: + with self.subTest(field=field, missing=missing): + rows = cycle(0) + for row in rows: + row[field] = missing + rows += cycle(5, obj=2) + self.equivalent(rows) + + def test_md_none_key_matches_original_custom_equality(self): + from traincheck.trace.types import MD_NONE + + for field in ["obj_id", "process_id", "thread_id"]: + rows = cycle(0) + for row in rows: + row[field] = MD_NONE() + self.equivalent(rows) + + def test_mixed_md_none_and_null_keys(self): + from traincheck.trace.types import MD_NONE + + rows = [ + event(0, "__init__", obj=None), + event(1, "__enter__", obj=MD_NONE()), + event(2, "__exit__", obj=MD_NONE()), + ] + self.equivalent(rows) + + def test_nullable_numeric_keys(self): + frame = pd.DataFrame(cycle(0) + cycle(5, obj=None) + cycle(10, obj=2)) + for col in ["obj_id", "process_id", "thread_id"]: + frame[col] = frame[col].astype("Int64") + self.equivalent(frame) + + def test_missing_pairs_and_wrong_event_types(self): + rows = [ + event(0, "__init__"), + event(1, "__enter__", kind=PRE), + event(2, "__exit__"), + ] + rows += [event(4, "__init__", obj=2), event(5, "__enter__", obj=2)] + rows += [event(6, "__enter__", obj=3), event(7, "__exit__", obj=3)] + rows += cycle(10, obj=4) + self.equivalent(rows) + + def test_infinite_missing_and_nullable_timestamps(self): + rows = [ + event(float("nan"), "__init__", obj=2), + event(0, "__init__"), + event(float("nan"), "__enter__"), + event(1, "__enter__"), + event(float("inf"), "__exit__"), + ] + self.equivalent(rows) + frame = pd.DataFrame(rows) + frame["time"] = frame["time"].astype("Float64") + for module in [OLD, NEW]: + with self.assertRaises(TypeError): + run(module, frame) + + def test_large_integer_timestamps_preserve_precision(self): + for dtype in ["int64", "uint64", "Int64", "UInt64", "float64", "object"]: + for start in [2**53, 2**60, 2**63 - 4]: + with self.subTest(dtype=dtype, start=start): + frame = pd.DataFrame(cycle(start)) + frame["time"] = frame["time"].astype(dtype) + self.equivalent(frame) + for start in [2**53, 2**60, 2**63 - 8]: + # Object dtype retains a float init query against exact integers. + frame = pd.DataFrame(cycle(start)) + frame["time"] = pd.Series( + [float(start), start + 1, start + 2], dtype=object + ) + self.equivalent(frame) + + def test_duplicate_dataframe_indexes_and_mixed_key_types(self): + rows = cycle(0, obj="1") + cycle(5, obj=1) + cycle(10, obj=True) + self.equivalent(pd.DataFrame(rows, index=[4] * len(rows))) + + def test_no_context_or_no_function(self): + self.equivalent(pd.DataFrame({"time": [0]})) + self.equivalent([event(0, "unrelated")]) + self.equivalent([event(0, "__enter__")]) + + def test_ignore_autograd_contexts(self): + rows = cycle(0, name="torch.autograd.grad_mode.enable_grad") + rows += cycle(3, name="torch.autograd.profiler.record_function") + rows += cycle(6) + _, new = self.equivalent(rows) + self.assertEqual(len(next(iter(new.context_manager_states.values()))), 1) + + def test_stage_split_rebuild_and_cache(self): + rows = cycle(0, stage="init") + cycle(3, stage="train") + rows += [ + event(10, "__init__", obj=2, stage="init"), + event(11, "__enter__", obj=2, stage="train"), + event(12, "__exit__", obj=2, stage="train"), + ] + old, new = self.equivalent(rows) + expected = old.get_traces_for_stage() + actual = new.get_traces_for_stage() + self.assertEqual(list(expected), list(actual)) + for stage in expected: + self.assertIs(type(expected[stage]), OLD.TracePandas) + self.assertIs(type(actual[stage]), NEW.TracePandas) + self.assertEqual(encode(expected[stage]), encode(actual[stage])) + self.assertIs(new.get_traces_for_stage(), actual) + cached = new.context_manager_states + self.assertIs(new._index_context_manager_meta_vars(), cached) + + def test_bindings_and_signature_lookup_remain_per_init(self): + rows = cycle(0, args={"0": {"object": {}}, "1": {"int": 7}}) + rows += cycle(5, args={"0": {"object": {}}, "1": {"int": 9}}) + signatures = [ + inspect.Signature( + [ + inspect.Parameter("self", inspect.Parameter.POSITIONAL_OR_KEYWORD), + inspect.Parameter(name, inspect.Parameter.POSITIONAL_OR_KEYWORD), + ] + ) + for name in ["first", "second"] + ] + outputs = [] + for module in [OLD, NEW]: + with patch.object( + module, "load_signature_from_class_method_name", side_effect=signatures + ) as loader: + outputs.append(run(module, pd.DataFrame(rows))[1]) + self.assertEqual(loader.call_count, 2) + self.assertEqual(*outputs) + + def test_randomized_row_order_and_reused_identity_oracle(self): + for seed in range(80): + rng = random.Random(seed) + rows = [] + for _ in range(20): + rows.extend( + cycle( + rng.randrange(20), + obj=rng.randrange(3), + pid=rng.randrange(2), + tid=rng.randrange(2), + name=rng.choice(["fixtures.A", "fixtures.B"]), + ) + ) + rng.shuffle(rows) + with self.subTest(seed=seed): + self.equivalent(rows) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/test_dynamic_typename.py b/tests/test_dynamic_typename.py new file mode 100644 index 00000000..1ff4d61a --- /dev/null +++ b/tests/test_dynamic_typename.py @@ -0,0 +1,135 @@ +"""Runtime type names must tolerate dynamic metadata without calling it.""" + +import unittest + +import torch + +from traincheck.utils import typename + + +class DynamicMetadata: + __module__ = "fixtures.dynamic" + + def __getattr__(self, name: str) -> object: + if name in {"__qualname__", "__name__"}: + return lambda: None + raise AttributeError(name) + + +class UnsafeTruthValue: + def __bool__(self) -> bool: + raise AssertionError("Metadata truth value must not be evaluated") + + +class VisibleType: + pass + + +class ReportedClass: + __module__ = "fixtures.dynamic" + + def __getattribute__(self, name: str) -> object: + if name == "__class__": + return VisibleType + attribute: object = super().__getattribute__(name) + return attribute + + +class DynamicTypenameTests(unittest.TestCase): + def test_dynamic_callable_metadata_uses_class_name(self) -> None: + for runtime in [False, True]: + self.assertEqual( + typename(DynamicMetadata(), is_runtime=runtime), + "fixtures.dynamic.DynamicMetadata", + ) + + def test_non_string_name_values_fall_back_without_truth_testing(self) -> None: + for value in [object(), 17, ["name"], UnsafeTruthValue()]: + + class Metadata: + __module__ = "fixtures.dynamic" + + def __getattr__(self, name: str) -> object: + if name == "__name__": + return value + raise AttributeError(name) + + for runtime in [False, True]: + self.assertEqual( + typename(Metadata(), is_runtime=runtime), + "fixtures.dynamic.Metadata", + ) + + def test_valid_string_names_still_take_precedence(self) -> None: + class Metadata: + __module__ = "fixtures.dynamic" + + def __getattr__(self, name: str) -> object: + if name == "__qualname__": + return "Outer.call" + if name == "__name__": + raise AssertionError("Qualified name should take precedence") + raise AttributeError(name) + + self.assertEqual(typename(Metadata()), "fixtures.dynamic.Outer.call") + + def test_valid_unqualified_name_is_preserved(self) -> None: + value = DynamicMetadata() + setattr(value, "__name__", "exported") + self.assertEqual(typename(value), "fixtures.dynamic.exported") + + def test_reported_class_fallback_is_preserved(self) -> None: + self.assertEqual(typename(ReportedClass()), "fixtures.dynamic.VisibleType") + + def test_standard_functions_descriptors_tensors_and_parameters(self) -> None: + self.assertEqual(typename(len), "len") + self.assertEqual(typename(str.upper), "str.upper") + tensor = torch.zeros(1) + self.assertEqual(typename(tensor), tensor.type()) + self.assertEqual(typename(torch.nn.Parameter(tensor)), "torch.nn.Parameter") + + def test_invalid_reported_class_name_retains_final_assertion(self) -> None: + class InvalidName: + def __getattribute__(self, name: str) -> object: + if name == "__module__": + return "fixtures.dynamic" + if name in {"__qualname__", "__name__"}: + return None + if name == "__class__": + return InvalidClassName() + raise AttributeError(name) + + class InvalidClassName: + __name__ = 17 + + for runtime in (False, True): + with self.assertRaisesRegex( + AssertionError, "prefix and last_name should be str" + ): + typename(InvalidName(), is_runtime=runtime) + + def test_unexpected_metadata_failure_propagates(self) -> None: + error = ValueError("metadata unavailable") + + class FailingMetadata: + def __getattr__(self, name: str) -> object: + if name == "__qualname__": + raise error + raise AttributeError(name) + + with self.assertRaises(ValueError) as caught: + typename(FailingMetadata()) + self.assertIs(caught.exception, error) + + def test_real_pytorch_dynamic_ops_wrapper(self) -> None: + from torch._inductor.virtualized import ops + + self.assertEqual(typename(ops), "torch._inductor.virtualized.OpsWrapper") + self.assertEqual( + typename(ops, is_runtime=True), + "torch._inductor.virtualized.OpsWrapper", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_example_reservoir.py b/tests/test_example_reservoir.py new file mode 100644 index 00000000..d632ab0a --- /dev/null +++ b/tests/test_example_reservoir.py @@ -0,0 +1,528 @@ +"""Reservoir invariants, weighted population merges, and actual merge call sites.""" + +import multiprocessing +import random +import unittest +from collections import Counter +from concurrent.futures import ProcessPoolExecutor +from dataclasses import dataclass +from functools import partial +from unittest.mock import patch + +import traincheck.config.config as config +from traincheck.infer_engine import InferEngine +from traincheck.invariant.base_cls import ( + APIParam, + Example, + ExampleList, + GroupedPreconditions, + Hypothesis, + Invariant, + Preconditions, + UnconditionalPrecondition, +) +from traincheck.invariant.contain_relation import PARENT_GROUP_NAME, APIContainRelation +from traincheck.parallel import config_snapshot, worker_init + +GROUP = PARENT_GROUP_NAME + + +def example(value, group=GROUP): + return Example({group: [{"id": value}]}) + + +def values(reservoir): + return [item.get_group(GROUP)[0]["id"] for item in reservoir.examples] + + +def reservoir(items, cap, seed=42, groups=None): + with ( + patch.object(config, "MAX_EXAMPLE_LIST_SIZE", cap), + patch.object(config, "EXAMPLE_SAMPLING_SEED", seed), + ): + result = ExampleList({GROUP} if groups is None else groups) + for item in items: + result.add_example(example(item)) + return result + + +def snapshot(reservoir): + return ( + reservoir.num_seen, + reservoir.max_size, + set(reservoir.group_names), + list(reservoir.examples), + reservoir._rng.getstate(), + ) + + +def hypothesis(prefix, positive, negative, cap): + precondition = GroupedPreconditions( + {GROUP: Preconditions([UnconditionalPrecondition()])} + ) + invariant = Invariant( + APIContainRelation, + [ + APIParam("torch.nn.modules.activation.ReLU.forward"), + APIParam("torch.nn.modules.dropout.Dropout.forward"), + ], + precondition, + "ReLU.forward() contains Dropout.forward()", + ) + return Hypothesis( + invariant, + reservoir([f"{prefix}-positive-{i}" for i in range(positive)], cap), + reservoir([f"{prefix}-negative-{i}" for i in range(negative)], cap), + ) + + +def worker_reservoir_result(count): + result = ExampleList({GROUP}) + for index in range(count): + result.add_example(example(index)) + return ( + result.max_size, + config.EXAMPLE_SAMPLING_SEED, + result.num_seen, + values(result), + ) + + +@dataclass +class ReservoirFixtureTrace: + """A picklable trace carrying populations, without any trace parser work.""" + + prefix: str + positive_count: int + negative_count: int + + def is_stage_annotated(self): + return False + + +class ReservoirFixtureRelation(APIContainRelation): + @classmethod + def generate_hypothesis(cls, trace): + # Read config directly here: patching construction inside the fixture + # would mask a missing config_snapshot/worker_init propagation field. + precondition = GroupedPreconditions( + {GROUP: Preconditions([UnconditionalPrecondition()])} + ) + invariant = Invariant( + cls, + [APIParam("fixture.parent"), APIParam("fixture.child")], + precondition, + "fixture.parent contains fixture.child", + ) + positive, negative = ExampleList({GROUP}), ExampleList({GROUP}) + for kind, count, examples in [ + ("positive", trace.positive_count, positive), + ("negative", trace.negative_count, negative), + ]: + for index in range(count): + examples.add_example(example(f"{trace.prefix}-{kind}-{index}")) + return [Hypothesis(invariant, positive, negative)] + + +class ExampleReservoirConstructionTests(unittest.TestCase): + def test_from_iterable_streams_through_capped_add_and_counts_every_item(self): + applied = [] + original = ExampleList.add_example + + def observed_add(result, item): + original(result, item) + applied.append(result.num_seen) + self.assertLessEqual(len(result), 3) + + def items(): + for index in range(37): + self.assertEqual( + len(applied), index, "input must be consumed incrementally" + ) + yield example(index) + + with ( + patch.object(config, "MAX_EXAMPLE_LIST_SIZE", 3), + patch.object(ExampleList, "add_example", observed_add), + ): + result = ExampleList.from_iterable_of_examples(items(), {GROUP}) + self.assertEqual(len(result), 3) + self.assertEqual(result.num_seen, 37) + self.assertEqual(applied, list(range(1, 38))) + + def test_unlimited_zero_and_empty_iterables(self): + for cap, expected in [(None, list(range(12))), (0, []), (20, list(range(12)))]: + with ( + self.subTest(cap=cap), + patch.object(config, "MAX_EXAMPLE_LIST_SIZE", cap), + ): + result = ExampleList.from_iterable_of_examples( + (example(i) for i in range(12)), {GROUP} + ) + self.assertEqual(values(result), expected) + self.assertEqual(result.num_seen, 12) + empty = ExampleList.from_iterable_of_examples(iter(()), {GROUP}) + self.assertEqual(len(empty), 0) + self.assertEqual(empty.num_seen, 0) + + def test_invalid_caps_fail_at_construction(self): + for cap in [-1, True, False, 1.5, "3"]: + with ( + self.subTest(cap=cap), + patch.object(config, "MAX_EXAMPLE_LIST_SIZE", cap), + ): + with self.assertRaises(ValueError): + ExampleList({GROUP}) + + def test_from_iterable_validates_group_membership(self): + with self.assertRaises((AssertionError, ValueError)): + ExampleList.from_iterable_of_examples([example(1, "different")], {GROUP}) + + +class ExampleReservoirMergeTests(unittest.TestCase): + def assert_rejected_without_mutation(self, left, right): + prior_left, prior_right = snapshot(left), snapshot(right) + rng = random.Random(761) + rng_before = rng.getstate() + with self.assertRaises(ValueError): + left.merge(right, rng=rng) + self.assertEqual(snapshot(left), prior_left) + self.assertEqual(snapshot(right), prior_right) + self.assertEqual(rng.getstate(), rng_before) + + def test_exact_unlimited_and_under_cap_order_without_rng_consumption(self): + for cap in [None, 10]: + with self.subTest(cap=cap): + left = reservoir(["a", "a", "b"], cap) + right = reservoir(["c", "a"], cap) + before = snapshot(right) + rng = random.Random(54) + rng_before = rng.getstate() + self.assertIsNone(left.merge(right, rng=rng)) + self.assertEqual(values(left), ["a", "a", "b", "c", "a"]) + self.assertEqual(left.num_seen, 5) + self.assertEqual(snapshot(right), before) + self.assertEqual(rng.getstate(), rng_before) + + def test_empty_destination_adopts_groups_and_empty_source_preserves_order(self): + left = reservoir([], 3, groups=set()) + right = reservoir([1, 2], 3) + left.merge(right, rng=random.Random(1)) + self.assertEqual(left.group_names, {GROUP}) + self.assertEqual(values(left), [1, 2]) + self.assertEqual(left.num_seen, 2) + + uninitialized = reservoir([], 3, groups=set()) + rng = random.Random(3) + before = rng.getstate() + uninitialized.merge(reservoir([], 3), rng=rng) + self.assertEqual(uninitialized.group_names, {GROUP}) + self.assertEqual(uninitialized.num_seen, 0) + self.assertEqual(rng.getstate(), before) + left.merge(reservoir([], 3), rng=random.Random(2)) + self.assertEqual(values(left), [1, 2]) + self.assertEqual(left.num_seen, 2) + + def test_zero_capacity_accumulates_counts_and_preserves_no_items(self): + left, right = reservoir(range(20), 0), reservoir(range(7), 0) + left.merge(right, rng=random.Random(7)) + left.add_example(example("next")) + self.assertEqual(left.num_seen, 28) + self.assertEqual(len(left), 0) + + def test_unequal_populations_are_counted_and_source_remains_unchanged(self): + left, right = reservoir(range(10), 4, 11), reservoir(range(10, 100), 4, 19) + prior_right = snapshot(right) + left.merge(right, rng=random.Random(37)) + self.assertEqual(left.num_seen, 100) + self.assertEqual(len(left), 4) + self.assertTrue(set(values(left)).issubset(range(100))) + self.assertEqual(snapshot(right), prior_right) + for value in range(100, 117): + left.add_example(example(value)) + self.assertEqual(left.num_seen, 117) + self.assertEqual(len(left), 4) + + def test_duplicate_examples_are_a_multiset_not_a_set(self): + left, right = reservoir(["same"] * 7, 3), reservoir(["same"] * 11, 3) + left.merge(right, rng=random.Random(42)) + self.assertEqual(left.num_seen, 18) + self.assertEqual(values(left), ["same", "same", "same"]) + + def test_different_caps_are_allowed_when_source_has_enough_information(self): + left, right = reservoir(range(10), 5), reservoir([10, 11], 2) + left.merge(right, rng=random.Random(8)) + self.assertEqual((len(left), left.num_seen), (5, 12)) + left, right = reservoir(range(7), 2), reservoir(range(7, 17), None) + left.merge(right, rng=random.Random(9)) + self.assertEqual((len(left), left.num_seen), (2, 17)) + + def test_rejects_self_group_mismatch_and_incompatible_sampled_sources(self): + left = reservoir(range(3), 3) + self.assert_rejected_without_mutation(left, left) + right = reservoir([], 3, groups={"other"}) + self.assert_rejected_without_mutation(left, right) + self.assert_rejected_without_mutation( + reservoir(range(7), 5), reservoir(range(9), 2) + ) + self.assert_rejected_without_mutation( + reservoir(range(3), None), reservoir(range(9), 2) + ) + + def test_rejects_underfilled_and_malformed_reservoir_states(self): + for corrupt_left in [False, True]: + left, right = reservoir(range(9), 3), reservoir(range(12), 3) + (left if corrupt_left else right).examples.pop() + self.assert_rejected_without_mutation(left, right) + for malformed in [-1, True, 2.5]: + left, right = reservoir(range(3), 3), reservoir(range(3), 3) + right.num_seen = malformed + self.assert_rejected_without_mutation(left, right) + + def test_weighted_merge_and_continued_add_have_expected_marginals(self): + # Fixed seed enumeration is deterministic, not a fresh statistical draw. + # Concatenating retained samples gives 'left' ~75%, not the required30%. + before_add = after_add = 0 + trials = 2000 + for seed in range(trials): + left = reservoir(["left"], 3, 2 * seed + 1) + right = reservoir(range(9), 3, 2 * seed + 2) + left.merge(right, rng=random.Random(100000 + seed)) + before_add += "left" in values(left) + for index in range(10): + left.add_example(example(f"new-{index}")) + after_add += "left" in values(left) + self.assertAlmostEqual(before_add / trials, 3 / 10, delta=0.04) + self.assertAlmostEqual(after_add / trials, 3 / 20, delta=0.04) + + def test_nested_merge_tree_preserves_marginals_with_shared_or_distinct_source_seeds( + self, + ): + # An independent sequential merge RNG must be threaded through the entire + # uneven tree. Replayed per-list merge streams can over-select 'e' (~27%). + trials = 4000 + for same_source_seed in [True, False]: + merged, continued = Counter(), Counter() + for seed in range(trials): + source_seeds = ( + [seed] * 5 + if same_source_seed + else [31 * seed + i for i in range(5)] + ) + a, b, c, d, e = [ + reservoir([name], 1, value) + for name, value in zip("abcde", source_seeds) + ] + rng = random.Random(200000 + seed) + a.merge(b, rng=rng) + a.merge(c, rng=rng) + d.merge(e, rng=rng) + a.merge(d, rng=rng) + self.assertEqual(a.num_seen, 5) + merged.update(values(a)) + a.add_example(example("f")) + self.assertEqual(a.num_seen, 6) + continued.update(values(a)) + with self.subTest(same_source_seed=same_source_seed): + for name in "abcde": + self.assertAlmostEqual(merged[name] / trials, 1 / 5, delta=0.025) + for name in "abcdef": + self.assertAlmostEqual(continued[name] / trials, 1 / 6, delta=0.025) + + def test_sampled_unequal_sources_then_add_preserve_every_occurrence_marginal(self): + # Both sources have already sampled, using the same construction seed. + # Continuing the destination's old RNG replays draws used by the longer + # source and biases some occurrence marginals to ~14% or ~20%, versus + # 2/12. Checking just aggregate source weights would hide this effect. + merged, continued = Counter(), Counter() + trials = 6000 + for seed in range(trials): + left = reservoir(range(3), 2, seed) + right = reservoir(range(3, 10), 2, seed) + left.merge(right, rng=random.Random(300000 + seed)) + self.assertEqual((left.num_seen, len(left)), (10, 2)) + merged.update(values(left)) + for index in (10, 11): + left.add_example(example(index)) + self.assertEqual((left.num_seen, len(left)), (12, 2)) + continued.update(values(left)) + for occurrence in range(10): + self.assertAlmostEqual(merged[occurrence] / trials, 2 / 10, delta=0.02) + for occurrence in range(12): + self.assertAlmostEqual(continued[occurrence] / trials, 2 / 12, delta=0.02) + + +class ReservoirIntegrationTests(unittest.TestCase): + def test_actual_generate_hypothesis_matches_workers_1_2_4_fork_and_spawn(self): + traces = [ + ReservoirFixtureTrace("a", 17, 11), + ReservoirFixtureTrace("b", 31, 19), + ReservoirFixtureTrace("c", 53, 29), + ] + + def run(workers): + generated = InferEngine(traces, num_workers=workers).generate_hypothesis() + self.assertEqual(len(generated), 1) + item, trace_indexes = next(iter(generated.items())) + self.assertEqual(trace_indexes, [0, 1, 2]) + result = [] + for examples, count in [ + (item.positive_examples, 101), + (item.negative_examples, 59), + ]: + self.assertEqual((examples.max_size, len(examples)), (5, 5)) + self.assertEqual(examples.num_seen, count) + # Include the continuation state: pickling must preserve it and + # the parent must consume the same merge stream in both paths. + result.append((values(examples), count, examples._rng.getstate())) + return result + + with ( + patch("traincheck.infer_engine.relation_pool", [ReservoirFixtureRelation]), + patch.object(config, "MAX_EXAMPLE_LIST_SIZE", 5), + patch.object(config, "EXAMPLE_SAMPLING_SEED", 907), + ): + expected = run(1) + for method in [ + value + for value in ("spawn", "fork") + if value in multiprocessing.get_all_start_methods() + ]: + executor = partial( + ProcessPoolExecutor, mp_context=multiprocessing.get_context(method) + ) + for workers in (1, 2, 4): + with ( + self.subTest(method=method, workers=workers), + patch("traincheck.infer_engine.ProcessPoolExecutor", executor), + ): + self.assertEqual(run(workers), expected) + + def test_actual_engine_cross_trace_merge_keeps_caps_and_total_counts(self): + first, second = hypothesis("a", 17, 8, 3), hypothesis("b", 29, 11, 3) + engine, trace_indexes, lookup = InferEngine([]), {}, {} + rng = random.Random(712) + engine._merge_hypotheses([first], 0, trace_indexes, lookup, rng) + engine._merge_hypotheses([second], 1, trace_indexes, lookup, rng) + self.assertEqual(len(trace_indexes), 1) + self.assertEqual(next(iter(trace_indexes.values())), [0, 1]) + self.assertEqual( + (len(first.positive_examples), first.positive_examples.num_seen), (3, 46) + ) + self.assertEqual( + (len(first.negative_examples), first.negative_examples.num_seen), (3, 19) + ) + + def test_actual_stage_merge_keeps_caps_and_total_counts(self): + first, second = hypothesis("a", 17, 8, 3), hypothesis("b", 29, 11, 3) + + class Stages: + def is_stage_annotated(self): + return True + + def get_traces_for_stage(self): + return {"training": "training", "evaluation": "evaluation"} + + def get_all_stages(self): + return {"training", "evaluation"} + + by_stage = {"training": first, "evaluation": second} + + def inferred(stage): + item = by_stage[stage] + return [item.invariant], [], [item] + + with patch.object(APIContainRelation, "_infer", side_effect=inferred): + invariants, failures, successful = APIContainRelation.infer(Stages(), True) + self.assertEqual((len(invariants), len(failures), len(successful)), (1, 0, 1)) + self.assertEqual( + ( + len(successful[0].positive_examples), + successful[0].positive_examples.num_seen, + ), + (3, 46), + ) + self.assertEqual( + ( + len(successful[0].negative_examples), + successful[0].negative_examples.num_seen, + ), + (3, 19), + ) + + def test_sampled_generalization_does_not_treat_retained_union_as_population(self): + from traincheck.invariant.contain_relation import _merge_hypotheses + + hypotheses = [hypothesis("a", 20, 9, 3), hypothesis("b", 40, 15, 3)] + with patch( + "traincheck.invariant.contain_relation.calc_likelihood", + side_effect=AssertionError("sampled union must not be generalized"), + ): + result = _merge_hypotheses(hypotheses) + self.assertEqual( + [id(item) for item in result], [id(item) for item in hypotheses] + ) + + def test_complete_generalization_deduplicates_known_union_then_applies_cap(self): + from traincheck.invariant.contain_relation import _merge_hypotheses + + first, second = hypothesis("a", 0, 0, 10), hypothesis("b", 0, 0, 10) + first.positive_examples = reservoir(range(10), 10) + second.positive_examples = reservoir(range(10, 20), 10) + first.negative_examples = reservoir(range(10, 20), 10) + second.negative_examples = reservoir(range(10), 10) + expected = values(reservoir(range(20), 10)) + # Repeating one complete input must not double-count the known union. + for inputs in ([first, second], [first, second, first]): + with ( + self.subTest(inputs=len(inputs)), + patch.object(config, "MAX_EXAMPLE_LIST_SIZE", 10), + patch.object(config, "EXAMPLE_SAMPLING_SEED", 42), + ): + merged = _merge_hypotheses(inputs) + self.assertEqual(len(merged), 1) + positive, negative = ( + merged[0].positive_examples, + merged[0].negative_examples, + ) + self.assertEqual((len(positive), positive.num_seen), (10, 20)) + self.assertEqual((len(negative), negative.num_seen), (0, 0)) + self.assertEqual(values(positive), expected) + self.assertEqual(negative.group_names, {GROUP}) + self.assertEqual(first.positive_examples.num_seen, 10) + self.assertEqual(second.positive_examples.num_seen, 10) + + def test_worker_snapshot_carries_reservoir_settings_under_fork_and_spawn(self): + with ( + patch.object(config, "MAX_EXAMPLE_LIST_SIZE", 7), + patch.object(config, "EXAMPLE_SAMPLING_SEED", 907), + ): + settings = config_snapshot() + expected = worker_reservoir_result(29) + self.assertEqual(settings["MAX_EXAMPLE_LIST_SIZE"], 7) + self.assertEqual(settings["EXAMPLE_SAMPLING_SEED"], 907) + for method in [ + value + for value in ("spawn", "fork") + if value in multiprocessing.get_all_start_methods() + ]: + with ( + self.subTest(method=method), + ProcessPoolExecutor( + max_workers=2, + mp_context=multiprocessing.get_context(method), + initializer=worker_init, + initargs=([], settings), + ) as executor, + ): + futures = [ + executor.submit(worker_reservoir_result, 29) for _ in range(2) + ] + self.assertEqual( + [future.result(timeout=60) for future in futures], + [expected, expected], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_fresh_validation_contracts.py b/tests/test_fresh_validation_contracts.py new file mode 100644 index 00000000..1379c61d --- /dev/null +++ b/tests/test_fresh_validation_contracts.py @@ -0,0 +1,277 @@ +"""Negative-path contracts for emitted reports and observed call identities.""" + +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from tests.test_fresh_validation_coverage import frame, output_rule +from tools.fresh_validation.check_coverage import ( + ObserverContractError, + decision_record, + finalize_report, + observe, + summarize, +) +from tools.fresh_validation.compare_study import compare, losses +from tools.fresh_validation.contracts import boolean, integer, json_object, required +from tools.fresh_validation.fault_control import digest, mutate +from traincheck.invariant.base_cls import CheckerResult, Invariant +from traincheck.invariant.consistency_transient_vars import ConsistentOutputRelation +from traincheck.trace.trace import Trace + + +def renamed_local_checker( + trace: Trace, inv: Invariant, check_relation_first: bool +) -> CheckerResult: + # This still has the exact output-predicate AST but its local identity was + # renamed. The previous observer silently emitted {func_call_id: None}. + for renamed_call in trace.get_func_call_ids("torch.fake"): + func_call_event = trace.query_func_call_event(renamed_call) + returned_tensors = [func_call_event] + if len(returned_tensors) == 0: + raise AssertionError("Fixture unexpectedly empty") + return CheckerResult(None, inv, True, True) + + +def malformed_local_checker( + trace: Trace, inv: Invariant, check_relation_first: bool +) -> CheckerResult: + for real_call in trace.get_func_call_ids("torch.fake"): + func_call_event = trace.query_func_call_event(real_call) + func_call_id = None + returned_tensors = [func_call_event, func_call_id] + if len(returned_tensors) == 0: + raise AssertionError("Fixture unexpectedly empty") + return CheckerResult(None, inv, True, True) + + +def catching_local_checker( + trace: Trace, inv: Invariant, check_relation_first: bool +) -> CheckerResult: + for renamed_call in trace.get_func_call_ids("torch.fake"): + try: + func_call_event = trace.query_func_call_event(renamed_call) + returned_tensors = [func_call_event] + if len(returned_tensors) == 0: + raise AssertionError("Fixture unexpectedly empty") + except ValueError: + # The observer must retain failure even when the checker catches it. + continue + return CheckerResult(None, inv, True, True) + + +def catching_then_failing_checker( + trace: Trace, inv: Invariant, check_relation_first: bool +) -> CheckerResult: + for renamed_call in trace.get_func_call_ids("torch.fake"): + try: + func_call_event = trace.query_func_call_event(renamed_call) + returned_tensors = [func_call_event] + if len(returned_tensors) == 0: + raise AssertionError("Fixture unexpectedly empty") + except ValueError: + pass + raise RuntimeError("checker failed after catching observer error") + + +class UnsupportedOutput(ConsistentOutputRelation): + pass + + +class ValidationContractTests(unittest.TestCase): + def test_required_family_identity_does_not_use_another_family_fallback( + self, + ) -> None: + with self.assertRaisesRegex(ObserverContractError, "func_call_id"): + decision_record( + "ConsistentOutputRelation", {"parent_func_call_id": "parent"} + ) + with self.assertRaisesRegex(ObserverContractError, "parent_func_call_id"): + decision_record("APIContainRelation", {"func_call_id": "child"}) + + def test_pair_requires_both_record_identities(self) -> None: + with self.assertRaisesRegex(ObserverContractError, "func_call_id"): + decision_record( + "DistinctArgumentRelation", + {"event1": {"func_call_id": "a"}, "event2": {}}, + ) + + def test_identity_wrong_types_fail_explicitly(self) -> None: + for identity in (None, False, 42, ["call"]): + with ( + self.subTest(identity=identity), + self.assertRaises(ObserverContractError), + ): + decision_record("ConsistentOutputRelation", {"func_call_id": identity}) + + def test_checker_source_drift_is_observer_error_not_successful_null_identity( + self, + ) -> None: + for checker in ( + renamed_local_checker, + malformed_local_checker, + catching_local_checker, + ): + with ( + self.subTest(checker=checker.__name__), + patch.object(ConsistentOutputRelation, "static_check_all", checker), + ): + report = observe( + frame([{"torch.FloatTensor": {"dtype": "torch.float32"}}]), + output_rule(), + ) + self.assertEqual(report["status"], "observer_error") + self.assertEqual(report["coverage_status"], "error") + self.assertIsNone(report["check_passed"]) + self.assertIsNone(report["active"]) + self.assertFalse(report["coverage_observed"]) + self.assertEqual(report["decision_records"], []) + + def test_observer_failure_remains_primary_if_checker_then_raises(self) -> None: + with patch.object( + ConsistentOutputRelation, "static_check_all", catching_then_failing_checker + ): + report = observe( + frame([{"torch.FloatTensor": {"dtype": "torch.float32"}}]), + output_rule(), + ) + self.assertEqual(report["status"], "observer_error") + self.assertEqual(report["coverage_status"], "error") + self.assertFalse(report["coverage_observed"]) + self.assertIsNone(report["active"]) + self.assertEqual(report["error"]["type"], "ObserverContractError") + self.assertEqual(report["checker_error_after_observer"]["type"], "RuntimeError") + + def test_disabled_and_unsupported_counts_are_unknown_not_zero(self) -> None: + trace = frame([{"torch.FloatTensor": {"dtype": "torch.float32"}}]) + direct = observe(trace, output_rule(), enabled=False) + rule = output_rule() + rule.relation = UnsupportedOutput + unsupported = observe(trace, rule) + for report in (direct, unsupported): + summary = summarize([report]) + self.assertIsNone(summary["active_invariants"]) + self.assertIsNone(summary["dormant_invariants"]) + self.assertEqual(summary["coverage_unobserved_invariants"], 1) + self.assertEqual(summary["observed_active_invariants"], 0) + self.assertEqual(direct["coverage_status"], "disabled") + self.assertEqual(unsupported["coverage_status"], "unsupported") + self.assertIsNone(summarize([unsupported])["violated_invariants"]) + self.assertEqual(summarize([direct])["violated_invariants"], 0) + + def test_completed_report_requires_checker_payload(self) -> None: + report = observe( + frame([{"torch.FloatTensor": {"dtype": "torch.float32"}}]), output_rule() + ) + del report["checker_result"] + with self.assertRaisesRegex(ObserverContractError, "checker_result"): + finalize_report(report) + + def test_observed_report_requires_decision_counters(self) -> None: + report = observe( + frame([{"torch.FloatTensor": {"dtype": "torch.float32"}}]), output_rule() + ) + del report["decisions_completed"] + with self.assertRaisesRegex(ObserverContractError, "decisions_completed"): + finalize_report(report) + + def test_checker_error_does_not_imply_zero_violations(self) -> None: + rule = output_rule() + with patch.object( + rule, "check", side_effect=ValueError("fixture checker error") + ): + report = observe( + frame([{"torch.FloatTensor": {"dtype": "torch.float32"}}]), + rule, + enabled=False, + ) + self.assertEqual(report["status"], "checker_error") + self.assertIsNone(summarize([report])["violated_invariants"]) + self.assertEqual(summarize([report])["observed_violated_invariants"], 0) + + def test_cli_bootstrap_preserves_composed_traincheck_source(self) -> None: + adapter = ( + Path(__file__).resolve().parents[1] + / "tools/fresh_validation/benchmark_workload.py" + ) + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + composed = root / "composed" + (composed / "traincheck").mkdir(parents=True) + module = composed / "traincheck/__init__.py" + module.write_text("SOURCE = 'fixture'\n") + code = ( + "import runpy; runpy.run_path(" + + repr(str(adapter)) + + "); import traincheck; print(traincheck.__file__)" + ) + result = subprocess.run( + [sys.executable, "-c", code], + cwd=root, + env={ + **os.environ, + "PYTHONPATH": str(composed), + "PYTHONDONTWRITEBYTECODE": "1", + }, + check=True, + capture_output=True, + text=True, + ) + self.assertEqual(Path(result.stdout.strip()), module) + + def test_json_required_fields_and_wrong_types_are_not_coerced(self) -> None: + with self.assertRaisesRegex(ValueError, "required"): + required({}, "required") + with self.assertRaises(ValueError): + boolean("false", "active") + with self.assertRaises(ValueError): + integer(True, "count") + with self.assertRaises(ValueError): + json_object({"invalid": object()}) + + def test_empty_loss_records_do_not_establish_equality(self) -> None: + for steps in ([], [{"step": 1}]): + with self.subTest(steps=steps), self.assertRaisesRegex(ValueError, "loss"): + losses({"steps": steps}) + + def test_truncated_loss_rows_do_not_match_claimed_completed_steps(self) -> None: + with self.assertRaisesRegex(ValueError, "completed_steps"): + losses({"steps": [{"step": 1, "loss": 1.0}], "completed_steps": 2}) + + def test_incomplete_phase_set_cannot_report_study_completed(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + with self.assertRaisesRegex(ValueError, "Missing completed phases"): + compare(Path(temporary)) + + def test_fault_index_cannot_silently_select_last_rule(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + trace, rules, coverage = ( + root / name for name in ("trace.jsonl", "rules.jsonl", "coverage.json") + ) + trace.write_text( + json.dumps({"type": "function_call (post)", "func_call_id": "a"}) + "\n" + ) + rules.write_text(json.dumps(output_rule().to_dict()) + "\n") + coverage.write_text( + json.dumps( + { + "inputs_unchanged": True, + "observer_enabled": True, + "input_sha256": {str(p): digest(p) for p in (trace, rules)}, + "per_invariant": [{"invariant_index": -1}], + } + ) + ) + with self.assertRaisesRegex(ValueError, "out of range"): + mutate([trace], rules, coverage, root / "fault") + self.assertFalse((root / "fault").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_fresh_validation_coverage.py b/tests/test_fresh_validation_coverage.py new file mode 100644 index 00000000..a0519e42 --- /dev/null +++ b/tests/test_fresh_validation_coverage.py @@ -0,0 +1,278 @@ +"""Small real-checker counter oracles, including dormant and early-exit cases.""" + +import copy +import logging +import unittest + +import pandas as pd + +from tools.fresh_validation.check_coverage import observe +from traincheck.instrumentor.tracer import TraceLineType +from traincheck.invariant.base_cls import ( + PT, + APIParam, + GroupedPreconditions, + InputOutputParam, + Invariant, + Precondition, + PreconditionClause, + Preconditions, + UnconditionalPrecondition, + VarTypeParam, +) +from traincheck.invariant.consistency_transient_vars import ( + ConsistentInputOutputRelation, + ConsistentOutputRelation, + ThresholdRelation, +) +from traincheck.invariant.contain_relation import ( + APIContainRelation, + cache_events_scanner, +) +from traincheck.invariant.DistinctArgumentRelation import ( + EXP_GROUP_NAME, + DistinctArgumentRelation, +) +from traincheck.trace.trace_pandas import TracePandas + + +def frame(outputs, *, steps=None, inputs=None): + rows = [] + for i, output in enumerate(outputs): + tensor = {"torch.FloatTensor": {"shape": [2], "dtype": "torch.float32"}} + common = { + "func_call_id": f"call-{i}", + "function": "torch.fake", + "process_id": 1, + "thread_id": 2, + "is_bound_method": False, + "obj_id": None, + "meta_vars.step": steps[i] if steps else i, + "args": {"0": copy.deepcopy(tensor if inputs is None else inputs[i])}, + "kwargs": {}, + "return_values": copy.deepcopy(output), + } + rows += [ + {**common, "type": TraceLineType.FUNC_CALL_PRE, "time": i * 3}, + {**common, "type": TraceLineType.FUNC_CALL_POST, "time": i * 3 + 1}, + ] + return TracePandas(pd.DataFrame(rows), truncate_incomplete_func_calls=False) + + +def precondition(group="pre_event", enabled=True): + return GroupedPreconditions( + {group: Preconditions([UnconditionalPrecondition()] if enabled else [])} + ) + + +def output_rule(enabled=True): + return Invariant( + ConsistentOutputRelation, + [ + APIParam("torch.fake"), + VarTypeParam("torch.Tensor", "dtype", const_value="torch.float32"), + ], + precondition(enabled=enabled), + "fixture output dtype", + ) + + +class CoverageTests(unittest.TestCase): + def setUp(self): + logging.disable(logging.CRITICAL) + self.good = {"torch.FloatTensor": {"shape": [2], "dtype": "torch.float32"}} + + def assert_same_result(self, make_trace, inv): + cache_events_scanner.clear() + observed = observe(make_trace(), inv) + cache_events_scanner.clear() + plain = observe(make_trace(), inv, enabled=False) + self.assertEqual(observed.get("checker_result"), plain.get("checker_result")) + self.assertIsNone(plain["active"]) + self.assertIsNone(plain["dormant"]) + self.assertEqual(observed["status"], "completed", observed) + return observed + + def test_healthy_real_decisions(self): + got = self.assert_same_result(lambda: frame([self.good] * 3), output_rule()) + self.assertEqual( + ( + got["opportunities"], + got["precondition_group_passes"], + got["decisions_completed"], + ), + (3, 3, 3), + ) + self.assertTrue(got["active"]) + + def test_all_preconditions_false_are_dormant(self): + got = self.assert_same_result( + lambda: frame([self.good] * 3), output_rule(False) + ) + self.assertEqual( + ( + got["opportunities"], + got["precondition_group_failures"], + got["decisions_completed"], + ), + (3, 3, 0), + ) + self.assertTrue(got["dormant"]) + + def test_missing_io_value_passes_precondition_but_runs_no_predicate(self): + inv = Invariant( + ConsistentInputOutputRelation, + [ + InputOutputParam(None, 0, "torch.Tensor", ("shape",), None, True), + APIParam("torch.fake"), + InputOutputParam(None, 0, "torch.Tensor", ("missing",), None, False), + ], + precondition(), + "missing value", + ) + got = self.assert_same_result(lambda: frame([self.good]), inv) + self.assertEqual(got["precondition_group_passes"], 1) + self.assertEqual(got["decisions_completed"], 0) + self.assertTrue(got["dormant"]) + + def test_distinct_single_call_has_zero_pairs_despite_triggered(self): + inv = Invariant( + DistinctArgumentRelation, + [APIParam("torch.fake")], + precondition(EXP_GROUP_NAME), + "distinct", + ) + got = self.assert_same_result(lambda: frame([self.good]), inv) + self.assertEqual( + ( + got["opportunities"], + got["precondition_group_passes"], + got["decisions_completed"], + ), + (0, 1, 0), + ) + self.assertTrue(got["checker_triggered"]) + self.assertTrue(got["dormant"]) + + def test_first_failure_leaves_unvisited_remainder(self): + wrong = {"torch.FloatTensor": {"shape": [2], "dtype": "torch.float64"}} + got = self.assert_same_result( + lambda: frame([wrong, self.good, self.good]), output_rule() + ) + self.assertEqual( + ( + got["opportunities"], + got["decisions_completed"], + got["unvisited_calls_after_stop"], + ), + (3, 1, 2), + ) + self.assertFalse(got["check_passed"]) + + def test_missing_tensor_is_completed_output_violation(self): + got = self.assert_same_result(lambda: frame([{"int": 1}]), output_rule()) + self.assertEqual(got["decisions_completed"], 1) + self.assertFalse(got["check_passed"]) + + def test_distinct_pair_count_and_first_violation(self): + inv = Invariant( + DistinctArgumentRelation, + [APIParam("torch.fake")], + precondition(EXP_GROUP_NAME), + "distinct", + ) + got = self.assert_same_result( + lambda: frame([self.good] * 3, steps=[1, 1, 1]), inv + ) + self.assertEqual((got["opportunities"], got["decisions_completed"]), (3, 1)) + self.assertFalse(got["check_passed"]) + self.assertIsNone(got["unvisited_calls_after_stop"]) + self.assertEqual(got["unvisited_pairs_after_stop"], 2) + + def test_io_actual_comparison_and_failure(self): + inv = Invariant( + ConsistentInputOutputRelation, + [ + InputOutputParam(None, 0, "torch.Tensor", ("shape",), None, True), + APIParam("torch.fake"), + InputOutputParam(None, 0, "torch.Tensor", ("shape",), None, False), + ], + precondition(), + "shape agreement", + ) + for shape, passed in (([2], True), ([9], False)): + got = self.assert_same_result( + lambda: frame([{"torch.FloatTensor": {"shape": shape}}]), inv + ) + self.assertEqual( + (got["decisions_completed"], got["check_passed"]), (1, passed) + ) + + def test_containment_counts_parent_decisions_even_when_empty(self): + cache_events_scanner.clear() + inv = Invariant( + APIContainRelation, + [APIParam("torch.fake"), APIParam("torch.child")], + precondition("parent_func_call_pre"), + "requires child", + ) + got = self.assert_same_result(lambda: frame([self.good] * 3), inv) + self.assertEqual((got["opportunities"], got["decisions_completed"]), (3, 1)) + self.assertFalse(got["check_passed"]) + + def test_requested_variable_precondition_skipped_without_proxy_data(self): + conditions = precondition("parent_func_call_pre") + conditions.grouped_preconditions["var_events"] = Preconditions( + [ + Precondition( + [ + PreconditionClause( + "meta_vars.phase", str, PT.CONSTANT, None, {"train"} + ) + ] + ) + ] + ) + inv = Invariant( + APIContainRelation, + [APIParam("torch.fake"), VarTypeParam("torch.Tensor", "shape")], + conditions, + "requires variable checking", + ) + + def bound_trace(): + trace = frame([self.good]) + trace.events["is_bound_method"] = True + trace.events["obj_id"] = 123 + return trace + + got = self.assert_same_result(bound_trace, inv) + self.assertTrue(got["variable_precondition_requested"]) + self.assertTrue(got["variable_check_skipped"]) + self.assertFalse(got["fully_validated_active"]) + + def test_threshold_comparison_after_successful_extraction(self): + # torch.clamp has a real signature; max=3 is the checker threshold. + def make_trace(): + trace = frame([{"torch.FloatTensor": {"max": 2}}]) + trace.events["function"] = "torch.clamp" + trace.events["kwargs"] = [{"max": {"int": 3}}] * 2 + return trace + + inv = Invariant( + ThresholdRelation, + [ + InputOutputParam("max", None, "int", None, "torch.clamp", True), + APIParam("torch.clamp"), + InputOutputParam(None, 0, "torch.Tensor", ("max",), None, False), + ], + precondition(), + "max threshold", + ) + got = self.assert_same_result(make_trace, inv) + self.assertEqual(got["decisions_completed"], 1) + self.assertTrue(got["check_passed"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_fresh_validation_fault.py b/tests/test_fresh_validation_fault.py new file mode 100644 index 00000000..a308506d --- /dev/null +++ b/tests/test_fresh_validation_fault.py @@ -0,0 +1,81 @@ +"""The fallback must activate the real frozen checker, without other edits.""" + +import json +import tempfile +import unittest +from pathlib import Path + +import pandas as pd + +from tests.test_fresh_validation_coverage import frame, output_rule +from tools.fresh_validation.check_coverage import observe +from tools.fresh_validation.fault_control import differences, digest, mutate +from traincheck.trace.trace_pandas import TracePandas + + +class FaultControlTests(unittest.TestCase): + def run_mutation(self, attribute, shape): + rule = output_rule() + if attribute == "shape": + rule.params[1].attr_name = "shape" + rule.params[1].const_value = tuple(shape) + good = {"torch.FloatTensor": {"shape": shape, "dtype": "torch.float32"}} + traced = frame([good, good]) + healthy = observe(traced, rule) + healthy["invariant_index"] = 0 + before = traced.events.to_dict("records") + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + trace, invariants, coverage = ( + root / x for x in ("trace.log", "rules.jsonl", "coverage.json") + ) + trace.write_text("".join(json.dumps(row) + "\n" for row in before)) + invariants.write_text(json.dumps(rule.to_dict()) + "\n") + coverage.write_text( + json.dumps( + { + "inputs_unchanged": True, + "observer_enabled": True, + "input_sha256": { + str(p): digest(p) for p in (trace, invariants) + }, + "per_invariant": [healthy], + } + ) + ) + manifest = mutate([trace], invariants, coverage, root / "fault") + after = [ + json.loads(line) + for line in (root / "fault/traces/trace.log").read_text().splitlines() + ] + self.assertEqual(len(differences(before, after)), 1) + self.assertEqual(before[0], after[0]) + self.assertEqual(before[2:], after[2:]) + self.assertEqual(manifest["attribute"], attribute) + self.assertEqual( + manifest["edits"][0]["path"][-1], 0 if attribute == "shape" else "dtype" + ) + checked = observe( + TracePandas(pd.DataFrame(after), truncate_incomplete_func_calls=False), + rule, + ) + self.assertTrue(healthy["check_passed"]) + self.assertEqual(checked["status"], "completed") + self.assertFalse(checked["check_passed"]) + self.assertEqual(checked["precondition_group_passes"], 1) + self.assertEqual(checked["decisions_completed"], 1) + self.assertEqual( + checked["decision_records"][0]["func_call_id"], manifest["func_call_id"] + ) + self.assertTrue( + any( + row["func_call_id"] == manifest["func_call_id"] + for row in checked["checker_result"]["trace"] + ) + ) + + def test_no_shape_rule_uses_live_dtype_and_detects_exact_edited_call(self): + self.run_mutation("dtype", []) + + def test_mutable_shape_keeps_original_one_dimension_control(self): + self.run_mutation("shape", [2, 3]) diff --git a/tests/test_identity_cache.py b/tests/test_identity_cache.py new file mode 100644 index 00000000..4093d74a --- /dev/null +++ b/tests/test_identity_cache.py @@ -0,0 +1,194 @@ +"""Identity lookup equivalence during precondition inference.""" + +import copy +import json +import logging +import pickle +import unittest +from unittest.mock import patch + +import numpy as np +import pandas as pd + +from traincheck.invariant.precondition import find_precondition_from_single_group +from traincheck.trace.trace_pandas import TracePandas + + +class LegacyIdentityTrace(TracePandas): + def get_process_ids(self): + if "process_id" not in self.events.columns: + return [] + return self.events["process_id"].dropna().unique().tolist() + + def get_thread_ids(self): + if "thread_id" not in self.events.columns: + return [] + return self.events["thread_id"].dropna().unique().tolist() + + +def make(frame, cls=TracePandas): + return cls(frame.copy(deep=True), truncate_incomplete_func_calls=False) + + +def frame(process_ids=(1, 2, 1), thread_ids=(4, 3, 4)): + return pd.DataFrame( + { + "time": range(len(process_ids)), + "process_id": process_ids, + "thread_id": thread_ids, + } + ) + + +class IdentityCacheTests(unittest.TestCase): + def setUp(self): + previous = logging.root.manager.disable + logging.disable(logging.CRITICAL) + self.addCleanup(logging.disable, previous) + + def assert_equivalent(self, data): + old, new = make(data, LegacyIdentityTrace), make(data) + for _ in range(2): + for method in ("get_process_ids", "get_thread_ids"): + expected, actual = getattr(old, method)(), getattr(new, method)() + self.assertEqual(expected, actual) + self.assertEqual(list(map(type, expected)), list(map(type, actual))) + return old, new + + def test_values_types_and_encounter_order(self): + for values in ( + pd.Series([8, 2, 8, -1], dtype="int64"), + pd.Series([2**63 + 7, 2**63 + 2, None, 2**63 + 7], dtype="UInt64"), + pd.Series([8, pd.NA, 2, 8], dtype="Int64"), + pd.Series([8.5, np.nan, 2.5, 8.5]), + pd.Series(["z", None, "a", "z"], dtype="string"), + pd.Series(["z", None, "a", "z"], dtype="category"), + pd.Series([True, 1, False, 0], dtype=object), + pd.Series([None, np.nan, pd.NA, None], dtype=object), + ): + with self.subTest(dtype=str(values.dtype), values=repr(values.tolist())): + self.assert_equivalent(frame(values, values)) + + def test_randomized_identity_columns(self): + rng = np.random.default_rng(42) + for _ in range(80): + values = rng.integers(-8, 8, size=50).astype(float) + values[rng.random(50) < 0.2] = np.nan + self.assert_equivalent(frame(values, values[::-1])) + + def test_empty_and_missing_columns(self): + for data in (frame([], []), pd.DataFrame({"time": [0]})): + self.assert_equivalent(data) + + def test_returned_lists_are_independent(self): + trace = make(frame()) + result = trace.get_process_ids() + result[:] = [999] + self.assertEqual(trace.get_process_ids(), [1, 2]) + result = trace.get_thread_ids() + result.clear() + self.assertEqual(trace.get_thread_ids(), [4, 3]) + + def test_frame_replacement_rebuilds_same_size_cache(self): + trace = make(frame()) + trace.get_process_ids() + trace.get_thread_ids() + trace.events = frame((9, 7, 9), (5, 6, 5)) + self.assertEqual(trace.get_process_ids(), [9, 7]) + self.assertEqual(trace.get_thread_ids(), [5, 6]) + + def test_same_frame_length_change_rebuilds_cache(self): + trace = make(frame()) + trace.get_process_ids() + trace.get_thread_ids() + trace.events.loc[3] = [3, 9, 7] + self.assertEqual(trace.get_process_ids(), [1, 2, 9]) + self.assertEqual(trace.get_thread_ids(), [4, 3, 7]) + trace.events.drop(index=[1, 3], inplace=True) + self.assertEqual(trace.get_process_ids(), [1]) + self.assertEqual(trace.get_thread_ids(), [4]) + + def test_stage_traces_and_instances_have_separate_caches(self): + data = frame() + data["meta_vars.stage"] = ["init", "train", "init"] + trace = make(data) + trace.get_process_ids() + stages = trace.get_traces_for_stage() + self.assertEqual(stages["init"].get_process_ids(), [1]) + self.assertEqual(stages["train"].get_process_ids(), [2]) + self.assertEqual(make(frame((9,), (7,))).get_process_ids(), [9]) + + def test_pickled_initialized_cache(self): + trace = make(frame()) + trace.get_process_ids() + restored = pickle.loads(pickle.dumps(trace)) + self.assertIs(restored._identity_values_source, restored.events) + self.assertEqual(restored.get_process_ids(), [1, 2]) + self.assertEqual(restored.get_thread_ids(), [4, 3]) + + def test_meta_vars_membership_preserves_independent_id_checks(self): + old, new = self.assert_equivalent(frame((1, 2), (3, 4))) + for pid, tid in ((1, 3), (1, 4), (2, 3), (9, 3), (1, 9)): + self.assertEqual( + old.get_meta_vars(0, pid, tid), new.get_meta_vars(0, pid, tid) + ) + self.assertEqual(new.get_meta_vars(0, 1, 4), {}) + self.assertIsNone(new.get_meta_vars(0, 9, 3)) + + def test_repeated_meta_vars_queries_only_scan_each_column_once(self): + trace = make(frame()) + original = pd.Series.unique + scanned = [] + + def unique(series): + scanned.append(series.name) + return original(series) + + with patch.object(pd.Series, "unique", unique): + for _ in range(100): + self.assertEqual(trace.get_meta_vars(0, 1, 4), {}) + self.assertEqual(scanned, ["process_id", "thread_id"]) + + def test_real_precondition_inference_matches_legacy(self): + old, new = self.assert_equivalent(frame()) + positive = [ + [ + { + "time": i, + "process_id": 1, + "thread_id": 4, + "function": "fixtures.step", + "meta_vars.phase": "train", + } + ] + for i in range(12) + ] + negative = [ + [ + { + "time": i, + "process_id": 2, + "thread_id": 3, + "function": "fixtures.step", + "meta_vars.phase": "eval", + } + ] + for i in range(8) + ] + results = [] + for trace in (old, new): + pos, neg = copy.deepcopy(positive), copy.deepcopy(negative) + clauses = find_precondition_from_single_group(pos, neg, [trace]) + results.append( + ( + sorted(json.dumps(c.to_dict(), sort_keys=True) for c in clauses), + pos, + neg, + ) + ) + self.assertTrue(results[0][0]) + self.assertEqual(*results) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_inference_sampling.py b/tests/test_inference_sampling.py index bfd87977..b861ebf5 100644 --- a/tests/test_inference_sampling.py +++ b/tests/test_inference_sampling.py @@ -102,17 +102,22 @@ def examples(items): def simple_hypothesis(positive_count=3, negative_count=0): def examples(count, prefix): - # Direct construction deliberately reaches the >5000 precondition branch; - # add_example's separate reservoir would otherwise cap the input first. - return ExampleList.from_iterable_of_examples( - [ - Example( - {PARENT_GROUP_NAME: [{"id": f"{prefix}-{i}", "function": PARENT}]} - ) - for i in range(count) - ], - {PARENT_GROUP_NAME}, - ) + # This fixture deliberately reaches the >5000 precondition branch; + # disable the independent reservoir cap while constructing its input. + with patch("traincheck.config.config.MAX_EXAMPLE_LIST_SIZE", None): + return ExampleList.from_iterable_of_examples( + [ + Example( + { + PARENT_GROUP_NAME: [ + {"id": f"{prefix}-{i}", "function": PARENT} + ] + } + ) + for i in range(count) + ], + {PARENT_GROUP_NAME}, + ) return Hypothesis( Invariant( diff --git a/tests/test_parallel_config.py b/tests/test_parallel_config.py new file mode 100644 index 00000000..6e534669 --- /dev/null +++ b/tests/test_parallel_config.py @@ -0,0 +1,77 @@ +"""Worker settings are required and validated before any process state changes.""" + +import unittest +from contextlib import ExitStack +from unittest.mock import patch + +import traincheck.config.config as config +import traincheck.parallel as parallel +import traincheck.utils as utils +from traincheck.trace.trace import Trace + + +class WorkerConfigTests(unittest.TestCase): + def setUp(self) -> None: + stack = ExitStack() + self.addCleanup(stack.close) + stack.enter_context(patch.multiple(config, **parallel.config_snapshot())) + self.original_traces: list[Trace] = [] + stack.enter_context( + patch.object(parallel, "_WORKER_TRACES", self.original_traces) + ) + stack.enter_context(patch.object(utils, "_suppress_inner_progress", False)) + + def assert_rejected_unchanged( + self, snapshot: dict[str, object], error: type[Exception] + ) -> None: + before = parallel.config_snapshot() + with self.assertRaises(error): + parallel.worker_init([], snapshot) + self.assertEqual(parallel.config_snapshot(), before) + self.assertIs(parallel.get_worker_traces(), self.original_traces) + self.assertFalse(utils._suppress_inner_progress) + + def test_all_fields_are_required(self) -> None: + for field in parallel.config_snapshot(): + snapshot: dict[str, object] = dict(parallel.config_snapshot()) + del snapshot[field] + with self.subTest(field=field): + self.assert_rejected_unchanged(snapshot, KeyError) + + def test_unknown_fields_are_rejected(self) -> None: + snapshot: dict[str, object] = dict(parallel.config_snapshot()) + snapshot["UNRECOGNIZED_SETTING"] = 1 + self.assert_rejected_unchanged(snapshot, ValueError) + + def test_malformed_fields_leave_all_state_unchanged(self) -> None: + invalid: dict[str, list[object]] = { + "ENABLE_PRECOND_SAMPLING": [1, None, "true"], + "PRECOND_SAMPLING_THRESHOLD": [True, 1.5, "100"], + "MAX_EXAMPLE_LIST_SIZE": [True, -1, 2.5, "10"], + "EXAMPLE_SAMPLING_SEED": [object(), {}, []], + } + for field, values in invalid.items(): + for value in values: + snapshot: dict[str, object] = dict(parallel.config_snapshot()) + snapshot[field] = value + with self.subTest(field=field, value=value): + self.assert_rejected_unchanged(snapshot, ValueError) + + def test_supported_seeds_and_uncapped_or_zero_capacity(self) -> None: + for seed in (None, 9, -8, 1.5, "seed", b"seed", bytearray(b"seed")): + for capacity in (None, 0, 7): + snapshot: dict[str, object] = { + "ENABLE_PRECOND_SAMPLING": False, + "PRECOND_SAMPLING_THRESHOLD": 0, + "MAX_EXAMPLE_LIST_SIZE": capacity, + "EXAMPLE_SAMPLING_SEED": seed, + } + traces: list[Trace] = [] + parallel.worker_init(traces, snapshot) + self.assertEqual(parallel.config_snapshot(), snapshot) + self.assertIs(parallel.get_worker_traces(), traces) + self.assertTrue(utils._suppress_inner_progress) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_precondition_state_isolation.py b/tests/test_precondition_state_isolation.py new file mode 100644 index 00000000..186ec7ea --- /dev/null +++ b/tests/test_precondition_state_isolation.py @@ -0,0 +1,174 @@ +"""Keep pruning state local to one precondition solve, including its children.""" + +import copy +import json +import logging +import subprocess +import sys +import unittest +from pathlib import Path +from typing import Literal, TypeAlias +from unittest.mock import patch + +import pandas as pd + +import traincheck.invariant.precondition as precondition +from traincheck.invariant.base_cls import PT, Precondition, PreconditionClause +from traincheck.trace import Trace +from traincheck.trace.trace_pandas import TracePandas + +Record: TypeAlias = dict[str, object] +Example: TypeAlias = list[Record] +ObservedChild: TypeAlias = tuple[set[PreconditionClause], bool, int] + + +def example(phase: str, signal: bool | None = None) -> Example: + record: Record = { + "time": 0, + "process_id": 1, + "thread_id": 2, + "function": "fixture.step", + "meta_vars.phase": phase, + } + if signal is not None: + record["meta_vars.signal"] = signal + return [record] + + +def trace() -> TracePandas: + return TracePandas( + pd.DataFrame({"time": [0, 1], "process_id": [1, 1], "thread_id": [2, 2]}), + truncate_incomplete_func_calls=False, + ) + + +def solve( + name: Literal["A", "B"], *, _pruned_clauses: set[PreconditionClause] | None = None +) -> list[Precondition]: + fixtures: dict[str, tuple[list[Example], list[Example]]] = { + "A": ([example("train", True)], [example("train", False)]), + "B": ([example("train")], [example("eval")]), + } + positive, negative = copy.deepcopy(fixtures[name]) + return precondition.find_precondition_from_single_group( + positive, negative, [trace()], _pruned_clauses=_pruned_clauses + ) + + +def constant(prop: str, dtype: type, value: str | int | bool) -> PreconditionClause: + return PreconditionClause(prop, dtype, PT.CONSTANT, None, {value}) + + +class PreconditionStateIsolationTests(unittest.TestCase): + def setUp(self) -> None: + previous = logging.root.manager.disable + logging.disable(logging.CRITICAL) + self.addCleanup(logging.disable, previous) + + def assert_phase_condition(self, result: list[Precondition]) -> None: + self.assertEqual( + result, [Precondition([constant("meta_vars.phase", str, "train")])] + ) + + def test_prior_independent_solve_does_not_prune_required_clause(self) -> None: + self.assertEqual( + solve("A"), + [Precondition([constant("meta_vars.signal", bool, True)])], + ) + self.assert_phase_condition(solve("B")) + + def test_reversed_independent_order_preserves_later_result(self) -> None: + self.assert_phase_condition(solve("B")) + solve("A") + self.assert_phase_condition(solve("B")) + + def test_repeated_independent_solve(self) -> None: + for _ in range(3): + self.assert_phase_condition(solve("B")) + + def test_explicit_empty_set_is_updated_and_reused(self) -> None: + state: set[PreconditionClause] = set() + solve("A", _pruned_clauses=state) + self.assertIn(constant("meta_vars.phase", str, "train"), state) + # Explicit state still means to suppress these clauses in this solve. + self.assertEqual(solve("B", _pruned_clauses=state), []) + self.assert_phase_condition(solve("B")) + + def test_recursive_siblings_share_state_but_new_roots_do_not(self) -> None: + original = precondition.find_precondition_from_single_group + root_children: list[list[ObservedChild]] = [] + for _ in range(2): + children: list[ObservedChild] = [] + + def observe( + positive_examples: list[Example], + negative_examples: list[Example], + traces: list[Trace], + keys_to_skip: list[str], + *, + _pruned_clauses: set[PreconditionClause], + _skip_pruning: bool, + _current_depth: int, + ) -> list[Precondition]: + children.append((_pruned_clauses, _skip_pruning, _current_depth)) + return original( + positive_examples, + negative_examples, + traces, + keys_to_skip, + _pruned_clauses=_pruned_clauses, + _skip_pruning=_skip_pruning, + _current_depth=_current_depth, + ) + + positive = [example("train"), example("eval")] + negative = [example("test")] + with patch.object( + precondition, "find_precondition_from_single_group", side_effect=observe + ): + result = original(positive, negative, [trace()]) + self.assertEqual(len(result), 2) + self.assertTrue( + all( + any(condition.verify(item) for condition in result) + for item in positive + ) + ) + self.assertTrue( + all(not condition.verify(negative[0]) for condition in result) + ) + self.assertEqual(len(children), 2) + self.assertIs(children[0][0], children[1][0]) + self.assertTrue(all(skip and depth == 1 for _, skip, depth in children)) + self.assertIn(constant("process_id", int, 1), children[0][0]) + root_children.append(children) + self.assertIsNot(root_children[0][0][0], root_children[1][0][0]) + + def test_fresh_and_warm_processes_find_same_clause(self) -> None: + outputs: list[str] = [] + for order in (("B",), ("A", "B"), ("B", "A", "B")): + script = ( + "import json\n" + "from tests.test_precondition_state_isolation import solve\n" + f"results = [solve(name) for name in {order!r}]\n" + "print(json.dumps([item.to_dict() for item in results[-1]], sort_keys=True))\n" + ) + child = subprocess.run( + [sys.executable, "-c", script], + cwd=Path(__file__).resolve().parents[1], + capture_output=True, + text=True, + check=True, + ) + outputs.append(child.stdout.strip()) + self.assertEqual(outputs[0], outputs[1]) + self.assertEqual(outputs[0], outputs[2]) + expected = [Precondition([constant("meta_vars.phase", str, "train")])] + self.assertEqual( + outputs[0], + json.dumps([item.to_dict() for item in expected], sort_keys=True), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_time_range_index.py b/tests/test_time_range_index.py new file mode 100644 index 00000000..7ea110e9 --- /dev/null +++ b/tests/test_time_range_index.py @@ -0,0 +1,325 @@ +"""Differential tests against the unchanged pandas filtering path.""" + +import copy +import logging +import math +import pickle +import unittest +from unittest.mock import patch + +import numpy as np +import pandas as pd + +from traincheck.instrumentor.tracer import TraceLineType as T +from traincheck.trace.trace_pandas import TracePandas + + +class LegacyRangeTrace(TracePandas): + def _get_func_call_time_positions(self, *args): + return None + + +def frame(times=(1, 2, 3, 4), ids=("z", "z", "a", "a")): + n = len(times) + return pd.DataFrame( + { + "time": times, + "type": [T.FUNC_CALL_PRE, T.FUNC_CALL_POST] * (n // 2), + "process_id": pd.Series([1] * n, dtype="int64"), + "thread_id": pd.Series([2] * n, dtype="int64"), + "func_call_id": ids, + "function": ["fixture.step"] * n, + "args": [{} for _ in range(n)], + "kwargs": [{} for _ in range(n)], + "exception": ["fixture failure"] * n, + } + ) + + +def make(data, cls=TracePandas): + return cls(data.copy(deep=True), truncate_incomplete_func_calls=False) + + +def value(v): + if isinstance(v, dict): + return {k: value(x) for k, x in v.items()} + if isinstance(v, list): + return [value(x) for x in v] + if isinstance(v, float) and math.isnan(v): + return (type(v).__name__, "nan") + return (type(v).__name__, v) + + +def outcome(trace, bounds, process=1, thread=2): + try: + events = trace.query_func_call_events_within_time(bounds, process, thread) + return ("result", [(type(e).__name__, value(e.__dict__)) for e in events]) + except Exception as error: + return ("exception", type(error).__name__, str(error)) + + +class IntLike(int): + pass + + +class EqualType: + def __init__(self, text): + self.text = text + + def __eq__(self, other): + return self.text == other + + def __hash__(self): + return hash(self.text) + + +class TimeRangeIndexTests(unittest.TestCase): + def setUp(self): + previous = logging.root.manager.disable + logging.disable(logging.CRITICAL) + self.addCleanup(logging.disable, previous) + + def compare(self, data, queries): + old, new = make(data, LegacyRangeTrace), make(data) + for query in queries: + with self.subTest(query=query, dtypes=str(data.dtypes)): + self.assertEqual(outcome(old, *query), outcome(new, *query)) + return old, new + + def test_strict_bounds_ties_reverse_and_empty(self): + data = frame((1, 2, 2, 3)) + _, new = self.compare( + data, + [ + ((a, b),) + for a, b in [(0, 4), (1, 3), (1, 2), (2, 3), (2, 2), (4, 0), (9, 10)] + ], + ) + self.assertIsNotNone(new._func_time_positions) + self.assertEqual( + [ + e.pre_record["func_call_id"] + for e in new.query_func_call_events_within_time((0, 4), 1, 2) + ], + ["a", "z"], + ) + + def test_empty_native_frame_and_extreme_identity_keys(self): + _, empty = self.compare(frame().iloc[:0], [((0, 5),)]) + self.assertEqual(empty._get_func_call_time_positions((0, 5), 1, 2), []) + for pid, tid in [(-(2**63), 2**63 - 1), (2**63 - 1, -(2**63))]: + data = frame() + data["process_id"] = pd.Series([pid] * 4, dtype="int64") + data["thread_id"] = pd.Series([tid] * 4, dtype="int64") + _, trace = self.compare(data, [((0, 5), pid, tid)]) + self.assertEqual( + trace._get_func_call_time_positions((0, 5), pid, tid), [0, 1, 2, 3] + ) + + def test_float_nan_inf_signed_zero(self): + data = frame( + (-math.inf, -0.0, 0.0, math.inf, math.nan, math.nan), + ("z", "z", "a", "a", "missing", "missing"), + ) + self.compare( + data, + [ + ((a, b),) + for a, b in [ + (-math.inf, math.inf), + (-1.0, 1.0), + (-0.0, 0.0), + (math.inf, -math.inf), + (math.nan, 9.0), + ] + ], + ) + + def test_integer_extremes_and_precision_fallback(self): + for times in [ + (2**53, 2**53 + 1, 2**53 + 2, 2**53 + 3), + (-(2**63), -(2**63) + 1, 2**63 - 2, 2**63 - 1), + ]: + data = frame(times) + old, new = self.compare( + data, + [ + ((times[0], times[-1]),), + ((float(times[0]), float(times[-1])),), + ((-(2**63) - 1, 2**63),), + ], + ) + self.assertIsNone( + new._get_func_call_time_positions( + (float(times[0]), float(times[-1])), 1, 2 + ) + ) + data = frame(tuple(float(2**53 + i) for i in range(4))) + self.compare( + data, [((2**53 + 1, 2**53 + 3),), ((float(2**53), float(2**53 + 4)),)] + ) + + def test_numeric_query_types_fallback(self): + data = frame() + for process, thread in [ + (True, 2), + (1.0, 2.0), + (np.int64(1), np.int64(2)), + (IntLike(1), 2), + (2**64, 2), + (None, 2), + ]: + _, new = self.compare(data, [((0, 5), process, thread)]) + self.assertIsNone( + new._get_func_call_time_positions((0, 5), process, thread) + ) + for bounds in [ + (np.int64(0), np.int64(5)), + (IntLike(0), 5), + [0, 5], + (0,), + None, + ("a", "b"), + ]: + self.compare(data, [(bounds,)]) + + def test_other_processes_threads_and_missing_identity(self): + data = pd.concat([frame(), frame()], ignore_index=True) + data.loc[4:, "process_id"] = 3 + data.loc[4:, "thread_id"] = 4 + self.compare( + data, [((0, 5), pid, tid) for pid, tid in [(1, 2), (3, 4), (1, 4), (99, 2)]] + ) + + def test_unsorted_input_duplicate_labels_preserves_row_order(self): + data = frame((3, 4, 1, 2)) + data.index = [4, 4, 1, 1] + self.compare(data, [((0, 5),), ((0, 3),)]) + backwards = data.iloc[[1, 0, 2, 3]].copy() + self.assertEqual( + self.compare(backwards, [((0, 5),)])[1].query_func_call_events_within_time( + (0, 0), 1, 2 + ), + [], + ) + self.assertEqual(outcome(make(backwards), (0, 5))[1], "AssertionError") + + def test_duplicate_incomplete_and_exception_records(self): + for data in [ + frame().iloc[:3], + pd.concat([frame(), frame().iloc[:1]], ignore_index=True), + ]: + self.compare(data, [((0, 5),), ((0, 3),)]) + data = frame() + data.loc[1, "type"] = T.FUNC_CALL_POST_EXCEPTION + self.compare(data, [((0, 5),), ((0, 3),)]) + + def test_null_mixed_and_categorical_call_ids(self): + for ids in [ + pd.Series(["z", "z", None, None]), + pd.Series([1, 1, "a", "a"], dtype=object), + pd.Categorical(["z", "z", "a", "a"], categories=["z", "a", "unused"]), + pd.Series(["z", "z", "a", "a"], dtype="string"), + ]: + data = frame() + data["func_call_id"] = ids + self.compare(data, [((0, 5),), ((9, 10),)]) + + def test_time_and_identity_dtype_fallback(self): + for column in ["time", "process_id", "thread_id"]: + for dtype in ["object", "Int64", "Float64", "category", "uint64", "int32"]: + data = frame() + data[column] = data[column].astype(dtype) + _, new = self.compare(data, [((0, 5),)]) + self.assertIsNone(new._get_func_call_time_positions((0, 5), 1, 2)) + data = frame() + data[column] = data[column].astype(float) + data.loc[0, column] = np.nan + self.compare(data, [((0, 5),), ((0.0, 5.0),)]) + + def test_type_nulls_extensions_and_custom_comparison(self): + for dtype in ["string", "category"]: + data = frame() + data["type"] = data["type"].astype(dtype) + _, new = self.compare(data, [((0, 5),)]) + self.assertIsNone(new._get_func_call_time_positions((0, 5), 1, 2)) + for kind in [None, np.nan, pd.NA, EqualType(T.FUNC_CALL_PRE)]: + data = frame() + data.at[0, "type"] = kind + self.compare(data, [((0, 5),), ((9, 10),)]) + + def test_non_function_events_are_excluded(self): + data = pd.concat([frame(), frame()], ignore_index=True) + data.loc[4:, "type"] = "unrelated" + self.compare(data, [((0, 5),)]) + + def test_missing_and_duplicate_columns_keep_exceptions(self): + for col in ["type", "process_id", "thread_id", "func_call_id"]: + self.compare(frame().drop(columns=col), [((0, 5),)]) + data = pd.concat([frame(), frame()[["type"]]], axis=1) + self.compare(data, [((0, 5),)]) + + def test_cache_rebuild_on_replacement_and_length_change(self): + old, new = self.compare(frame(), [((0, 5),)]) + for trace in [old, new]: + trace.events = frame((10, 11, 12, 13)) + self.assertEqual(outcome(old, (9, 14)), outcome(new, (9, 14))) + for trace in [old, new]: + trace.events.drop(index=[2, 3], inplace=True) + self.assertEqual(outcome(old, (9, 14)), outcome(new, (9, 14))) + self.assertEqual(new._func_time_length, 2) + + def test_fresh_events_and_record_dicts(self): + trace = make(frame()) + first = trace.query_func_call_events_within_time((0, 5), 1, 2) + second = trace.query_func_call_events_within_time((0, 5), 1, 2) + self.assertIsNot(first[0], second[0]) + self.assertIsNot(first[0].pre_record, second[0].pre_record) + first[0].pre_record["function"] = "mutated" + self.assertEqual(second[0].pre_record["function"], "fixture.step") + self.assertEqual( + trace.query_func_call_events_within_time((0, 5), 1, 2)[0].pre_record[ + "function" + ], + "fixture.step", + ) + + def test_repeated_query_avoids_type_isin_rescan(self): + trace = make(frame()) + with patch.object( + pd.Series, "isin", side_effect=AssertionError("unexpected full scan") + ): + for _ in range(10): + self.assertEqual( + len(trace.query_func_call_events_within_time((0, 5), 1, 2)), 2 + ) + + def test_random_windows_and_permuted_rows(self): + rng = np.random.default_rng(91) + data = pd.concat( + [ + frame( + (i * 4 + 1, i * 4 + 2, i * 4 + 3, i * 4 + 4), + (f"{i}z", f"{i}z", f"{i}a", f"{i}a"), + ) + for i in range(20) + ], + ignore_index=True, + ) + for _ in range(10): + permuted = data.iloc[rng.permutation(len(data))].copy() + self.compare( + permuted, + [((int(a), int(b)),) for a, b in rng.integers(-2, 83, size=(20, 2))], + ) + + def test_pickled_index_retains_frame_and_results(self): + trace = make(frame()) + expected = outcome(trace, (0, 5)) + restored = pickle.loads(pickle.dumps(trace)) + self.assertIs(restored._func_time_source, restored.events) + self.assertEqual(outcome(restored, (0, 5)), expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_torchscript_compat.py b/tests/test_torchscript_compat.py new file mode 100644 index 00000000..8d6c7937 --- /dev/null +++ b/tests/test_torchscript_compat.py @@ -0,0 +1,307 @@ +"""Builtin-registration contracts and fresh-process bias-GELU regressions.""" + +import argparse +import copy +import gc +import json +import os +import subprocess +import sys +import tempfile +import unittest +import weakref +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from traincheck.instrumentor import torchscript_compat +from traincheck.instrumentor.torchscript_compat import ( + _REGISTERED_WRAPPERS, + preserve_torchscript_builtin, +) + + +def bias_gelu(bias: torch.Tensor, value: torch.Tensor) -> torch.Tensor: + """Real-source-file fixture for the tanh-based fused bias-GELU expression.""" + x = bias + value + return ( + x * 0.5 * (1.0 + torch.tanh(0.7978845608028654 * x * (1.0 + 0.044715 * x * x))) + ) + + +def evaluate(function): + bias = torch.linspace(-0.7, 0.9, 8, dtype=torch.float64, requires_grad=True) + value = ( + torch.linspace(-2.3, 2.1, 24, dtype=torch.float64) + .reshape(3, 8) + .requires_grad_() + ) + output = function(bias, value) + grad_bias, grad_value = torch.autograd.grad(output.square().sum(), (bias, value)) + return { + "output": output.detach().tolist(), + "grad_bias": grad_bias.tolist(), + "grad_value": grad_value.tolist(), + } + + +def probe(mode: str) -> dict[str, object]: + """Run one control in its own interpreter so JIT caches cannot leak.""" + from traincheck.instrumentor import tracer + + torch.set_num_threads(1) + torch.set_num_interop_threads(1) + records: list[dict[str, object]] = [] + registry = torch.jit._builtins + original = torch.tanh + result: dict[str, object] = { + "mode": mode, + "torch": torch.__version__, + "initial_registry_initialized": registry._builtin_table is not None, + } + if mode.endswith("warm"): + result["original_operator_before_wrapping"] = registry._find_builtin(original) + + with ( + patch.object(tracer.config, "DISABLE_WRAPPER", False), + patch.object( + tracer, + "dump_trace_API", + side_effect=lambda record: records.append(copy.deepcopy(record)), + ), + ): + if mode != "baseline": + if mode == "integrated-fixed-warm": + wrapped = tracer.wrapper( + original, + False, + False, + False, + dump_args=True, + dump_ret=True, + handle_proxy=False, + ) + else: + # Disable only registration to reproduce the former wrapper; + # the helper controls then apply the production helper directly. + with patch.object( + tracer, + "preserve_torchscript_builtin", + return_value={"status": "not-registered", "operator": None}, + ): + wrapped = tracer.wrapper( + original, + False, + False, + False, + dump_args=True, + dump_ret=True, + handle_proxy=False, + ) + if mode.startswith("fixed"): + result["candidate"] = preserve_torchscript_builtin( + original, wrapped, torch + ) + torch.tanh = wrapped + try: + result["eager"] = evaluate(bias_gelu) + result["eager_records"] = copy.deepcopy(records) + records.clear() + scripted = torch.jit.script(bias_gelu) + result["graph_has_aten_tanh"] = "aten::tanh" in str(scripted.graph) + result["scripted"] = evaluate(scripted) + result["scripted_records"] = copy.deepcopy(records) + result["script_compilation"] = "passed" + except Exception as error: + result["script_compilation"] = "failed" + result["exception_type"] = type(error).__name__ + result["exception"] = str(error) + finally: + result["current_operator_after_compilation"] = registry._find_builtin( + torch.tanh + ) + torch.tanh = original + return result + + +class BuiltinCompatibilityTests(unittest.TestCase): + def fake_torch(self, operator): + table, events = {}, [] + + def find(function): + events.append(("find", function)) + return operator + + def register(function, mapped_operator): + events.append(("register", function, mapped_operator)) + table[id(function)] = mapped_operator + + builtins = SimpleNamespace(_find_builtin=find, _register_builtin=register) + return SimpleNamespace(jit=SimpleNamespace(_builtins=builtins)), table, events + + def test_copies_exact_original_mapping(self): + fake, table, events = self.fake_torch("aten::tanh") + + def original(): + pass + + def wrapped(): + pass + + result = preserve_torchscript_builtin(original, wrapped, fake) + self.assertEqual(result, {"status": "preserved", "operator": "aten::tanh"}) + self.assertEqual( + events, [("find", original), ("register", wrapped, "aten::tanh")] + ) + self.assertEqual(table[id(wrapped)], "aten::tanh") + + def test_unregistered_function_is_unchanged(self): + fake, table, events = self.fake_torch(None) + original = lambda: None + wrapped = lambda: None + self.assertEqual( + preserve_torchscript_builtin(original, wrapped, fake)["status"], + "not-registered", + ) + self.assertEqual(table, {}) + self.assertEqual(events, [("find", original)]) + self.assertNotIn(wrapped, _REGISTERED_WRAPPERS) + + def test_unavailable_private_api_is_noop(self): + original = lambda: None + wrapped = lambda: None + missing_register = SimpleNamespace( + jit=SimpleNamespace( + _builtins=SimpleNamespace(_find_builtin=lambda f: "aten::tanh") + ) + ) + for fake in [SimpleNamespace(), missing_register]: + self.assertEqual( + preserve_torchscript_builtin(original, wrapped, fake)["status"], + "unsupported-private-api", + ) + self.assertNotIn(wrapped, _REGISTERED_WRAPPERS) + + def test_registered_wrapper_cannot_leave_stale_reusable_id(self): + fake, table, events = self.fake_torch("aten::tanh") + original = lambda: None + wrapped = lambda: None + weak = weakref.ref(wrapped) + address = id(wrapped) + preserve_torchscript_builtin(original, wrapped, fake) + events.clear() + del wrapped + gc.collect() + self.assertIsNotNone(weak()) + self.assertEqual(table[address], "aten::tanh") + self.assertEqual(_REGISTERED_WRAPPERS[weak()], "aten::tanh") + + +class TorchScriptWrapperTests(unittest.TestCase): + def test_fresh_process_bias_gelu_semantics_and_eager_traces(self): + registry = getattr(getattr(torch, "jit", None), "_builtins", None) + if not all( + callable(getattr(registry, name, None)) + for name in ["_find_builtin", "_register_builtin"] + ): + self.skipTest("PyTorch lacks the private builtin-registration APIs") + + source = Path(torchscript_compat.__file__).resolve().parents[2] + environment = os.environ | { + "PYTHONPATH": str(source), + "PYTHONDONTWRITEBYTECODE": "1", + "OMP_NUM_THREADS": "1", + "MKL_NUM_THREADS": "1", + "OPENBLAS_NUM_THREADS": "1", + "CUDA_VISIBLE_DEVICES": "", + } + modes = [ + "baseline", + "buggy-unprimed", + "buggy-warm", + "fixed-unprimed", + "fixed-warm", + "integrated-fixed-warm", + ] + results = {} + with tempfile.TemporaryDirectory(prefix="traincheck-torchscript-") as temporary: + for mode in modes: + output = Path(temporary) / (mode + ".json") + completed = subprocess.run( + [ + sys.executable, + str(Path(__file__).resolve()), + "--probe", + mode, + "--output", + str(output), + ], + cwd=source, + env=environment, + capture_output=True, + text=True, + timeout=90, + ) + self.assertEqual( + completed.returncode, 0, completed.stdout + completed.stderr + ) + results[mode] = json.loads(output.read_text()) + + baseline = results["baseline"] + checks = 0 + for mode in modes[:-1]: + result = results[mode] + with self.subTest(mode=mode): + self.assertEqual(result["eager"], baseline["eager"]) + checks += 1 + if mode.startswith("buggy"): + self.assertEqual(result["script_compilation"], "failed") + self.assertEqual(result["exception_type"], "TypeError") + self.assertIn("builtin_function_or_method", result["exception"]) + self.assertIsNone(result["current_operator_after_compilation"]) + else: + self.assertEqual(result["script_compilation"], "passed") + self.assertTrue(result["graph_has_aten_tanh"]) + self.assertEqual(result["scripted"], baseline["scripted"]) + self.assertEqual( + result["current_operator_after_compilation"], "aten::tanh" + ) + self.assertEqual(result["scripted_records"], []) + checks += 1 + if mode != "baseline": + self.assert_paired_eager_record(result) + checks += 1 + integrated = results["integrated-fixed-warm"] + self.assertEqual(integrated["script_compilation"], "passed") + self.assertTrue(integrated["graph_has_aten_tanh"]) + self.assertEqual(integrated["scripted"], baseline["scripted"]) + self.assertEqual(integrated["eager"], baseline["eager"]) + self.assertEqual(integrated["scripted_records"], []) + self.assertEqual(integrated["current_operator_after_compilation"], "aten::tanh") + self.assert_paired_eager_record(integrated) + checks += 1 + self.assertEqual(checks, 13) + + def assert_paired_eager_record(self, result): + records = result["eager_records"] + self.assertEqual(len(records), 2) + self.assertEqual(records[0]["type"], "function_call (pre)") + self.assertEqual(records[1]["type"], "function_call (post)") + self.assertEqual(records[0]["func_call_id"], records[1]["func_call_id"]) + self.assertEqual(records[0]["function"], records[1]["function"]) + self.assertIn("tanh", records[0]["function"]) + self.assertTrue(records[0]["args"]) + self.assertTrue(records[1]["return_values"]) + + +if __name__ == "__main__": + if "--probe" in sys.argv: + parser = argparse.ArgumentParser() + parser.add_argument("--probe", required=True) + parser.add_argument("--output", required=True, type=Path) + arguments = parser.parse_args() + arguments.output.write_text(json.dumps(probe(arguments.probe)) + "\n") + else: + unittest.main() diff --git a/tests/test_torchscript_contracts.py b/tests/test_torchscript_contracts.py new file mode 100644 index 00000000..fa841ed1 --- /dev/null +++ b/tests/test_torchscript_contracts.py @@ -0,0 +1,161 @@ +"""Explicit unsupported, malformed, and failed TorchScript boundary contracts.""" + +import threading +import unittest +import warnings +from types import SimpleNamespace +from typing import Callable +from unittest.mock import patch + +from traincheck.instrumentor import torchscript_compat, tracer +from traincheck.instrumentor.torchscript_compat import preserve_torchscript_builtin + + +def original(value: int = 1) -> int: + return value + 1 + + +def candidate(value: int = 1) -> int: + return original(value) + + +def framework(find: object, register: object) -> SimpleNamespace: + return SimpleNamespace( + jit=SimpleNamespace( + _builtins=SimpleNamespace(_find_builtin=find, _register_builtin=register) + ) + ) + + +class TorchScriptContractTests(unittest.TestCase): + def test_present_noncallable_callback_is_rejected(self) -> None: + def find(function: Callable[..., object]) -> str: + return "aten::tanh" + + def register(function: Callable[..., object], operator: str) -> None: + self.fail("Invalid capabilities must not register a wrapper") + + malformed_callbacks: tuple[object, ...] = (None, 17, "callable", []) + for malformed in malformed_callbacks: + for name, fake in ( + ("_find_builtin", framework(malformed, register)), + ("_register_builtin", framework(find, malformed)), + ): + with self.subTest(name=name, malformed=malformed): + with self.assertRaisesRegex(TypeError, name): + preserve_torchscript_builtin(original, candidate, fake) + + def test_missing_callback_does_not_hide_an_invalid_present_callback(self) -> None: + fake = SimpleNamespace( + jit=SimpleNamespace(_builtins=SimpleNamespace(_register_builtin=None)) + ) + with self.assertRaisesRegex(TypeError, "_register_builtin"): + preserve_torchscript_builtin(original, candidate, fake) + + def test_malformed_operator_is_rejected_before_registration(self) -> None: + registered: list[str] = [] + + def register(function: Callable[..., object], operator: str) -> None: + registered.append(operator) + + malformed_operators: tuple[object, ...] = ("", 1, False, [], {}) + for malformed in malformed_operators: + + def find(function: Callable[..., object]) -> object: + return malformed + + with self.subTest(operator=malformed): + with self.assertRaisesRegex(TypeError, "nonempty string or None"): + preserve_torchscript_builtin( + original, candidate, framework(find, register) + ) + self.assertEqual(registered, []) + self.assertNotIn(candidate, torchscript_compat._REGISTERED_WRAPPERS) + + def test_find_failure_propagates_unchanged(self) -> None: + error = RuntimeError("registry lookup failed") + + def find(function: Callable[..., object]) -> str | None: + raise error + + def register(function: Callable[..., object], operator: str) -> None: + self.fail("Failed lookup must not register a wrapper") + + with self.assertRaises(RuntimeError) as caught: + preserve_torchscript_builtin(original, candidate, framework(find, register)) + self.assertIs(caught.exception, error) + self.assertNotIn(candidate, torchscript_compat._REGISTERED_WRAPPERS) + + def test_registration_failure_propagates_without_retention(self) -> None: + error = RuntimeError("registry update failed") + + def find(function: Callable[..., object]) -> str: + return "aten::tanh" + + def register(function: Callable[..., object], operator: str) -> None: + raise error + + with self.assertRaises(RuntimeError) as caught: + preserve_torchscript_builtin(original, candidate, framework(find, register)) + self.assertIs(caught.exception, error) + self.assertNotIn(candidate, torchscript_compat._REGISTERED_WRAPPERS) + + def test_wrapper_exposes_unsupported_and_warns_once(self) -> None: + with ( + patch.object(tracer, "torch", SimpleNamespace()), + patch.object(torchscript_compat, "_UNSUPPORTED_WARNING_EMITTED", False), + warnings.catch_warnings(record=True) as captured, + ): + warnings.simplefilter("always") + first = tracer.wrapper( + original, False, False, False, disable_dump=True, handle_proxy=False + ) + second = tracer.wrapper( + original, False, False, False, disable_dump=True, handle_proxy=False + ) + self.assertEqual(first(3), 4) + self.assertEqual(second(8), 9) + self.assertEqual(len(captured), 1) + self.assertIn( + "Eager instrumentation remains available", str(captured[0].message) + ) + for wrapped in (first, second): + # Function metadata is deliberately public to instrumentation users. + self.assertEqual( + getattr(wrapped, "_traincheck_torchscript_builtin"), + {"status": "unsupported-private-api", "operator": None}, + ) + + def test_unsupported_warning_is_once_across_threads(self) -> None: + with ( + patch.object(torchscript_compat, "_UNSUPPORTED_WARNING_EMITTED", False), + warnings.catch_warnings(record=True) as captured, + ): + warnings.simplefilter("always") + workers = [ + threading.Thread(target=torchscript_compat.warn_unsupported_torchscript) + for _ in range(8) + ] + for worker in workers: + worker.start() + for worker in workers: + worker.join(5) + self.assertFalse(worker.is_alive()) + self.assertEqual(len(captured), 1) + + def test_wrapper_propagates_metadata_failure(self) -> None: + def find(function: Callable[..., object]) -> object: + return 3 + + def register(function: Callable[..., object], operator: str) -> None: + self.fail("Malformed operator must not be registered") + + with patch.object(tracer, "torch", framework(find, register)): + with self.assertRaisesRegex(TypeError, "nonempty string or None"): + tracer.wrapper( + original, False, False, False, disable_dump=True, handle_proxy=False + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_trace_integrity.py b/tests/test_trace_integrity.py new file mode 100644 index 00000000..26d9e0b2 --- /dev/null +++ b/tests/test_trace_integrity.py @@ -0,0 +1,272 @@ +"""Regression coverage for tracer self-reentry and concurrent buffer flushing.""" + +import copy +import json +import threading +import unittest +from queue import Queue +from unittest.mock import patch + +from traincheck.instrumentor import dumper, tracer + + +def wrapped(fn, name="outer", scan_proxy=False): + def call(*args, **kwargs): + return tracer.function_wrapper( + fn, + name, + False, + scan_proxy, + False, + True, + None, + True, + None, + False, + {}, + False, + *args, + **kwargs, + ) + + return call + + +class TracingTests(unittest.TestCase): + def setUp(self): + self.records = [] + self.patches = [ + patch.object(tracer.config, "DISABLE_WRAPPER", False), + patch.object( + tracer, + "dump_trace_API", + side_effect=lambda r: self.records.append(copy.deepcopy(r)), + ), + ] + for p in self.patches: + p.start() + self.addCleanup(p.stop) + + def test_serializer_reentry_does_not_trace_itself(self): + helper = wrapped(lambda x: x, "serializer_helper") + with patch.object( + tracer, "var_to_serializable", side_effect=lambda x: helper(x) + ): + self.assertEqual(wrapped(lambda x: x + 1)(3), 4) + self.assertEqual([r["function"] for r in self.records], ["outer", "outer"]) + + def test_encoding_reentry_and_nested_guard_restoration(self): + helper = wrapped(lambda: "encoded", "encoder_helper") + + def emit(record): + helper() + self.records.append(copy.deepcopy(record)) + + with ( + patch.object(tracer, "dump_trace_API", side_effect=emit), + patch.object(tracer, "var_to_serializable", side_effect=copy.deepcopy), + ): + wrapped(lambda x: x)(1) + with tracer.suppress_tracing(): + tracer.to_dict_return_value(1) + self.assertTrue(tracer.tracing_suppressed()) + self.assertFalse(tracer.tracing_suppressed()) + self.assertEqual([r["function"] for r in self.records], ["outer", "outer"]) + + def test_nested_user_calls_and_mutable_snapshots(self): + inner = wrapped(lambda x: x.append(2), "inner") + + def outer(x): + inner(x) + return x + + with patch.object(tracer, "var_to_serializable", side_effect=copy.deepcopy): + self.assertEqual(wrapped(outer)([1]), [1, 2]) + self.assertEqual( + [r["function"] for r in self.records], ["outer", "inner", "inner", "outer"] + ) + self.assertEqual(self.records[0]["args"][0], [1]) + self.assertEqual(self.records[-1]["return_values"], [1, 2]) + self.assertEqual( + self.records[0]["func_call_id"], self.records[-1]["func_call_id"] + ) + self.assertEqual( + self.records[1]["func_call_id"], self.records[2]["func_call_id"] + ) + + def test_real_tensor_conversion_preserves_pre_post_shape(self): + import torch + + value = torch.ones(2) + + def resize(tensor): + tensor.resize_(3) + return tensor + + self.assertIs(wrapped(resize)(value), value) + before = next(iter(self.records[0]["args"][0].values())) + after = next(iter(self.records[-1]["return_values"].values())) + self.assertEqual(before["shape"], (2,)) + self.assertEqual(after["shape"], (3,)) + self.assertEqual(self.records[0]["meta_vars"], self.records[-1]["meta_vars"]) + + def test_generator_application_calls_remain_visible(self): + inner = wrapped(lambda: 7, "inner") + + def values(): + yield inner() + + with patch.object(tracer, "var_to_serializable", side_effect=lambda x: repr(x)): + self.assertEqual(wrapped(lambda x: list(x), scan_proxy=True)(values()), [7]) + self.assertEqual( + [r["function"] for r in self.records], ["inner", "inner", "outer", "outer"] + ) + self.assertFalse(tracer.tracing_suppressed()) + + def test_conversion_exception_restores_guard(self): + with patch.object( + tracer, "var_to_serializable", side_effect=ValueError("conversion") + ): + with self.assertRaisesRegex(ValueError, "conversion"): + tracer.to_dict_args_kwargs((1,), {}) + with patch.object(tracer, "var_to_serializable", side_effect=copy.deepcopy): + wrapped(lambda x: x)(1) + self.assertEqual(len(self.records), 2) + + def test_frozen_framework_exception_preserved(self): + from dataclasses import dataclass + + @dataclass(frozen=True, slots=True) + class FrameworkSignal(Exception): + reason: str + + error = FrameworkSignal("cache bypass") + + def fail(): + raise error + + try: + wrapped(fail)() + except FrameworkSignal as caught: + self.assertIs(caught, error) + else: + self.fail("Original exception was swallowed") + self.assertFalse(tracer.tracing_suppressed()) + + def test_original_exception_preserved(self): + error = ValueError("original") + + def fail(): + raise error + + with self.assertRaises(ValueError) as caught: + wrapped(fail)() + self.assertIs(caught.exception, error) + self.assertEqual( + self.records[-1]["type"], tracer.TraceLineType.FUNC_CALL_POST_EXCEPTION + ) + self.assertEqual( + self.records[0]["func_call_id"], self.records[-1]["func_call_id"] + ) + + def test_conversion_suppression_is_thread_local(self): + entered, release = threading.Event(), threading.Event() + + def convert(x): + if x == "blocked": + entered.set() + if not release.wait(5): + raise TimeoutError("test release") + return x + + with patch.object(tracer, "var_to_serializable", side_effect=convert): + thread = threading.Thread( + target=tracer.to_dict_args_kwargs, args=(("blocked",), {}) + ) + thread.start() + try: + self.assertTrue(entered.wait(5)) + self.assertEqual(wrapped(lambda x: x)(3), 3) + finally: + release.set() + thread.join(5) + self.assertEqual(len(self.records), 2) + + +class BufferTests(unittest.TestCase): + def make_buffer(self, queue): + # Drive the periodic thread's flush explicitly, so scheduling is controlled. + with patch.object(threading.Thread, "start"): + return dumper.TraceBuffer( + lambda: queue, buffer_size=100, flush_interval=3600 + ) + + def test_concurrent_flush_exactly_once(self): + entered, release, second_done = ( + threading.Event(), + threading.Event(), + threading.Event(), + ) + + class BlockingQueue(Queue): + def put(self, item): + if not entered.is_set(): + entered.set() + if not release.wait(5): + raise TimeoutError("test release") + super().put(item) + + queue = BlockingQueue() + buffer = self.make_buffer(queue) + buffer.add_trace({"id": 1}) + first = threading.Thread(target=buffer._flush) + + def flush_again(): + buffer._flush() + second_done.set() + + second = threading.Thread(target=flush_again) + first.start() + try: + self.assertTrue(entered.wait(5)) + second.start() + second_done.wait(0.1) + finally: + release.set() + first.join(5) + second.join(5) + rows = [] + while not queue.empty(): + rows.extend(json.loads(row) for row in queue.get_nowait().splitlines()) + self.assertEqual(rows, [{"id": 1}]) + + def test_concurrent_producer_and_flush_order_and_snapshot(self): + queue = Queue() + buffer = self.make_buffer(queue) + done = threading.Event() + + def flush(): + while not done.is_set(): + buffer._flush() + + thread = threading.Thread(target=flush) + thread.start() + try: + item = {"id": 0} + for i in range(2000): + item["id"] = i + buffer.add_trace(item) + finally: + done.set() + thread.join(5) + buffer._flush() + rows = [] + while not queue.empty(): + rows.extend( + json.loads(row)["id"] for row in queue.get_nowait().splitlines() + ) + self.assertEqual(rows, list(range(2000))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_tracing_contracts.py b/tests/test_tracing_contracts.py new file mode 100644 index 00000000..1564692d --- /dev/null +++ b/tests/test_tracing_contracts.py @@ -0,0 +1,73 @@ +"""Failure contracts for recorder state and buffered writes.""" + +import threading +import unittest +from queue import Queue +from unittest.mock import patch + +from traincheck.instrumentor import dumper, tracer + + +class TracingContractTests(unittest.TestCase): + def test_application_scope_restores_suppression_after_exception(self) -> None: + error = ValueError("application failure") + with tracer.suppress_tracing(): + with self.assertRaises(ValueError) as caught: + with tracer.application_tracing(): + self.assertFalse(tracer.tracing_suppressed()) + raise error + self.assertIs(caught.exception, error) + self.assertTrue(tracer.tracing_suppressed()) + self.assertFalse(tracer.tracing_suppressed()) + + def test_new_thread_initializes_independent_state(self) -> None: + observations: list[bool] = [] + + def observe() -> None: + observations.append(tracer.tracing_suppressed()) + with tracer.suppress_tracing(): + observations.append(tracer.tracing_suppressed()) + observations.append(tracer.tracing_suppressed()) + + with tracer.suppress_tracing(): + thread = threading.Thread(target=observe) + thread.start() + thread.join(5) + self.assertFalse(thread.is_alive()) + self.assertTrue(tracer.tracing_suppressed()) + self.assertEqual(observations, [False, True, False]) + self.assertFalse(tracer.tracing_suppressed()) + + def test_failed_enqueue_retains_batch_and_propagates(self) -> None: + queue: Queue[str] = Queue() + with patch.object(threading.Thread, "start"): + buffer = dumper.TraceBuffer(lambda: queue, buffer_size=100) + buffer.add_trace({"id": 1}) + error = RuntimeError("queue unavailable") + with patch.object(queue, "put", side_effect=error): + with self.assertRaises(RuntimeError) as caught: + buffer._flush() + self.assertIs(caught.exception, error) + buffer._flush() + self.assertEqual(queue.get_nowait(), '{"id":1}') + self.assertTrue(queue.empty()) + buffer._flush() + self.assertTrue(queue.empty()) + + def test_serialization_failure_preserves_existing_batch(self) -> None: + queue: Queue[str] = Queue() + with patch.object(threading.Thread, "start"): + buffer = dumper.TraceBuffer(lambda: queue, buffer_size=100) + buffer.add_trace({"id": 1}) + error = TypeError("unsupported record") + with patch.object(dumper, "serialize", side_effect=error): + with self.assertRaises(TypeError) as caught: + buffer.add_trace({"id": object()}) + self.assertIs(caught.exception, error) + buffer._flush() + self.assertEqual(queue.get_nowait(), '{"id":1}') + self.assertTrue(queue.empty()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/benchmarks/identity_cache.py b/tools/benchmarks/identity_cache.py new file mode 100644 index 00000000..e42582e9 --- /dev/null +++ b/tools/benchmarks/identity_cache.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python3 +"""Deterministic identity-lookup microbenchmark, with no external trace input. + +Run against the candidate checkout, for example: + python -m tools.benchmarks.identity_cache > results.json +Small smoke test: + python -m tools.benchmarks.identity_cache --rows 1000 --calls 20 + +The legacy subclass changes only the two identity methods back to their +pre-cache implementation. Each trial constructs a fresh trace over the same +read-only DataFrame, then makes the same ordered get_meta_vars calls. The first +lookup/cache build is included. Three repetitions alternate execution order. +Input generation, correctness hashing, and garbage collection are not timed. + +This measures identity scans, not full inference. The synthetic trace has no +context managers. No RSS/PSS, peak memory, or multiprocessing scaling is measured. +The frame-memory estimate is not whole-process memory. Cache tuples add O(P+T) +stored values; each return still allocates a fresh O(P+T) list. The extra source +reference retains a frame until invalidation and is not a DataFrame copy. +""" + +import argparse +import gc +import hashlib +import inspect +import json +import logging +import os +import platform +import statistics +import subprocess +import sys +import time +from pathlib import Path +from typing import Literal, TypedDict + +import numpy as np +import pandas as pd + +import traincheck.trace.trace_pandas as trace_module +from traincheck.trace.trace_pandas import TracePandas + +TimingField = Literal["construction_seconds", "queries_seconds", "total_seconds"] +TIMING_FIELDS: tuple[TimingField, ...] = ( + "construction_seconds", + "queries_seconds", + "total_seconds", +) + + +class TrialResult(TypedDict): + repetition: int + position: int + variant: str + construction_seconds: float + queries_seconds: float + total_seconds: float + result_sha256: str + none_results: int + empty_dict_results: int + + +class Measurement(TrialResult, total=False): + cached_identity_counts: dict[str, int] + cache_tuple_containers_shallow_bytes: int + + +class LegacyIdentityTrace(TracePandas): + """Original lookup expressions; all other behavior uses the same checkout.""" + + def get_process_ids(self) -> list[object]: + if "process_id" not in self.events.columns: + return [] + return self.events["process_id"].dropna().unique().tolist() + + def get_thread_ids(self) -> list[object]: + if "thread_id" not in self.events.columns: + return [] + return self.events["thread_id"].dropna().unique().tolist() + + +def synthetic_frame(rows: int) -> pd.DataFrame: + sequence = np.arange(rows, dtype=np.int64) + process_ids = ((sequence // 16) % 8 + 100).astype(np.float64) + thread_ids = (sequence % 16 + 1000).astype(np.float64) + # NaNs exercise the same dropna work as mixed event traces. They never + # remove every occurrence of an identity at the minimum supported size. + process_ids[sequence % 97 == 0] = np.nan + thread_ids[sequence % 193 == 0] = np.nan + return pd.DataFrame( + {"time": sequence, "process_id": process_ids, "thread_id": thread_ids} + ) + + +def fixed_queries(calls: int) -> list[tuple[int, int, int]]: + # Six valid, one unknown-process, and one unknown-thread call per cycle. + # Process/thread presence remains independent, matching get_meta_vars. + return [ + ( + i, + -1 if i % 8 == 6 else 100 + i % 8, + -1 if i % 8 == 7 else 1000 + (i * 3) % 16, + ) + for i in range(calls) + ] + + +def digest(value: object) -> str: + return hashlib.sha256( + json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + + +def trial( + cls: type[TracePandas], + data: pd.DataFrame, + queries: list[tuple[int, int, int]], + *, + repetition: int, + position: int, + variant: str, +) -> Measurement: + gc.collect() + begin = time.perf_counter() + trace = cls(data, truncate_incomplete_func_calls=False) + constructed = time.perf_counter() + results = [trace.get_meta_vars(*query) for query in queries] + finished = time.perf_counter() + record: Measurement = { + "repetition": repetition, + "position": position, + "variant": variant, + "construction_seconds": constructed - begin, + "queries_seconds": finished - constructed, + "total_seconds": finished - begin, + "result_sha256": digest(results), + "none_results": sum(value is None for value in results), + "empty_dict_results": sum(value == {} for value in results), + } + if cls is TracePandas: + assert trace._identity_values_source is trace.events + record["cached_identity_counts"] = { + key: len(values) for key, values in trace._identity_values.items() + } + record["cache_tuple_containers_shallow_bytes"] = sum( + sys.getsizeof(values) for values in trace._identity_values.values() + ) + return record + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--rows", type=int, default=573494) + parser.add_argument("--calls", type=int, default=1000) + parser.add_argument("--repetitions", type=int, default=3) + args = parser.parse_args() + if args.rows < 256 or args.calls < 8 or args.repetitions < 1: + parser.error("require rows >= 256, calls >= 8, repetitions >= 1") + if not hasattr(TracePandas, "_get_identity_values"): + parser.error("import a candidate checkout containing the identity cache") + + logging.disable(logging.CRITICAL) + data = synthetic_frame(args.rows) + queries = fixed_queries(args.calls) + input_hash = hashlib.sha256( + pd.util.hash_pandas_object(data, index=True).to_numpy().tobytes() + ).hexdigest() + expected: list[dict[str, object] | None] = [ + None if i % 8 in (6, 7) else {} for i in range(args.calls) + ] + expected_hash = digest(expected) + measurements: list[Measurement] = [] + variants = {"legacy": LegacyIdentityTrace, "candidate": TracePandas} + for repetition in range(args.repetitions): + order = ["legacy", "candidate"] + if repetition % 2: + order.reverse() + for position, variant in enumerate(order): + record = trial( + variants[variant], + data, + queries, + repetition=repetition + 1, + position=position + 1, + variant=variant, + ) + assert record["result_sha256"] == expected_hash, record + measurements.append(record) + + # Catch accidental mutation of the shared input by either implementation. + assert ( + hashlib.sha256( + pd.util.hash_pandas_object(data, index=True).to_numpy().tobytes() + ).hexdigest() + == input_hash + ) + medians = { + variant: { + metric: statistics.median( + record[metric] + for record in measurements + if record["variant"] == variant + ) + for metric in TIMING_FIELDS + } + for variant in variants + } + report = { + "scope": "construction plus repeated identity-checked get_meta_vars calls", + "settings": vars(args), + "input": { + "sha256_pandas_row_hashes": input_hash, + "query_sha256": digest(queries), + "dtypes": {key: str(value) for key, value in data.dtypes.items()}, + "dataframe_deep_bytes": int(data.memory_usage(index=True, deep=True).sum()), + "context_manager_count": 0, + "process_count": 8, + "thread_count": 16, + }, + "runtime": { + "python": platform.python_version(), + "platform": platform.platform(), + "host_class": f"local {platform.system()} {platform.machine()} workstation", + "logical_cpu_count": os.cpu_count(), + "pandas": pd.__version__, + "numpy": np.__version__, + "native_thread_settings": { + name: os.environ.get(name) + for name in ( + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + ) + }, + "candidate_source_sha256": hashlib.sha256( + Path(inspect.getfile(trace_module)).read_bytes() + ).hexdigest(), + "candidate_commit": subprocess.check_output( + [ + "git", + "-C", + str(Path(inspect.getfile(trace_module)).resolve().parents[2]), + "rev-parse", + "HEAD", + ], + text=True, + ).strip(), + "candidate_source_matches_commit": subprocess.run( + [ + "git", + "-C", + str(Path(inspect.getfile(trace_module)).resolve().parents[2]), + "diff", + "--quiet", + "HEAD", + "--", + "traincheck/trace/trace_pandas.py", + ], + check=False, + ).returncode + == 0, + "legacy_methods_source_sha256": hashlib.sha256( + inspect.getsource(LegacyIdentityTrace).encode() + ).hexdigest(), + }, + "measurements": measurements, + "medians": medians, + "median_total_speedup": ( + medians["legacy"]["total_seconds"] / medians["candidate"]["total_seconds"] + ), + "correctness": { + "all_results_equal_expected": True, + "result_sha256": expected_hash, + "input_unchanged": True, + }, + "limitations": [ + "Synthetic identity-lookup microbenchmark, not completed inference.", + "No context-manager work or trace-file loading; input generation excluded.", + "No RSS/PSS, peak-memory, or multiprocessing memory-scaling measurement.", + "Cache stores O(P+T) values and returns fresh O(P+T) lists.", + "Stable identity values, dtypes, and row order required within one frame.", + ], + } + print(json.dumps(report, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tools/benchmarks/time_range_index.py b/tools/benchmarks/time_range_index.py new file mode 100644 index 00000000..c7ae93a0 --- /dev/null +++ b/tools/benchmarks/time_range_index.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +"""Compare exact pandas filtering with the time-range index on synthetic calls. + +Run: python -m tools.benchmarks.time_range_index > results.json +Each trial uses a fresh TracePandas. Construction and the first lazy index build +are included, along with fresh event/record construction for every query. Input +creation, result hashing, and garbage collection are outside the timed region. +Repetitions alternate legacy/candidate order. This isolates window queries; it +is not full inference and does not measure peak process memory. +""" + +import argparse +import gc +import hashlib +import inspect +import json +import logging +import os +import platform +import statistics +import time +from pathlib import Path +from typing import Literal, TypedDict + +import numpy as np +import pandas as pd + +import traincheck.trace.trace_pandas as trace_module +from traincheck.instrumentor.tracer import TraceLineType +from traincheck.trace.trace_pandas import TracePandas + +TimingField = Literal[ + "construction_seconds", + "first_query_seconds", + "remaining_queries_seconds", + "total_seconds", +] +TIMING_FIELDS: tuple[TimingField, ...] = ( + "construction_seconds", + "first_query_seconds", + "remaining_queries_seconds", + "total_seconds", +) + + +class TrialResult(TypedDict): + repetition: int + order: int + variant: str + construction_seconds: float + first_query_seconds: float + remaining_queries_seconds: float + total_seconds: float + result_sha256: str + + +class Measurement(TrialResult, total=False): + indexed_rows: int + identity_groups: int + + +class LegacyRangeTrace(TracePandas): + def _get_func_call_time_positions( + self, time_range: object, process_id: object, thread_id: object + ) -> None: + return None + + +def digest(value: object) -> str: + return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest() + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--rows", type=int, default=200000) + parser.add_argument("--queries", type=int, default=1000) + parser.add_argument("--repetitions", type=int, default=3) + args = parser.parse_args() + if args.rows < 4 or args.rows % 2 or args.queries < 2 or args.repetitions < 1: + parser.error("require even rows >= 4, queries >= 2, repetitions >= 1") + logging.disable(logging.CRITICAL) + row = np.arange(args.rows, dtype=np.int64) + call = row // 2 + data = pd.DataFrame( + { + "time": row.astype(np.float64), + "type": np.where( + row % 2 == 0, TraceLineType.FUNC_CALL_PRE, TraceLineType.FUNC_CALL_POST + ), + "process_id": 100 + call % 4, + "thread_id": 1000 + call % 8, + "func_call_id": [f"call-{i}" for i in call], + "function": "fixture.step", + } + ) + queries: list[tuple[tuple[float, float], int, int]] = [] + for i in range(args.queries): + index = (i * 97) % (args.rows // 2) + queries.append( + ( + (float(index * 2) - 0.5, float(index * 2) + 1.5), + 100 + index % 4, + 1000 + index % 8, + ) + ) + + def input_hash() -> str: + return hashlib.sha256( + pd.util.hash_pandas_object(data, index=True).to_numpy().tobytes() + ).hexdigest() + + before = input_hash() + records: list[Measurement] = [] + for repetition in range(args.repetitions): + variants = [("legacy", LegacyRangeTrace), ("candidate", TracePandas)] + if repetition % 2: + variants.reverse() + for order, (name, cls) in enumerate(variants): + gc.collect() + start = time.perf_counter() + trace = cls(data, truncate_incomplete_func_calls=False) + constructed = time.perf_counter() + results = [trace.query_func_call_events_within_time(*queries[0])] + first = time.perf_counter() + results.extend( + trace.query_func_call_events_within_time(*query) + for query in queries[1:] + ) + end = time.perf_counter() + assert all(len(events) == 1 for events in results) + summary = [ + [ + { + "class": type(event).__name__, + "pre": event.pre_record, + "post": event.post_record, + } + for event in events + ] + for events in results + ] + record: Measurement = { + "repetition": repetition + 1, + "order": order + 1, + "variant": name, + "construction_seconds": constructed - start, + "first_query_seconds": first - constructed, + "remaining_queries_seconds": end - first, + "total_seconds": end - start, + "result_sha256": digest(summary), + } + if name == "candidate": + assert trace._func_time_positions is not None + record["indexed_rows"] = sum( + len(positions) + for _, positions in trace._func_time_positions.values() + ) + record["identity_groups"] = len(trace._func_time_positions) + records.append(record) + assert input_hash() == before + assert len({record["result_sha256"] for record in records}) == 1 + medians = { + name: { + metric: statistics.median( + record[metric] for record in records if record["variant"] == name + ) + for metric in TIMING_FIELDS + } + for name in ["legacy", "candidate"] + } + print( + json.dumps( + { + "scope": "synthetic function-call window queries, including construction and lazy build; not full inference or peak memory", + "settings": vars(args), + "runtime": { + "python": platform.python_version(), + "platform": platform.platform(), + "pandas": pd.__version__, + "numpy": np.__version__, + "native_threads": { + name: os.environ.get(name) + for name in [ + "OMP_NUM_THREADS", + "OPENBLAS_NUM_THREADS", + "MKL_NUM_THREADS", + ] + }, + }, + "source_sha256": hashlib.sha256( + Path(inspect.getfile(trace_module)).read_bytes() + ).hexdigest(), + "input_sha256": before, + "input_unchanged": True, + "all_ordered_results_equal": True, + "measurements": records, + "medians": medians, + "median_total_ratio": medians["legacy"]["total_seconds"] + / medians["candidate"]["total_seconds"], + }, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/tools/benchmarks/trace_pandas_indexes.py b/tools/benchmarks/trace_pandas_indexes.py new file mode 100644 index 00000000..6f29b7de --- /dev/null +++ b/tools/benchmarks/trace_pandas_indexes.py @@ -0,0 +1,156 @@ +"""Reproduce bounded synthetic TracePandas indexing measurements from a checkout. + +Run: python -m tools.benchmarks.trace_pandas_indexes --output measurements.json +No training, cluster access, or saved trace files are required. Each lookup run +includes its first lazy-index construction. The legacy oracle preserves the four +original query methods; supporting runtime types are identical on both sides. +""" + +import argparse +import hashlib +import inspect +import json +import logging +import platform +import statistics +import time +from collections.abc import Callable +from pathlib import Path +from typing import TypedDict + +import numpy as np +import pandas as pd + +import traincheck.trace.trace_pandas as candidate +from tests.fixtures import legacy_trace_pandas as legacy +from tests.test_call_index import pair +from tests.test_context_index import cycle, encode, event +from traincheck.trace.trace_pandas import TracePandas + + +class ComparisonResult(TypedDict): + elapsed_seconds: dict[str, list[float]] + median_seconds: dict[str, float] + speedup: float + exact_results_equal: bool + + +def compare( + frame: pd.DataFrame, + operation: Callable[[type[TracePandas], pd.DataFrame], object], + repetitions: int, +) -> ComparisonResult: + seconds: dict[str, list[float]] = {"legacy": [], "candidate": []} + expected: object = None + for repetition in range(repetitions): + variants = [ + ("legacy", legacy.TracePandas), + ("candidate", candidate.TracePandas), + ] + if repetition % 2: + variants.reverse() + for name, trace_type in variants: + events = frame.copy(deep=True) + started = time.perf_counter() + value = operation(trace_type, events) + seconds[name].append(time.perf_counter() - started) + if expected is None: + expected = value + assert value == expected, f"{name} result differs from the legacy oracle" + medians = {name: statistics.median(values) for name, values in seconds.items()} + return { + "elapsed_seconds": seconds, + "median_seconds": medians, + "speedup": medians["legacy"] / medians["candidate"], + "exact_results_equal": True, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mode", choices=["all", "context", "call"], default="all") + parser.add_argument("--repetitions", type=int, default=3) + parser.add_argument("--contexts", type=int, default=500) + parser.add_argument("--noise", type=int, default=50_000) + parser.add_argument("--calls", type=int, default=20_000) + parser.add_argument( + "--queries", type=int, default=500, help="Pre/post pairs queried" + ) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + for name in ["repetitions", "contexts", "calls", "queries"]: + if getattr(args, name) < 1: + parser.error(f"--{name} must be positive") + if args.noise < 0: + parser.error("--noise must be nonnegative") + logging.disable(logging.WARNING) + results: dict[str, object] = {} + if args.mode in ("all", "context"): + rows = [] + for index in range(args.contexts): + rows.extend( + cycle(index * 3, obj=index % 20, name=f"fixtures.Ctx{index % 4}") + ) + rows.extend( + event(args.contexts * 3 + index, "math_call") for index in range(args.noise) + ) + frame = pd.DataFrame(rows) + + def prepare(trace_type: type[TracePandas], events: pd.DataFrame) -> object: + return encode(trace_type(events, truncate_incomplete_func_calls=False)) + + results["context"] = { + "rows": len(frame), + "contexts": args.contexts, + "timed_work": "constructor plus context-state serialization", + **compare(frame, prepare, args.repetitions), + } + if args.mode in ("all", "call"): + frame = pd.DataFrame( + [ + record + for index in range(args.calls) + for record in pair(f"call-{index:08}", start=index * 2) + ] + ) + wanted = [ + f"call-{(index * 117) % args.calls:08}" for index in range(args.queries) + ] + + def lookup(trace_type: type[TracePandas], events: pd.DataFrame) -> object: + trace = trace_type(events, truncate_incomplete_func_calls=False) + return [ + ( + trace.get_pre_func_call_record(call_id), + trace.get_post_func_call_record(call_id), + ) + for call_id in wanted + ] + + results["call"] = { + "rows": len(frame), + "record_queries": args.queries * 2, + "timed_work": "constructor plus record lookups, including lazy-index construction", + **compare(frame, lookup, args.repetitions), + } + output = { + "description": "Synthetic fixtures, alternating variant order; no training or inference", + "repetitions": args.repetitions, + "runtime": { + "python": platform.python_version(), + "pandas": pd.__version__, + "numpy": np.__version__, + }, + "candidate_source_sha256": hashlib.sha256( + Path(inspect.getfile(candidate)).read_bytes() + ).hexdigest(), + "results": results, + } + rendered = json.dumps(output, indent=2) + "\n" + if args.output: + args.output.write_text(rendered) + print(rendered, end="") + + +if __name__ == "__main__": + main() diff --git a/tools/fresh_validation/ADAPTERS.md b/tools/fresh_validation/ADAPTERS.md new file mode 100644 index 00000000..c3f84528 --- /dev/null +++ b/tools/fresh_validation/ADAPTERS.md @@ -0,0 +1,29 @@ +# Bounded trace-impact workloads + +`benchmark_workload.py` reads TrainCheck-Benchmarks without modifying it. Run each invocation in a fresh process with the chosen TrainCheck source tree in `PYTHONPATH`: + +```sh +PYTHONPATH=/path/to/traincheck-variant python benchmark_workload.py \ + --benchmark-root /path/to/clean/TrainCheck-Benchmarks \ + --workload mnist --steps 5 --seed 42 --batch-size 2 \ + --device cuda --data-root /path/to/MNIST-cache \ + --output /path/to/new/result --mode instrumented --save-checkpoint +``` + +Use `--mode baseline` for uninstrumented execution. That mode does not import TrainCheck. Modes share source, batch preparation, model initialization, explicit training RNG reset, CUDA synchronization, and training math. The output directory must not already contain a run. No datasets or packages are downloaded. + +These are **model micro-workloads adapted from the benchmark**, not full end-to-end benchmark reproductions: + +| Name | Exact benchmark definitions | Training/data adaptation | +| --- | --- | --- | +| `mnist` | `instr-overhead/overhead-e2e/mnist/main.py:Net` | Original forward, NLL, Adadelta(lr=1.0). First cached MNIST training samples if `--data-root` is supplied; otherwise explicitly synthetic uniform images and random labels. Original normalization preserved. No epoch scheduler or evaluation. | +| `dcgan` | `instr-overhead/overhead-e2e/dcgan/main.py:Generator,Discriminator,weights_init` | Original discriminator real/fake BCE then generator BCE updates and Adam hyperparameters. ngf/ndf reduced from 64 to 8. Synthetic uniform RGB images and precomputed Gaussian noise; not torchvision FakeData's exact generation. No image visualization. | +| `transformer` | `false-positive/RNN/trainset/word_language_model-RNN_TANH-3epo/model.py:PositionalEncoding,TransformerModel` | Two 32-dimensional encoder layers, 64-dimensional feedforward, 16-token sequences, vocabulary from checked-in WikiText training prefix. NLL/backward/clipped manual SGD. This repo's main loop is RNN-specific; this is an explicit Transformer training adaptation. | + +`adaptation.json` identifies exact source paths/SHA256, extracted class source SHA256, dataset hashes and adaptation details, version, model configuration, parameter count, and initial state shapes/hash. Full class/function source spans are saved in `benchmark_definitions.py`; top-level source CLIs are never executed. Original benchmark files retain their original license/copyright. + +Instrumented mode calls the public `Instrumentor` programmatically over `torch`, with API-only tracing, normal API coverage (`use_full_instr=False`), no selected API list, no proxy/model tracking, interval 1, and no warmup. It enables tracing only during training bodies via explicit `start_step`; initialization, dataset preparation, metric extraction, timing synchronization, and checkpoint writes are excluded. In both modes the training RNG is reset to `seed + 1` after setup. `TC_ORIGIN_AUDIT=1` enables a separately supplied `origin_audit.install()` before instrumentation, for a distinct diagnostic run. + +`steps.jsonl` preserves completed step timing/losses even if a later step fails. `metrics.json` includes completion, error, canonical initial/final model-state hashes and state shapes. `--save-checkpoint` writes the final model state dictionaries after successful training; it does not save optimizer/RNG state and therefore is not by itself a resume-equivalence evaluation. Canonical state hashes cover sorted names, dtypes, shapes, and tensor bytes, avoiding `torch.save` archive metadata differences. + +For fresh validation, `--data-steps 20` fixes corpus preparation and model vocabulary across runs that execute different numbers of training steps. `adaptation.json` records both the preparation horizon and actual steps, together with the producer PID, runtime and imported tracing-module path. diff --git a/tools/fresh_validation/benchmark_workload.py b/tools/fresh_validation/benchmark_workload.py new file mode 100644 index 00000000..50379d24 --- /dev/null +++ b/tools/fresh_validation/benchmark_workload.py @@ -0,0 +1,626 @@ +#!/usr/bin/env python3 +"""Bounded, source-identified model micro-workloads for the trace bug study. + +Run each invocation in a fresh process with the desired TrainCheck variant on +PYTHONPATH. The benchmark checkout is only read. These are explicit adaptations, +not claims to reproduce the complete benchmark's data or end-to-end training. +""" + +from __future__ import annotations + +import argparse +import ast +import copy +import gzip +import hashlib +import json +import os +import random +import struct +import subprocess +import sys +import time +from collections.abc import Callable, Mapping, Sequence +from pathlib import Path +from typing import TYPE_CHECKING, Literal, Protocol, TypedDict + +if TYPE_CHECKING: + import torch + import torch.nn as nn + +if __package__ in (None, ""): + sys.path.append(str(Path(__file__).resolve().parents[2])) +from tools.fresh_validation.contracts import JSONObject, json_value # noqa: E402 + + +class TracingConfig(Protocol): + DISABLE_WRAPPER: bool + MODEL_TRACKER_STYLE: str | None + INSTR_DESCRIPTORS: bool + INSTRUMENTATION_POLICY: dict[str, int] + + +def configure_tracing(config: TracingConfig) -> None: + config.DISABLE_WRAPPER = True + config.MODEL_TRACKER_STYLE = None + config.INSTR_DESCRIPTORS = False + config.INSTRUMENTATION_POLICY.update(interval=1, warm_up=0) + + +class WorkloadOptions(argparse.Namespace): + benchmark_root: Path + workload: Literal["mnist", "dcgan", "transformer"] + steps: int + data_steps: int | None + seed: int + output: Path + mode: Literal["baseline", "instrumented"] + device: Literal["cpu", "cuda"] + batch_size: int + cpu_threads: int + data_root: Path | None + save_checkpoint: bool + + +class TensorMetadata(TypedDict): + shape: list[int] + dtype: str + + +class WorkloadMetrics(TypedDict): + success: bool + error: dict[str, str] | None + completed_steps: int + steps: list[dict[str, int | float]] + initial_model_state_sha256: str + final_model_state_sha256: str + state_shapes_unchanged: bool + state_shapes: dict[str, TensorMetadata] + timing: str + + +SOURCES = { + "mnist": "instr-overhead/overhead-e2e/mnist/main.py", + "dcgan": "instr-overhead/overhead-e2e/dcgan/main.py", + "transformer": "false-positive/RNN/trainset/word_language_model-RNN_TANH-3epo/model.py", +} + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def write_json(path: Path, value: object) -> None: + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def definitions(source: Path, names: Sequence[str]) -> str: + """Extract exact source spans, never execute the benchmark's top-level CLI.""" + text = source.read_text() + nodes = { + node.name: node + for node in ast.parse(text).body + if isinstance(node, (ast.ClassDef, ast.FunctionDef)) + } + missing = set(names) - nodes.keys() + if missing: + raise ValueError("Missing benchmark definitions: " + str(sorted(missing))) + segments: list[str] = [] + for name in names: + segment = ast.get_source_segment(text, nodes[name]) + if segment is None: + raise ValueError(f"Benchmark definition {name} has no source span") + segments.append(segment) + return "\n\n".join(segments) + "\n" + + +def mnist_raw(data_root: Path, count: int) -> tuple[bytes, bytes, JSONObject]: + """Read local torchvision-style IDX cache without network or torchvision.""" + candidates = [data_root / "MNIST" / "raw", data_root / "raw", data_root] + for root in candidates: + image = next( + ( + root / name + for name in ("train-images-idx3-ubyte", "train-images-idx3-ubyte.gz") + if (root / name).is_file() + ), + None, + ) + labels = next( + ( + root / name + for name in ("train-labels-idx1-ubyte", "train-labels-idx1-ubyte.gz") + if (root / name).is_file() + ), + None, + ) + if image and labels: + break + else: + raise FileNotFoundError( + "--data-root must contain the cached MNIST training IDX images and labels" + ) + + def prefix(path: Path, length: int) -> bytes: + if path.suffix == ".gz": + with gzip.open(path, "rb") as handle: + return handle.read(length) + with path.open("rb") as raw_handle: + return raw_handle.read(length) + + images = prefix(image, 16 + count * 28 * 28) + targets = prefix(labels, 8 + count) + magic, available, rows, cols = struct.unpack(">IIII", images[:16]) + label_magic, available_labels = struct.unpack(">II", targets[:8]) + if (magic, rows, cols, label_magic) != (2051, 28, 28, 2049): + raise ValueError("Unexpected MNIST IDX header") + if ( + min(available, available_labels) < count + or len(images) != 16 + count * 784 + or len(targets) != 8 + count + ): + raise ValueError("MNIST cache has fewer requested samples than needed") + return ( + images[16:], + targets[8:], + { + "kind": "cached_mnist_training_prefix", + "count": count, + "files": [{"path": str(p), "sha256": sha256(p)} for p in (image, labels)], + "ordering": "first samples in IDX order; no shuffle", + "normalization": "(uint8 / 255 - 0.1307) / 0.3081", + }, + ) + + +def text_prefix(path: Path, count: int) -> tuple[list[int], int, JSONObject]: + words = [] + with path.open() as handle: + for line in handle: + words.extend(line.split() + [""]) + if len(words) >= count: + break + if len(words) < count: + raise ValueError("WikiText source is shorter than the bounded workload") + words = words[:count] + vocab: dict[str, int] = {} + ids = [vocab.setdefault(word, len(vocab)) for word in words] + return ( + ids, + len(vocab), + { + "kind": "wikitext2_training_prefix_with_prefix_only_vocabulary", + "path": str(path), + "sha256": sha256(path), + "token_count": len(ids), + "vocabulary_size": len(vocab), + "prefix_sha256": hashlib.sha256("\n".join(words).encode()).hexdigest(), + "adaptation": "Whitespace tokenization plus ; vocabulary and tensors are bounded to the chosen prefix, unlike the full benchmark corpus.", + }, + ) + + +def module_class(namespace: Mapping[str, object], name: str) -> type[nn.Module]: + import torch.nn as nn + + value = namespace[name] + if not isinstance(value, type) or not issubclass(value, nn.Module): + raise ValueError(f"Benchmark definition {name} must be an nn.Module class") + return value + + +def prepare( + args: WorkloadOptions, +) -> tuple[dict[str, nn.Module], Callable[[int], dict[str, torch.Tensor]], JSONObject]: + import math + + import torch + import torch.nn as nn + import torch.nn.functional as F + + source = args.benchmark_root / SOURCES[args.workload] + symbols = { + "mnist": ["Net"], + "dcgan": ["weights_init", "Generator", "Discriminator"], + "transformer": ["PositionalEncoding", "TransformerModel"], + }[args.workload] + extracted = definitions(source, symbols) + (args.output / "benchmark_definitions.py").write_text(extracted) + ns: dict[str, object] = { + "__name__": "trace_impact_benchmark", + "torch": torch, + "nn": nn, + "F": F, + "math": math, + "nc": 3, + "nz": 100, + "ngf": 8, + "ndf": 8, + } + exec(compile(extracted, str(args.output / "benchmark_definitions.py"), "exec"), ns) + manifest: JSONObject = { + "workload_kind": "bounded_model_micro_workload", + "benchmark_source": str(source), + "benchmark_source_sha256": sha256(source), + "extracted_symbols": [symbol for symbol in symbols], + "extracted_definitions_sha256": hashlib.sha256(extracted.encode()).hexdigest(), + "steps": args.steps, + "seed": args.seed, + "batch_size": args.batch_size, + "device": args.device, + "cpu_threads": args.cpu_threads, + "precision": "float32", + "deterministic_algorithms": True, + "training_only_trace": True, + "excluded_phases": [ + "model initialization", + "data preparation", + "metric extraction", + "checkpoint writing", + ], + } + try: + manifest["benchmark_git_head"] = subprocess.check_output( + ["git", "-C", str(args.benchmark_root), "rev-parse", "HEAD"], text=True + ).strip() + except (OSError, subprocess.CalledProcessError): + manifest["benchmark_git_head"] = None + batch = args.batch_size + count = args.steps * batch + device = torch.device(args.device) + if args.workload == "mnist": + model = module_class(ns, "Net")().to(device) + optimizer = torch.optim.Adadelta(model.parameters(), lr=1.0) + if args.data_root: + raw_images, raw_labels, provenance = mnist_raw(args.data_root, count) + x = ( + torch.tensor(list(raw_images), dtype=torch.float32).reshape( + count, 1, 28, 28 + ) + / 255 + ) + y = torch.tensor(list(raw_labels), dtype=torch.int64) + else: + x = torch.rand(count, 1, 28, 28) + y = torch.randint(0, 10, (count,)) + provenance = { + "kind": "synthetic_uniform_images_and_random_labels", + "count": count, + "is_original_mnist": False, + "normalization": "(uniform[0,1] - 0.1307) / 0.3081", + } + x = ((x - 0.1307) / 0.3081).to(device) + y = y.to(device) + model.train() + + def step(index: int) -> dict[str, torch.Tensor]: + optimizer.zero_grad() + output = model(x[index * batch : (index + 1) * batch]) + loss = F.nll_loss(output, y[index * batch : (index + 1) * batch]) + loss.backward() + optimizer.step() + return {"loss": loss} + + models = {"model": model} + manifest.update( + data=provenance, + model_config={"architecture": "exact Net class"}, + update="Exact benchmark forward, NLL loss, backward, Adadelta(lr=1.0). Epoch scheduler/evaluation omitted.", + ) + elif args.workload == "dcgan": + generator = module_class(ns, "Generator")(1).to(device) + initializer = ns["weights_init"] + if not callable(initializer): + raise ValueError("Benchmark weights_init must be callable") + + def initialize(module: nn.Module) -> None: + result: object = initializer(module) + if result is not None: + raise ValueError("Benchmark weights_init must return None") + + generator.apply(initialize) + discriminator = module_class(ns, "Discriminator")(1).to(device) + discriminator.apply(initialize) + optimizer_d = torch.optim.Adam( + discriminator.parameters(), lr=0.0002, betas=(0.5, 0.999) + ) + optimizer_g = torch.optim.Adam( + generator.parameters(), lr=0.0002, betas=(0.5, 0.999) + ) + criterion = nn.BCELoss() + images = torch.rand(count, 3, 64, 64).to(device) + # FakeData/ToTensor in the benchmark also supplies [0,1] images, but its + # per-index generation is not identical to these explicitly synthetic inputs. + noise_batches = torch.randn(args.steps, batch, 100, 1, 1).to(device) + generator.train() + discriminator.train() + + def step(index: int) -> dict[str, torch.Tensor]: + discriminator.zero_grad() + real = images[index * batch : (index + 1) * batch] + label = torch.full((batch,), 1, dtype=real.dtype, device=device) + output = discriminator(real) + error_real = criterion(output, label) + error_real.backward() + fake = generator(noise_batches[index]) + label.fill_(0) + output = discriminator(fake.detach()) + error_fake = criterion(output, label) + error_fake.backward() + loss_d = error_real + error_fake + optimizer_d.step() + generator.zero_grad() + label.fill_(1) + output = discriminator(fake) + loss_g = criterion(output, label) + loss_g.backward() + optimizer_g.step() + return {"loss_discriminator": loss_d, "loss_generator": loss_g} + + models = {"generator": generator, "discriminator": discriminator} + manifest.update( + data={ + "kind": "synthetic_uniform_rgb_images_and_fixed_gaussian_noise", + "count": count, + "is_original_dataset": False, + "noise_precomputed_before_trace": True, + }, + model_config={ + "ngpu": 1, + "nz": 100, + "ngf": 8, + "ndf": 8, + "nc": 3, + "image_size": 64, + }, + update="Benchmark BCE discriminator real/fake updates then generator update; Adam lr=0.0002 betas=(0.5,0.999). Channels reduced from defaults 64 to 8. Image visualization and logging reductions omitted.", + ) + else: + seq = 16 + corpus = source.parent / "data/wikitext-2/train.txt" + ids, vocab, provenance = text_prefix(corpus, (args.steps * seq + 1) * batch) + data = ( + torch.tensor(ids, dtype=torch.int64) + .view(batch, -1) + .t() + .contiguous() + .to(device) + ) + model = module_class(ns, "TransformerModel")(vocab, 32, 2, 64, 2, 0.2).to( + device + ) + model.train() + + def step(index: int) -> dict[str, torch.Tensor]: + start = index * seq + inputs = data[start : start + seq] + targets = data[start + 1 : start + seq + 1].reshape(-1) + model.zero_grad() + output = model(inputs) + loss = F.nll_loss(output.reshape(-1, vocab), targets) + loss.backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), 0.25) + for parameter in model.parameters(): + if parameter.grad is not None: + parameter.data.add_(parameter.grad, alpha=-1.0) + return {"loss": loss} + + models = {"model": model} + manifest.update( + data=provenance, + model_config={ + "ntoken": vocab, + "ninp": 32, + "nhead": 2, + "nhid": 64, + "nlayers": 2, + "dropout_argument": 0.2, + "sequence_length": seq, + "lr": 1.0, + "clip": 0.25, + }, + update="Adapted language-model forward/NLL/backward/clipped manual SGD. Model classes copied exactly; the benchmark main.py training path is RNN-specific and is not executed. Transformer dimensions/vocabulary reduced.", + ) + return models, step, manifest + + +def model_digest( + models: Mapping[str, nn.Module] +) -> tuple[str, dict[str, TensorMetadata]]: + """Hash tensor contents, names, dtype and shape; independent of torch.save metadata.""" + import torch + + digest = hashlib.sha256() + shapes: dict[str, TensorMetadata] = {} + for model_name, model in sorted(models.items()): + for name, value in sorted(model.state_dict().items()): + key = model_name + "." + name + if not isinstance(value, torch.Tensor): + raise ValueError(f"Benchmark state {key} is not a tensor") + tensor = value.detach().cpu().contiguous() + metadata: TensorMetadata = { + "shape": list(tensor.shape), + "dtype": str(tensor.dtype), + } + shapes[key] = metadata + digest.update( + key.encode() + + b"\0" + + json.dumps(metadata, sort_keys=True).encode() + + b"\0" + ) + digest.update(tensor.numpy().tobytes()) + return digest.hexdigest(), shapes + + +def run(args: WorkloadOptions) -> None: + os.environ.setdefault("CUBLAS_WORKSPACE_CONFIG", ":4096:8") + os.environ["TRAINCHECK_OUTPUT_DIR"] = str(args.output / "traces") + import torch + + torch.set_num_threads(args.cpu_threads) + torch.set_num_interop_threads(1) + random.seed(args.seed) + torch.manual_seed(args.seed) + torch.use_deterministic_algorithms(True) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(args.seed) + torch.backends.cudnn.benchmark = False + torch.backends.cudnn.deterministic = True + prepare_args = copy.copy(args) + prepare_args.steps = args.data_steps or args.steps + if prepare_args.steps < args.steps: + raise ValueError("data_steps must cover every training step") + models, step, manifest = prepare(prepare_args) + manifest.update( + steps=args.steps, + prepared_data_steps=prepare_args.steps, + process_id=os.getpid(), + source_root=os.environ.get("PYTHONPATH"), + ) + initial_hash, shapes = model_digest(models) + manifest["adapter_sha256"] = sha256(Path(__file__)) + manifest["mode"] = args.mode + manifest["torch_version"] = str(torch.__version__) + manifest["parameter_count"] = sum( + p.numel() for model in models.values() for p in model.parameters() + ) + manifest["initial_model_state_sha256"] = initial_hash + manifest["state_shapes"] = json_value(shapes) + manifest["instrumentation"] = { + "target": "torch", + "models_to_track": [], + "use_full_instr": False, + "scan_proxy_in_args": False, + "funcs_to_instr": None, + "sampling_interval": 1, + "warm_up_steps": 0, + } + manifest["python_version"] = __import__("sys").version + write_json(args.output / "adaptation.json", manifest) + tc_config: TracingConfig | None = None + start_step: Callable[[], object] | None = None + if args.mode == "instrumented": + (args.output / "traces").mkdir() + import traincheck.instrumentor.tracer as tracer_module + from traincheck.config import config as active_config + from traincheck.instrumentor.control import start_step as active_start_step + from traincheck.instrumentor.tracer import Instrumentor + + tc_config = active_config + start_step = active_start_step + manifest["tracer_module"] = str(Path(tracer_module.__file__).resolve()) + configure_tracing(tc_config) + if os.environ.get("TC_ORIGIN_AUDIT"): + from origin_audit import install + + install() + count = Instrumentor( + torch, + scan_proxy_in_args=False, + use_full_instr=False, + funcs_to_instr=None, + API_dump_stack_trace=False, + sampling_interval=1, + warm_up_steps=0, + ).instrument() + manifest["instrumented_function_count"] = count + write_json(args.output / "adaptation.json", manifest) + # Imports/instrumentation are setup, and must not alter training RNG state. + random.seed(args.seed + 1) + torch.manual_seed(args.seed + 1) + if args.device == "cuda": + torch.cuda.manual_seed_all(args.seed + 1) + manifest["training_rng_seed"] = args.seed + 1 + write_json(args.output / "adaptation.json", manifest) + records: list[dict[str, int | float]] = [] + success = False + error: dict[str, str] | None = None + try: + for index in range(args.steps): + if args.device == "cuda": + torch.cuda.synchronize() + started = time.perf_counter() + if start_step: + start_step() + try: + losses = step(index) + finally: + if tc_config is not None: + tc_config.DISABLE_WRAPPER = True + if args.device == "cuda": + torch.cuda.synchronize() + elapsed = time.perf_counter() - started + record = { + "step": index + 1, + "seconds": elapsed, + **{name: float(loss.detach().item()) for name, loss in losses.items()}, + } + records.append(record) + with (args.output / "steps.jsonl").open("a") as handle: + handle.write(json.dumps(record, sort_keys=True) + "\n") + print(json.dumps(record, sort_keys=True), flush=True) + success = True + except BaseException as exc: + error = {"type": type(exc).__name__, "message": str(exc)} + raise + finally: + if tc_config is not None: + tc_config.DISABLE_WRAPPER = True + final_hash, final_shapes = model_digest(models) + result: WorkloadMetrics = { + "success": success, + "error": error, + "completed_steps": len(records), + "steps": records, + "initial_model_state_sha256": initial_hash, + "final_model_state_sha256": final_hash, + "state_shapes_unchanged": shapes == final_shapes, + "state_shapes": final_shapes, + "timing": "synchronized training-body wall seconds; includes start_step and wrapper-disable assignment; excludes loss extraction/data preparation/checkpoint", + } + write_json(args.output / "metrics.json", result) + if success and args.save_checkpoint: + torch.save( + {name: model.state_dict() for name, model in models.items()}, + args.output / "checkpoint.pt", + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--benchmark-root", type=Path, required=True) + parser.add_argument("--workload", choices=sorted(SOURCES), required=True) + parser.add_argument("--steps", type=int, default=5) + parser.add_argument( + "--data-steps", + type=int, + help="Fixed corpus/model preparation horizon shared by reference and holdout", + ) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--mode", choices=("baseline", "instrumented"), required=True) + parser.add_argument("--device", choices=("cpu", "cuda"), default="cpu") + parser.add_argument("--batch-size", type=int, default=2) + parser.add_argument("--cpu-threads", type=int, default=1) + parser.add_argument( + "--data-root", type=Path, help="Optional local MNIST IDX cache; never downloads" + ) + parser.add_argument("--save-checkpoint", action="store_true") + args = parser.parse_args(namespace=WorkloadOptions()) + if args.steps < 1 or args.batch_size < 1 or args.cpu_threads < 1: + parser.error("steps, batch-size, cpu-threads must be positive") + if args.data_root and args.workload != "mnist": + parser.error("--data-root currently applies only to MNIST") + args.benchmark_root = args.benchmark_root.resolve() + args.output = args.output.resolve() + args.output.mkdir(parents=True, exist_ok=True) + if (args.output / "adaptation.json").exists() or ( + args.output / "steps.jsonl" + ).exists(): + parser.error("Output already contains a run; choose a new directory") + run(args) + + +if __name__ == "__main__": + main() diff --git a/tools/fresh_validation/check_coverage.py b/tools/fresh_validation/check_coverage.py new file mode 100644 index 00000000..f0db31de --- /dev/null +++ b/tools/fresh_validation/check_coverage.py @@ -0,0 +1,782 @@ +"""Observe the real offline checker; never replace its predicate or learned rules. + +Counters have explicit units. API candidates are calls, DistinctArgument candidates +are within-step argument pairs, and precondition counters are group-verification +calls. Each checker stops at its first violation. No wall-time claims are made. +""" + +import argparse +import ast +import hashlib +import importlib.metadata +import inspect +import json +import logging +import math +import os +import sys +from collections import Counter +from collections.abc import Callable, Mapping +from pathlib import Path +from types import FrameType +from typing import Literal, TypeAlias, TypedDict +from unittest.mock import patch + +from traincheck.invariant import read_inv_file +from traincheck.invariant.base_cls import APIParam, GroupedPreconditions, Invariant +from traincheck.invariant.DistinctArgumentRelation import ( + get_event_data_per_function_per_step, +) +from traincheck.trace.trace import Trace +from traincheck.trace.trace_pandas import TracePandas, read_trace_file_Pandas +from traincheck.trace.types import ( + FuncCallExceptionEvent, + IncompleteFuncCallEvent, + MDNONEJSONEncoder, +) + +# Direct CLI execution must resolve this checkout's harness package even when +# PYTHONPATH intentionally points to a separately composed TrainCheck source. +if __package__ in (None, ""): + sys.path.append(str(Path(__file__).resolve().parents[2])) +from tools.fresh_validation.contracts import ( # noqa: E402 + JSONObject, + JSONValue, + boolean, + integer, + json_list, + json_object, + json_value, + mapping, + required, + sequence, + string, +) + + +class ObserverContractError(ValueError): + """Checker internals no longer satisfy the observer's verified contract.""" + + +class ObserverSites(TypedDict): + starts: dict[int, str] + ends: dict[int, int] + visits: set[int] + method_source_sha256: str + + +class CallDecision(TypedDict): + func_call_id: str + + +class PairDecision(TypedDict): + pair_func_call_ids: list[str] + + +Decision: TypeAlias = CallDecision | PairDecision + + +class PendingDecision(TypedDict): + start: int + end: int + record: Decision + + +class GroupCounts(TypedDict): + attempts: int + passes: int + failures: int + errors: int + + +class CandidateCounts(TypedDict): + candidate_calls: int + opportunities: int + opportunity_unit: str + incomplete_or_exception_calls: int | None + + +class OptionalReportFields(TypedDict, total=False): + invariant_index: int + checker_result: JSONObject + error: dict[str, str] + checker_error_after_observer: dict[str, str] + candidate_calls: int + opportunities: int + opportunity_unit: str + incomplete_or_exception_calls: int | None + method_source_sha256: str + decisions_started: int + decisions_completed: int + decisions_not_completed: int + visited_calls: int | None + precondition_group_attempts: int + precondition_group_passes: int + precondition_group_failures: int + precondition_group_errors: int + precondition_groups: dict[str, GroupCounts] + decision_records: list[Decision] + unvisited_calls_after_stop: int | None + unvisited_pairs_after_stop: int + variable_precondition_requested: bool + variable_check_skipped: bool + fully_validated_active: bool + + +class ObservationReport(OptionalReportFields): + # Optional detail fields are present only for the matching status. Core + # outcome/observation fields are always present; unknown is never false. + relation: str + description: str | None + identity_clauses: list[JSONObject] + observer_supported: bool + status: Literal[ + "completed", "checker_error", "observer_error", "unsupported_coverage" + ] + coverage_status: Literal["observed", "disabled", "unsupported", "error"] + coverage_observed: bool + active: bool | None + dormant: bool | None + check_passed: bool | None + checker_triggered: bool | None + + +class RuntimeMetadata(TypedDict): + python: str + executable: str + trace_module: str + packages: dict[str, str] + + +class CoverageOutput(TypedDict): + schema_version: Literal[2] + pid: int + check_relation_first: Literal[False] + observer_enabled: bool + runtime: RuntimeMetadata + harness_sha256: str + raw_process_ids: list[JSONValue] + raw_thread_ids: list[JSONValue] + trace_rows: int + input_sha256: dict[str, str] + inputs_unchanged: bool + summary: dict[str, int | None] + per_invariant: list[ObservationReport] + + +TraceCallback: TypeAlias = Callable[[FrameType, str, object], "TraceCallback | None"] + + +def decision_record(family: str, local_values: Mapping[str, object]) -> Decision: + """Read the exact family-specific identity fields, failing closed on drift.""" + try: + if family == "DistinctArgumentRelation": + ids = [] + for name in ("event1", "event2"): + event = mapping(required(local_values, name), name) + ids.append( + string(required(event, "func_call_id"), f"{name}.func_call_id") + ) + return {"pair_func_call_ids": ids} + name = ( + "parent_func_call_id" if family == "APIContainRelation" else "func_call_id" + ) + if family not in SUPPORTED: + raise ValueError(f"Unsupported observer family: {family}") + return {"func_call_id": string(required(local_values, name), name)} + except ValueError as exc: + raise ObserverContractError(str(exc)) from exc + + +def finalize_report(report: ObservationReport) -> ObservationReport: + """Validate the conditional schema before exposing a mutable report. + + OptionalReportFields is an internal construction accumulator, not permission + to omit completed-result or observed-coverage fields in emitted reports. + """ + status = report["status"] + coverage_status = report["coverage_status"] + if status not in { + "completed", + "checker_error", + "observer_error", + "unsupported_coverage", + }: + raise ObserverContractError(f"Unknown checker status: {status}") + if coverage_status not in {"observed", "disabled", "unsupported", "error"}: + raise ObserverContractError(f"Unknown coverage status: {coverage_status}") + try: + if status == "completed": + checker = mapping(required(report, "checker_result"), "checker_result") + passed = boolean(report["check_passed"], "check_passed") + triggered = boolean(report["checker_triggered"], "checker_triggered") + if ( + boolean( + required(checker, "check_passed"), "checker_result.check_passed" + ) + != passed + or boolean(required(checker, "triggered"), "checker_result.triggered") + != triggered + ): + raise ValueError("Checker outcome and report outcome disagree") + mapping(required(checker, "invariant"), "checker_result.invariant") + else: + if ( + report["check_passed"] is not None + or report["checker_triggered"] is not None + ): + raise ValueError("Incomplete checking must have unknown outcome") + if status in {"checker_error", "observer_error"}: + error = mapping(required(report, "error"), "error") + string(required(error, "type"), "error.type") + string(required(error, "message"), "error.message") + if coverage_status == "observed": + if report["coverage_observed"] is not True or status not in { + "completed", + "checker_error", + }: + raise ValueError("Observed coverage has incompatible checker status") + boolean(report["active"], "active") + boolean(report["dormant"], "dormant") + for key in ( + "candidate_calls", + "opportunities", + "decisions_started", + "decisions_completed", + "decisions_not_completed", + "precondition_group_attempts", + "precondition_group_passes", + "precondition_group_failures", + "precondition_group_errors", + ): + if integer(required(report, key), key) < 0: + raise ValueError(f"{key} cannot be negative") + for key in ( + "variable_precondition_requested", + "variable_check_skipped", + "fully_validated_active", + ): + boolean(required(report, key), key) + for key in ("opportunity_unit", "method_source_sha256"): + string(required(report, key), key) + groups = mapping( + required(report, "precondition_groups"), "precondition_groups" + ) + for name, raw_group in groups.items(): + group = mapping(raw_group, name) + for key in ("attempts", "passes", "failures", "errors"): + if integer(required(group, key), f"{name}.{key}") < 0: + raise ValueError(f"{name}.{key} cannot be negative") + if report["relation"] == "DistinctArgumentRelation": + if ( + required(report, "visited_calls") is not None + or required(report, "unvisited_calls_after_stop") is not None + ): + raise ValueError("Pair coverage must not claim call visit counts") + integer( + required(report, "unvisited_pairs_after_stop"), + "unvisited_pairs_after_stop", + ) + else: + integer(required(report, "visited_calls"), "visited_calls") + integer( + required(report, "unvisited_calls_after_stop"), + "unvisited_calls_after_stop", + ) + for raw in sequence( + required(report, "decision_records"), "decision_records" + ): + decision = mapping(raw, "decision") + if report["relation"] == "DistinctArgumentRelation": + pair = sequence( + required(decision, "pair_func_call_ids"), "pair_func_call_ids" + ) + if len(pair) != 2: + raise ValueError("Pair decisions require exactly two call IDs") + for identity in pair: + string(identity, "pair call ID") + else: + string(required(decision, "func_call_id"), "decision.func_call_id") + elif ( + report["coverage_observed"] is not False + or report["active"] is not None + or report["dormant"] is not None + ): + raise ValueError( + "Unobserved coverage must have unknown active/dormant states" + ) + if status == "unsupported_coverage" and coverage_status != "unsupported": + raise ValueError("Unsupported checker coverage must be marked unsupported") + if status == "observer_error" and coverage_status != "error": + raise ValueError("Observer failure must mark coverage as an error") + except ValueError as exc: + raise ObserverContractError(str(exc)) from exc + return report + + +def summarize(results: list[ObservationReport]) -> dict[str, int | None]: + counts: Counter[str] = Counter() + unobserved = 0 + unknown_outcomes = 0 + for result in results: + finalize_report(result) + counts[result["status"]] += 1 + observed = result["coverage_status"] == "observed" + if observed: + active, dormant = result["active"], result["dormant"] + assert active is not None and dormant is not None + counts["active_invariants"] += active + counts["dormant_invariants"] += dormant + counts["identity_bearing_active"] += ( + bool(result["identity_clauses"]) and active + ) + else: + unobserved += 1 + unknown_outcomes += result["check_passed"] is None + counts["violated_invariants"] += result["check_passed"] is False + counts["identity_bearing_invariants"] += bool(result["identity_clauses"]) + summary: dict[str, int | None] = dict(counts) + summary["coverage_unobserved_invariants"] = unobserved + summary["check_outcome_unknown_invariants"] = unknown_outcomes + summary["observed_violated_invariants"] = counts["violated_invariants"] + summary["violated_invariants"] = ( + None if unknown_outcomes else counts["violated_invariants"] + ) + # A disabled/unsupported/error observer cannot establish the total active or + # dormant population. Keep the observed subset explicitly separate. + for name in ("active_invariants", "dormant_invariants", "identity_bearing_active"): + summary["observed_" + name] = counts[name] + summary[name] = None if unobserved else counts[name] + return summary + + +SUPPORTED = { + "APIContainRelation", + "ConsistentOutputRelation", + "ConsistentInputOutputRelation", + "ThresholdRelation", + "DistinctArgumentRelation", +} + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def observer_sites(function: Callable[..., object], family: str) -> ObserverSites: + """Locate exact predicate ASTs in the installed method, failing closed on drift.""" + lines, first = inspect.getsourcelines(function) + # Dedent code by its method indentation. A zero-indented line inside a + # multiline diagnostic string defeats textwrap.dedent's common prefix. + indent = len(lines[0]) - len(lines[0].lstrip()) + source = "".join( + line[indent:] if line.startswith(" " * indent) else line for line in lines + ) + tree = ast.parse(source) + starts: dict[int, str] = {} + ends: dict[int, int] = {} + visits: set[int] = set() + predicates: dict[str, Callable[[str], bool]] = { + "APIContainRelation": lambda s: "skip_var_unchanged_check" in s + and "found_expected_child_event" in s, + "ConsistentInputOutputRelation": lambda s: s == "input_value != output_value", + "ConsistentOutputRelation": lambda s: s == "len(returned_tensors) == 0", + "ThresholdRelation": lambda s: s + in ("output_value >= threshold_value", "output_value <= threshold_value"), + "DistinctArgumentRelation": lambda s: s + == "is_arguments_list_same(event1['args'], event2['args'])", + } + for node in ast.walk(tree): + if isinstance(node, ast.If) and predicates[family](ast.unparse(node.test)): + line = first + node.lineno - 1 + starts[line] = ast.unparse(node.test) + if node.test.end_lineno is None: + raise ObserverContractError("Predicate has no ending source line") + ends[line] = first + node.test.end_lineno - 1 + if isinstance(node, ast.Assign) and any( + isinstance(t, ast.Name) and t.id in {"func_call_event", "parent_event_type"} + for t in node.targets + ): + visits.add(first + node.lineno - 1) + expected = 2 if family in {"ThresholdRelation", "DistinctArgumentRelation"} else 1 + if len(starts) != expected: + raise ObserverContractError( + f"Unrecognized checker source for {family}: {starts}" + ) + return { + "starts": starts, + "ends": ends, + "visits": visits, + "method_source_sha256": hashlib.sha256(source.encode()).hexdigest(), + } + + +def candidate_counts(trace: TracePandas, inv: Invariant) -> CandidateCounts: + family = inv.relation.__name__ + api = ( + inv.params[1] + if family in {"ConsistentInputOutputRelation", "ThresholdRelation"} + else inv.params[0] + ) + if not isinstance(api, APIParam): + raise ObserverContractError(f"{family} expected an APIParam") + calls = trace.get_func_call_ids(api.api_full_name) + if family == "DistinctArgumentRelation": + _, groups = get_event_data_per_function_per_step(trace, {api.api_full_name}) + pairs = sum( + math.comb(sum(map(len, records.values())), 2) + for records in groups.get(api.api_full_name, {}).values() + ) + return { + "candidate_calls": len(calls), + "opportunities": pairs, + "opportunity_unit": "within_step_argument_pair", + "incomplete_or_exception_calls": None, + } + if family == "APIContainRelation": + eligible = len(calls) + else: + eligible = sum( + not isinstance( + trace.query_func_call_event(call), + (FuncCallExceptionEvent, IncompleteFuncCallEvent), + ) + for call in calls + ) + return { + "candidate_calls": len(calls), + "opportunities": eligible, + "opportunity_unit": ( + "parent_call" + if family == "APIContainRelation" + else "complete_function_call" + ), + "incomplete_or_exception_calls": len(calls) - eligible, + } + + +def id_clauses(value: JSONValue) -> list[JSONObject]: + found: list[JSONObject] = [] + if isinstance(value, dict): + if isinstance(value.get("prop_name"), str) and value.get("prop_name") in { + "process_id", + "thread_id", + "meta_vars.process_id", + "meta_vars.thread_id", + }: + found.append(value) + for child in value.values(): + found.extend(id_clauses(child)) + elif isinstance(value, list): + for child in value: + found.extend(id_clauses(child)) + return found + + +def observe( + trace: TracePandas, inv: Invariant, enabled: bool = True +) -> ObservationReport: + """Run one unchanged inv.check(False), counting only observed completed decisions.""" + family = inv.relation.__name__ + assert inv.precondition is not None, "Invariant precondition must be set" + report: ObservationReport = { + "relation": family, + "description": inv.text_description, + "identity_clauses": id_clauses( + json_value( + json.loads( + json.dumps(inv.precondition.to_dict(), cls=MDNONEJSONEncoder) + ) + ) + ), + "observer_supported": family in SUPPORTED, + "status": "unsupported_coverage", + "coverage_status": "disabled" if not enabled else "unsupported", + "coverage_observed": False, + "active": None, + "dormant": None, + "check_passed": None, + "checker_triggered": None, + } + if not enabled: + # Direct oracle: no opportunity pre-scan, source parsing, or monkeypatches. + try: + result = inv.check(trace, check_relation_first=False) + result.calc_and_set_time_precentage( + trace.get_start_time(), trace.get_end_time() + ) + report.update( + { + "status": "completed", + "checker_result": json_object( + json.loads(json.dumps(result.to_dict(), cls=MDNONEJSONEncoder)) + ), + "check_passed": boolean( + result.check_passed, "checker.check_passed" + ), + "checker_triggered": boolean(result.triggered, "checker.triggered"), + } + ) + except Exception as exc: + report.update( + { + "status": "checker_error", + "error": {"type": type(exc).__name__, "message": str(exc)}, + "check_passed": None, + } + ) + report.update({"coverage_observed": False, "active": None, "dormant": None}) + return finalize_report(report) + if family not in SUPPORTED: + return finalize_report(report) + candidates = candidate_counts(trace, inv) + report.update(candidates) + sites = observer_sites(inv.relation.static_check_all, family) + report["method_source_sha256"] = sites["method_source_sha256"] + counts: Counter[str] = Counter( + decisions_started=0, + decisions_completed=0, + visited_calls=0, + precondition_group_attempts=0, + precondition_group_passes=0, + precondition_group_failures=0, + precondition_group_errors=0, + ) + groups: dict[str, GroupCounts] = {} + decision_records: list[Decision] = [] + variable_precondition_requested = ( + family == "APIContainRelation" + and "var_events" in inv.precondition.get_group_names() + and len(inv.precondition.get_group("var_events")) > 0 + and not inv.precondition.is_group_unconditional("var_events") + ) + variable_check_skipped = False + method_code = inv.relation.static_check_all.__code__ + pending: PendingDecision | None = None + observer_failure: ObserverContractError | None = None + original_verify = GroupedPreconditions.verify + + def verify( + precondition: GroupedPreconditions, + records: list[object], + group_name: str, + trace_for_metavars: Trace | None = None, + ) -> bool: + group = groups.setdefault( + group_name, {"attempts": 0, "passes": 0, "failures": 0, "errors": 0} + ) + counts["precondition_group_attempts"] += 1 + group["attempts"] += 1 + try: + result = original_verify( + precondition, records, group_name, trace_for_metavars + ) + except Exception: + counts["precondition_group_errors"] += 1 + group["errors"] += 1 + raise + label: Literal["passes", "failures"] = "passes" if result else "failures" + group[label] += 1 + counts["precondition_group_" + label] += 1 + return result + + def trace_event_impl( + frame: FrameType, event: str, arg: object + ) -> TraceCallback | None: + nonlocal pending, variable_check_skipped + if frame.f_code is not method_code: + return None + if event == "exception": + # An extraction error can be caught by the real method; it is not a decision. + pending = None + if pending is not None: + done = event == "return" + if event == "line": + if family == "ConsistentOutputRelation": + done = frame.f_lineno in sites["visits"] + else: + done = not pending["start"] <= frame.f_lineno <= pending["end"] + if done: + counts["decisions_completed"] += 1 + decision_records.append(pending["record"]) + pending = None + if event == "line": + if frame.f_lineno in sites["visits"]: + counts["visited_calls"] += 1 + if frame.f_lineno in sites["starts"] and pending is None: + if variable_precondition_requested: + try: + skipped = boolean( + required(frame.f_locals, "skip_var_unchanged_check"), + "skip_var_unchanged_check", + ) + except ValueError as exc: + raise ObserverContractError(str(exc)) from exc + if skipped: + variable_check_skipped = True + record = decision_record(family, frame.f_locals) + counts["decisions_started"] += 1 + pending = { + "start": frame.f_lineno, + "end": sites["ends"][frame.f_lineno], + "record": record, + } + return trace_event + + def trace_event(frame: FrameType, event: str, arg: object) -> TraceCallback | None: + nonlocal observer_failure + try: + return trace_event_impl(frame, event, arg) + except ObserverContractError as exc: + # A checker may catch an exception raised by sys.settrace. Retain it + # independently so source drift cannot become a completed check. + observer_failure = exc + raise + + previous_trace = sys.gettrace() + if enabled and previous_trace is not None: + raise RuntimeError("A pre-existing Python trace observer would be overwritten") + try: + with patch.object(GroupedPreconditions, "verify", verify): + sys.settrace(trace_event) + result = inv.check(trace, check_relation_first=False) + if observer_failure is not None: + raise observer_failure + result.calc_and_set_time_precentage( + trace.get_start_time(), trace.get_end_time() + ) + report.update( + { + "status": "completed", + "checker_result": json_object( + json.loads(json.dumps(result.to_dict(), cls=MDNONEJSONEncoder)) + ), + "check_passed": boolean( + result.check_passed, "checker.check_passed" + ), + "checker_triggered": boolean(result.triggered, "checker.triggered"), + } + ) + except Exception as exc: + failure = observer_failure if observer_failure is not None else exc + report.update( + { + "status": ( + "observer_error" + if isinstance(failure, ObserverContractError) + else "checker_error" + ), + "error": {"type": type(failure).__name__, "message": str(failure)}, + "check_passed": None, + } + ) + if observer_failure is not None and exc is not observer_failure: + report["checker_error_after_observer"] = { + "type": type(exc).__name__, + "message": str(exc), + } + finally: + if enabled: + sys.settrace(previous_trace) + report["decisions_started"] = counts["decisions_started"] + report["decisions_completed"] = counts["decisions_completed"] + report["visited_calls"] = counts["visited_calls"] + report["precondition_group_attempts"] = counts["precondition_group_attempts"] + report["precondition_group_passes"] = counts["precondition_group_passes"] + report["precondition_group_failures"] = counts["precondition_group_failures"] + report["precondition_group_errors"] = counts["precondition_group_errors"] + report["precondition_groups"] = groups + report["decision_records"] = decision_records + report["decisions_not_completed"] = ( + counts["decisions_started"] - counts["decisions_completed"] + ) + stopped = report["status"] != "completed" or report["check_passed"] is False + if family == "DistinctArgumentRelation": + report["visited_calls"] = None + report["unvisited_calls_after_stop"] = None + report["unvisited_pairs_after_stop"] = ( + max(0, candidates["opportunities"] - counts["decisions_started"]) + if stopped + else 0 + ) + else: + report["unvisited_calls_after_stop"] = ( + max(0, candidates["candidate_calls"] - counts["visited_calls"]) + if stopped + else 0 + ) + report["active"] = counts["decisions_completed"] > 0 + report["dormant"] = report["status"] == "completed" and not report["active"] + report["coverage_observed"] = report["status"] != "observer_error" + report["coverage_status"] = "observed" if report["coverage_observed"] else "error" + if not report["coverage_observed"]: + report["active"] = None + report["dormant"] = None + report["variable_precondition_requested"] = variable_precondition_requested + report["variable_check_skipped"] = variable_check_skipped + report["fully_validated_active"] = ( + report["active"] is True + and not variable_check_skipped + and report["status"] == "completed" + ) + return finalize_report(report) + + +class CoverageOptions(argparse.Namespace): + traces: list[Path] + invariants: Path + output: Path + observer_off: bool + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--traces", nargs="+", type=Path, required=True) + parser.add_argument("--invariants", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--observer-off", action="store_true") + args = parser.parse_args(namespace=CoverageOptions()) + logging.basicConfig(level=logging.ERROR) + import traincheck.utils + + traincheck.utils._suppress_inner_progress = True + before = {str(p): sha256(p) for p in [*args.traces, args.invariants]} + trace = read_trace_file_Pandas(list(map(str, args.traces))) + invariants = read_inv_file(str(args.invariants)) + results: list[ObservationReport] = [] + for index, inv in enumerate(invariants): + result = observe(trace, inv, enabled=not args.observer_off) + result["invariant_index"] = index + results.append(result) + summary = summarize(results) + import traincheck.trace.trace_pandas as trace_module + + output: CoverageOutput = { + "schema_version": 2, + "pid": os.getpid(), + "check_relation_first": False, + "observer_enabled": not args.observer_off, + "runtime": { + "python": sys.version, + "executable": sys.executable, + "trace_module": str(Path(trace_module.__file__).resolve()), + "packages": { + p: importlib.metadata.version(p) for p in ("torch", "pandas", "numpy") + }, + }, + "harness_sha256": sha256(Path(__file__)), + "raw_process_ids": json_list(trace.get_process_ids(), "raw_process_ids"), + "raw_thread_ids": json_list(trace.get_thread_ids(), "raw_thread_ids"), + "trace_rows": len(trace.events), + "input_sha256": before, + "inputs_unchanged": all(sha256(Path(p)) == h for p, h in before.items()), + "summary": dict(summary), + "per_invariant": results, + } + args.output.write_text(json.dumps(output, cls=MDNONEJSONEncoder, indent=2) + "\n") + print(json.dumps(summary), flush=True) + return int(any(result["status"] != "completed" for result in results)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/fresh_validation/compare_study.py b/tools/fresh_validation/compare_study.py new file mode 100644 index 00000000..1822556c --- /dev/null +++ b/tools/fresh_validation/compare_study.py @@ -0,0 +1,369 @@ +"""Verify actual paired-study outputs; missing observation never becomes a pass.""" + +import argparse +import hashlib +import json +import math +import sys +from collections import Counter +from collections.abc import Mapping +from pathlib import Path +from typing import TypedDict + +if __package__ in (None, ""): + sys.path.append(str(Path(__file__).resolve().parents[2])) +from tools.fresh_validation.contracts import ( # noqa: E402 + JSONObject, + boolean, + integer, + json_object, + json_value, + mapping, + number, + read_object, + required, + sequence, + string, +) + + +class TrainingComparison(TypedDict): + loss_trajectory_equal: bool + initial_state_sha256_equal: bool + final_state_sha256_equal: bool + shapes_equal: bool + + +class StudyComparison(TypedDict): + status: str + invariants_sha256: str + training_comparisons: dict[str, TrainingComparison] + benchmark_data_runtime_source_and_harness_matched: bool + observer_matches_direct_checker: bool + per_run_summary: dict[str, JSONObject] + per_relation: dict[str, dict[str, dict[str, int]]] + raw_ids: dict[str, JSONObject] + fault: JSONObject + limitations: list[str] + + +def require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + +def canonical(value: object) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":")) + + +def object_field(data: Mapping[str, object], key: str) -> Mapping[str, object]: + return mapping(required(data, key), key) + + +def objects_field(data: Mapping[str, object], key: str) -> list[Mapping[str, object]]: + return [mapping(item, key) for item in sequence(required(data, key), key)] + + +def bool_field(data: Mapping[str, object], key: str) -> bool: + return boolean(required(data, key), key) + + +def int_field(data: Mapping[str, object], key: str) -> int: + return integer(required(data, key), key) + + +def str_field(data: Mapping[str, object], key: str) -> str: + return string(required(data, key), key) + + +def losses(metric: Mapping[str, object]) -> list[dict[str, int | float]]: + rows = [ + { + key: number(value, key) + for key, value in step.items() + if key.startswith("loss") + } + for step in objects_field(metric, "steps") + ] + require( + bool(rows) and all(row for row in rows), + "Every training step must contain a loss", + ) + require( + all(math.isfinite(value) for row in rows for value in row.values()), + "Non-finite loss", + ) + require( + int_field(metric, "completed_steps") == len(rows), + "Loss row count does not match completed_steps", + ) + return rows + + +def compare(root: Path) -> StudyComparison: + def read(name: str) -> JSONObject: + return read_object(root / name) + + phases = {p.parent.name: read_object(p) for p in root.glob("*/phase-result.json")} + expected_phases = { + "reference", + "control", + "holdout", + "baseline-holdout", + "inference", + "check-reference", + "check-control", + "check-holdout", + "check-holdout-direct", + "prepare-fault", + "check-fault", + } + require( + expected_phases <= phases.keys(), + f"Missing completed phases: {sorted(expected_phases - phases.keys())}", + ) + for name, phase in phases.items(): + require( + int_field(phase, "exit_code") == 0 + and required(phase, "stop_reason") is None + and bool_field(phase, "source_unchanged"), + f"Phase {name} did not complete with unchanged source", + ) + roles = ("reference", "control", "holdout", "baseline-holdout") + metrics = {role: read(role + "/metrics.json") for role in roles} + adaptations = {role: read(role + "/adaptation.json") for role in roles} + require( + all( + bool_field(m, "success") and bool_field(m, "state_shapes_unchanged") + for m in metrics.values() + ), + "Training failed or state shapes changed", + ) + workload_keys = ( + "benchmark_source_sha256", + "extracted_definitions_sha256", + "adapter_sha256", + "data", + "prepared_data_steps", + "model_config", + "batch_size", + "device", + "cpu_threads", + "precision", + "deterministic_algorithms", + "torch_version", + "python_version", + ) + for role in roles: + for key in workload_keys: + require( + required(adaptations[role], key) + == required(adaptations["reference"], key), + f"Mismatched {role} adaptation field: {key}", + ) + invocations = [read(name + "/invocation.json") for name in phases] + for key in ("source_sha256", "harness_sha256"): + require( + all( + object_field(i, key) == object_field(invocations[0], key) + for i in invocations + ), + f"Mismatched phase {key}", + ) + require( + len({int_field(adaptations[role], "process_id") for role in roles}) == 4, + "Training phases must use distinct processes", + ) + for role in roles: + require( + int_field(metrics[role], "completed_steps") + == (10 if role in ("reference", "control") else 20), + f"Incorrect completed step count for {role}", + ) + if role != "baseline-holdout": + require( + str_field(adaptations[role], "tracer_module") + == str( + Path(str_field(adaptations[role], "source_root")) + / "traincheck/instrumentor/tracer.py" + ), + f"Wrong tracer source for {role}", + ) + require( + not str_field(object_field(adaptations[role], "data"), "kind").startswith( + "synthetic" + ), + "Synthetic data is outside this study protocol", + ) + + paired: dict[str, TrainingComparison] = {} + for name, left, right in ( + ("same_seed_fresh_control", "reference", "control"), + ("holdout_instrumentation_comparison", "holdout", "baseline-holdout"), + ): + m1, m2 = metrics[left], metrics[right] + checks: TrainingComparison = { + "loss_trajectory_equal": losses(m1) == losses(m2), + "initial_state_sha256_equal": str_field(m1, "initial_model_state_sha256") + == str_field(m2, "initial_model_state_sha256"), + "final_state_sha256_equal": str_field(m1, "final_model_state_sha256") + == str_field(m2, "final_model_state_sha256"), + "shapes_equal": object_field(m1, "state_shapes") + == object_field(m2, "state_shapes"), + } + require(all(checks.values()), f"Training comparison failed: {name}: {checks}") + paired[name] = checks + coverage = { + role: read("check-" + role + "/coverage.json") + for role in ("reference", "control", "holdout", "holdout-direct", "fault") + } + require( + all(bool_field(c, "inputs_unchanged") for c in coverage.values()), + "Coverage inputs changed", + ) + frozen_path = root / "inference/invariants.jsonl" + frozen_hash = hashlib.sha256(frozen_path.read_bytes()).hexdigest() + for checked in coverage.values(): + require( + str_field(object_field(checked, "input_sha256"), str(frozen_path)) + == frozen_hash, + "Frozen invariants changed", + ) + observed = objects_field(coverage["holdout"], "per_invariant") + direct = objects_field(coverage["holdout-direct"], "per_invariant") + require(len(observed) == len(direct), "Observer/direct result counts differ") + for a, b in zip(observed, direct): + require( + str_field(a, "status") == str_field(b, "status") == "completed", + "Checker did not complete", + ) + require( + canonical(object_field(a, "checker_result")) + == canonical(object_field(b, "checker_result")), + "Observer changed checker result", + ) + require( + required(b, "active") is None + and required(b, "dormant") is None + and not bool_field(b, "coverage_observed"), + "Disabled observation must remain unknown", + ) + + mutation = read("fault/mutation.json") + selected = int_field(mutation, "invariant_index") + fault_results = objects_field(coverage["fault"], "per_invariant") + require( + 0 <= selected < len(observed) and selected < len(fault_results), + "Fault invariant index is out of range", + ) + healthy, faulty = observed[selected], fault_results[selected] + require( + bool_field(healthy, "check_passed") and bool_field(healthy, "active"), + "Fault target was not healthy and active", + ) + require( + str_field(faulty, "status") == "completed" + and not bool_field(faulty, "check_passed"), + "Fault did not cause a completed violation", + ) + require( + int_field(faulty, "decisions_completed") > 0 + and int_field(faulty, "precondition_group_passes") > 0, + "Fault target did not activate", + ) + fault_identity = str_field(mutation, "func_call_id") + # This selected rule is ConsistentOutputRelation; both records require a + # single call identity. A pair record or missing ID is a malformed report. + require( + any( + str_field(d, "func_call_id") == fault_identity + for d in objects_field(faulty, "decision_records") + ), + "Fault decision identity not observed", + ) + require( + any( + str_field(event, "func_call_id") == fault_identity + for event in objects_field(object_field(faulty, "checker_result"), "trace") + ), + "Fault identity absent from checker violation", + ) + edits = objects_field(mutation, "edits") + require( + bool_field(mutation, "ids_unchanged") + and bool_field(mutation, "precondition_inputs_unchanged") + and len(edits) == 1, + "Fault changed more than one permitted field", + ) + for key in ("raw_process_ids", "raw_thread_ids"): + require( + required(coverage["fault"], key) == required(coverage["holdout"], key), + f"Fault altered {key}", + ) + families: dict[str, dict[str, dict[str, int]]] = {} + for role in ("reference", "control", "holdout", "fault"): + family_counts: dict[str, Counter[str]] = {} + for item in objects_field(coverage[role], "per_invariant"): + require( + str_field(item, "status") == "completed" + and bool_field(item, "coverage_observed"), + "Incomplete or unobserved coverage", + ) + counts = family_counts.setdefault(str_field(item, "relation"), Counter()) + counts.update( + invariants=1, + active=int(bool_field(item, "active")), + dormant=int(bool_field(item, "dormant")), + violations=int(not bool_field(item, "check_passed")), + opportunities=int_field(item, "opportunities"), + precondition_group_passes=int_field(item, "precondition_group_passes"), + actual_decisions=int_field(item, "decisions_completed"), + proxy_checks_skipped=int(bool_field(item, "variable_check_skipped")), + fully_validated_active=int(bool_field(item, "fully_validated_active")), + ) + families[role] = {name: dict(counts) for name, counts in family_counts.items()} + output: StudyComparison = { + "status": "study_completed", + "invariants_sha256": frozen_hash, + "training_comparisons": paired, + "benchmark_data_runtime_source_and_harness_matched": True, + "observer_matches_direct_checker": True, + "per_run_summary": { + r: json_object(required(coverage[r], "summary")) + for r in ("reference", "control", "holdout", "fault") + }, + "per_relation": families, + "raw_ids": { + r: { + "process": json_value(required(coverage[r], "raw_process_ids")), + "thread": json_value(required(coverage[r], "raw_thread_ids")), + } + for r in ("reference", "control", "holdout") + }, + "fault": { + "kind": str_field(mutation, "kind"), + "invariant_index": selected, + "healthy_decisions": int_field(healthy, "decisions_completed"), + "fault_decisions": int_field(faulty, "decisions_completed"), + "fault_precondition_group_passes": int_field( + faulty, "precondition_group_passes" + ), + "violation": True, + "edit": json_object(edits[0]), + }, + "limitations": [ + "One reference seed and one holdout seed per workload; no generalization estimate.", + "The first half of holdout inputs overlaps the reference inputs.", + "Offline checking; no online checker, distributed training, or runtime overhead claim.", + "Fault control is post-hoc trace sensitivity, not a reproduced training bug.", + "Source-exact checker stops at the first violation per invariant.", + ], + } + (root / "comparison.json").write_text(json.dumps(output, indent=2) + "\n") + return output + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, required=True) + print(json.dumps(compare(parser.parse_args().root.resolve()), indent=2)) diff --git a/tools/fresh_validation/contracts.py b/tools/fresh_validation/contracts.py new file mode 100644 index 00000000..831ebe17 --- /dev/null +++ b/tools/fresh_validation/contracts.py @@ -0,0 +1,91 @@ +"""Validated JSON boundaries shared by the correctness-study tools. + +Unknown JSON and frame metadata enter as ``object``. These helpers never replace +required fields with defaults, and distinguish booleans from numeric counts. +""" + +import json +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import TypeAlias, cast + +JSONValue: TypeAlias = ( + None | bool | int | float | str | list["JSONValue"] | dict[str, "JSONValue"] +) +JSONObject: TypeAlias = dict[str, JSONValue] + + +def mapping(value: object, context: str) -> Mapping[str, object]: + if not isinstance(value, Mapping) or not all(isinstance(key, str) for key in value): + raise ValueError(f"{context} must be an object with string keys") + # The check establishes the key type; read-only Mapping does not promise + # anything about external values before the individual field validators run. + return cast(Mapping[str, object], value) + + +def sequence(value: object, context: str) -> Sequence[object]: + if not isinstance(value, list): + raise ValueError(f"{context} must be a JSON array") + return value + + +def string(value: object, context: str) -> str: + if not isinstance(value, str): + raise ValueError(f"{context} must be a string") + return value + + +def integer(value: object, context: str) -> int: + if not isinstance(value, int) or isinstance(value, bool): + raise ValueError(f"{context} must be an integer") + return value + + +def boolean(value: object, context: str) -> bool: + if not isinstance(value, bool): + raise ValueError(f"{context} must be a boolean") + return value + + +def number(value: object, context: str) -> int | float: + if not isinstance(value, (int, float)) or isinstance(value, bool): + raise ValueError(f"{context} must be a number") + return value + + +def required(data: Mapping[str, object], key: str) -> object: + if key not in data: + raise ValueError(f"Missing required field: {key}") + return data[key] + + +def json_value(value: object, context: str = "JSON") -> JSONValue: + if value is None or isinstance(value, (bool, int, float, str)): + return value + if isinstance(value, list): + return [json_value(child, context) for child in value] + if isinstance(value, Mapping): + return { + key: json_value(child, f"{context}.{key}") + for key, child in mapping(value, context).items() + } + raise ValueError(f"{context} has a non-JSON value of type {type(value).__name__}") + + +def json_object(value: object, context: str = "JSON") -> JSONObject: + result = json_value(value, context) + if not isinstance(result, dict): + raise ValueError(f"{context} must be an object") + return result + + +def json_list(value: object, context: str) -> list[JSONValue]: + result = json_value(value, context) + if not isinstance(result, list): + raise ValueError(f"{context} must be an array") + return result + + +def read_object(path: Path) -> JSONObject: + value: object = json.loads(path.read_text()) + return json_object(value, str(path)) diff --git a/tools/fresh_validation/fault_control.py b/tools/fresh_validation/fault_control.py new file mode 100644 index 00000000..66e64e34 --- /dev/null +++ b/tools/fresh_validation/fault_control.py @@ -0,0 +1,305 @@ +"""Post-hoc, one-field trace sensitivity control for a live frozen output invariant. + +This does not inject a training bug. It changes one recorded tensor shape or dtype after +healthy training, preserves IDs and precondition inputs, and checks the same rules. +""" + +import argparse +import copy +import hashlib +import json +import sys +from pathlib import Path +from typing import TypedDict + +if __package__ in (None, ""): + sys.path.append(str(Path(__file__).resolve().parents[2])) +from tools.fresh_validation.contracts import ( # noqa: E402 + JSONObject, + JSONValue, + boolean, + integer, + json_object, + mapping, + read_object, + required, + sequence, + string, +) + + +class Edit(TypedDict): + file: str + zero_based_line: int + path: list[str | int] + attribute: str + before_value: JSONValue + after_value: JSONValue + raw_process_id: JSONValue + raw_thread_id: JSONValue + + +class Fingerprints(TypedDict): + original_sha256: str + fault_sha256: str + + +class MutationReport(TypedDict): + kind: str + selection: str + attribute: str + invariant_index: int + invariant: JSONObject + func_call_id: str + edits: list[Edit] + traces: dict[str, Fingerprints] + invariants_sha256: str + healthy_coverage_sha256: str + precondition_inputs_unchanged: bool + ids_unchanged: bool + invariants_not_retrained_or_rewritten: bool + + +def digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def differences( + before: object, after: object, path: tuple[str | int, ...] = () +) -> list[tuple[str | int, ...]]: + if type(before) is not type(after): + return [path] + if isinstance(before, dict): + before_dict = mapping(before, "before") + after_dict = mapping(after, "after") + if before_dict.keys() != after_dict.keys(): + return [path] + return [ + p + for k in before_dict + for p in differences(before_dict[k], after_dict[k], (*path, k)) + ] + if isinstance(before, list): + assert isinstance(after, list) + if len(before) != len(after): + return [path] + return [ + p + for i, (a, b) in enumerate(zip(before, after)) + for p in differences(a, b, (*path, i)) + ] + if before == after or ( + isinstance(before, float) and before != before and after != after + ): + return [] + return [path] + + +def replacement(attributes: JSONValue, attribute: str) -> JSONValue: + """Return one valid same-type unequal metadata value, or no eligible edit.""" + if not isinstance(attributes, dict): + return None + value = attributes.get(attribute) + if attribute == "shape": + if ( + isinstance(value, list) + and value + and isinstance(value[0], int) + and not isinstance(value[0], bool) + ): + return [value[0] + 1, *value[1:]] + elif attribute == "dtype" and value in ("torch.float32", "torch.float64"): + return "torch.float64" if value == "torch.float32" else "torch.float32" + return None + + +def mutate( + traces: list[Path], invariants: Path, healthy: Path, output: Path +) -> MutationReport: + rules = [ + json_object(json.loads(line), "invariant") + for line in invariants.read_text().splitlines() + ] + coverage = read_object(healthy) + if not boolean( + required(coverage, "inputs_unchanged"), "inputs_unchanged" + ) or not boolean(required(coverage, "observer_enabled"), "observer_enabled"): + raise ValueError( + "Fault selection requires unchanged inputs and enabled observation" + ) + input_hashes = mapping(required(coverage, "input_sha256"), "input_sha256") + + def verify_hash(path: Path) -> None: + if string(required(input_hashes, str(path)), str(path)) != digest(path): + raise ValueError(f"Input changed since coverage collection: {path}") + + verify_hash(invariants) + chosen: JSONObject | None = None + call_id: str | None = None + selected_index: int | None = None + post_events: dict[str, list[JSONObject]] = {} + for trace in traces: + verify_hash(trace) + for line in trace.read_bytes().splitlines(): + event = json_object(json.loads(line), "trace event") + if string(required(event, "type"), "event.type") == "function_call (post)": + identity = string(required(event, "func_call_id"), "event.func_call_id") + post_events.setdefault(identity, []).append(event) + # Prefer shape; dtype is a prospective fallback when no live mutable shape + # rule exists. Selection remains post hoc and does not change learned rules. + for attribute in ("shape", "dtype"): + for raw_item in sequence(required(coverage, "per_invariant"), "per_invariant"): + item = json_object(raw_item, "per_invariant item") + index = integer(required(item, "invariant_index"), "invariant_index") + if not 0 <= index < len(rules): + raise ValueError(f"Invariant index out of range: {index}") + rule = rules[index] + if ( + string(required(rule, "relation"), "relation") + != "ConsistentOutputRelation" + ): + continue + params = sequence(required(rule, "params"), "params") + if len(params) != 2: + raise ValueError("ConsistentOutputRelation requires two parameters") + output_param = mapping(params[1], "output parameter") + if not ( + string(required(output_param, "attr_name"), "attr_name") == attribute + and boolean(required(item, "active"), "active") + and boolean(required(item, "check_passed"), "check_passed") + and integer( + required(item, "decisions_completed"), "decisions_completed" + ) + > 0 + ): + continue + for raw_decision in sequence( + required(item, "decision_records"), "decision_records" + ): + decision = mapping(raw_decision, "decision") + identity = string( + required(decision, "func_call_id"), "decision.func_call_id" + ) + events = post_events.get(identity, []) + if len(events) != 1: + continue + values = events[0]["return_values"] + tensors = values if isinstance(values, list) else [values] + eligible = any( + "Tensor" in name and replacement(attributes, attribute) is not None + for tensor in tensors + if isinstance(tensor, dict) + for name, attributes in tensor.items() + ) + if eligible: + chosen, call_id, selected_index = item, identity, index + break + if chosen is not None: + break + if chosen is not None: + break + if chosen is None: + raise ValueError("No healthy, actually evaluated mutable shape or dtype rule") + assert call_id is not None and selected_index is not None + rule = rules[selected_index] + output.mkdir(parents=True, exist_ok=False) + (output / "traces").mkdir() + edits: list[Edit] = [] + fingerprints: dict[str, Fingerprints] = {} + for trace in traces: + verify_hash(trace) + raw_lines = trace.read_bytes().splitlines(keepends=True) + transformed = list(raw_lines) + for index, raw in enumerate(raw_lines): + event = json_object(json.loads(raw), "trace event") + if ( + string(required(event, "type"), "event.type") != "function_call (post)" + or string(required(event, "func_call_id"), "event.func_call_id") + != call_id + ): + continue + before = copy.deepcopy(event) + values = event["return_values"] + tensor_values = values if isinstance(values, list) else [values] + for value in tensor_values: + if not isinstance(value, dict): + continue + for typename, attributes in value.items(): + if "Tensor" not in typename or not isinstance(attributes, dict): + continue + new_value = replacement(attributes, attribute) + if new_value is None: + continue + old = copy.deepcopy(attributes[attribute]) + attributes[attribute] = new_value + diff = differences(before, event) + assert len(diff) == 1 + assert diff[0][:1] == ("return_values",) + assert ( + before["process_id"] == event["process_id"] + and before["thread_id"] == event["thread_id"] + ) + edits.append( + { + "file": trace.name, + "zero_based_line": index, + "path": list(diff[0]), + "attribute": attribute, + "before_value": old, + "after_value": attributes[attribute], + "raw_process_id": event["process_id"], + "raw_thread_id": event["thread_id"], + } + ) + transformed[index] = ( + json.dumps(event, separators=(",", ":")).encode() + b"\n" + ) + break + if edits: + break + target = output / "traces" / trace.name + target.write_bytes(b"".join(transformed)) + assert all( + a == b + for i, (a, b) in enumerate(zip(raw_lines, transformed)) + if not any( + edit["file"] == trace.name and edit["zero_based_line"] == i + for edit in edits + ) + ) + fingerprints[trace.name] = { + "original_sha256": digest(trace), + "fault_sha256": digest(target), + } + assert len(edits) == 1, edits + manifest: MutationReport = { + "kind": "post_hoc_trace_field_sensitivity_control_not_training_fault", + "selection": "first actually evaluated healthy mutable shape rule, otherwise a float32/float64 dtype rule", + "attribute": attribute, + "invariant_index": selected_index, + "invariant": rule, + "func_call_id": call_id, + "edits": edits, + "traces": fingerprints, + "invariants_sha256": digest(invariants), + "healthy_coverage_sha256": digest(healthy), + "precondition_inputs_unchanged": True, + "ids_unchanged": True, + "invariants_not_retrained_or_rewritten": True, + } + (output / "mutation.json").write_text(json.dumps(manifest, indent=2) + "\n") + return manifest + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--traces", nargs="+", type=Path, required=True) + parser.add_argument("--invariants", type=Path, required=True) + parser.add_argument("--healthy-coverage", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + print( + json.dumps( + mutate(args.traces, args.invariants, args.healthy_coverage, args.output) + ) + ) diff --git a/tools/fresh_validation/prepare_source.py b/tools/fresh_validation/prepare_source.py new file mode 100644 index 00000000..39eedf6d --- /dev/null +++ b/tools/fresh_validation/prepare_source.py @@ -0,0 +1,120 @@ +"""Build the explicitly composed, disposable validation source without changing Git.""" + +import argparse +import hashlib +import io +import json +import subprocess +import tarfile +from pathlib import Path +from typing import TypedDict + + +class SourceManifest(TypedDict): + base: str + trace_integrity: str + torchscript: str + state_fix_hunks_from: str + changed_python_files: list[str] + dynamic_typename_fix: str + source_sha256: dict[str, str] + + +BASE = "99d16905296d83495570fdcafa91370b242d2528" +INTEGRITY = "7d6c683db92d83ffd6db9fc9c11fbc0ebdd2c5d4" +TORCHSCRIPT = "9ac61d1140cf8679c24442a174ec12f4de069980" +STATE = "1b5aedffbeefd4cdd9d30036dbb0a833521526cf" +TYPENAME = "2f034ae5d0fab7e792b7c0a5bdc6a1acfeb8962c" + + +def digest(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def build(repo: Path, destination: Path) -> SourceManifest: + def git(*args: str) -> bytes: + return subprocess.check_output(["git", "-C", str(repo), *args]) + + destination.mkdir(parents=True, exist_ok=False) + with tarfile.open( + fileobj=io.BytesIO(git("archive", BASE, "traincheck", "tests")) + ) as tar: + for member in tar: + if member.isfile(): + member_path = destination / member.name + member_path.parent.mkdir(parents=True, exist_ok=True) + extracted = tar.extractfile(member) + assert ( + extracted is not None + ), "Regular archive member must have contents" + member_path.write_bytes(extracted.read()) + old_hashes = { + str(p.relative_to(destination)): digest(p.read_bytes()) + for p in destination.rglob("*.py") + } + # These instrumentor files have no changes on the inference/cache stack. + for path in ( + "traincheck/instrumentor/tracer.py", + "traincheck/instrumentor/dumper.py", + ): + assert git("show", f"{INTEGRITY}^:{path}") == (destination / path).read_bytes() + assert ( + git("show", f"{TYPENAME}^:traincheck/utils.py") + == (destination / "traincheck/utils.py").read_bytes() + ) + replacements = { + "traincheck/instrumentor/tracer.py": TORCHSCRIPT, + "traincheck/instrumentor/dumper.py": INTEGRITY, + "traincheck/instrumentor/torchscript_compat.py": TORCHSCRIPT, + "tests/test_trace_integrity.py": INTEGRITY, + "tests/test_torchscript_compat.py": TORCHSCRIPT, + "tests/test_precondition_state_isolation.py": STATE, + "traincheck/utils.py": TYPENAME, + "tests/test_dynamic_typename.py": TYPENAME, + } + for path, commit in replacements.items(): + (destination / path).write_bytes(git("show", f"{commit}:{path}")) + source_path = destination / "traincheck/invariant/precondition.py" + source = source_path.read_text() + old = "_pruned_clauses: set[PreconditionClause] = set()," + new = "_pruned_clauses: set[PreconditionClause] | None = None," + assert source.count(old) == 1 + source = source.replace(old, new) + # Copy only the two approved state hunks, preserving the base RNG isolation. + state_source = git("show", f"{STATE}:traincheck/invariant/precondition.py").decode() + comment = " # Recursive children share pruning state; independent solves start fresh.\n" + insertion = ( + comment + " if _pruned_clauses is None:\n _pruned_clauses = set()\n" + ) + assert insertion in state_source + anchor = state_source.split(insertion)[1][:100] + assert source.count(anchor) == 1 + source = source.replace(anchor, insertion + anchor, 1) + assert "group_rng = random.Random(42)" in source + source_path.write_text(source) + hashes = { + str(p.relative_to(destination)): digest(p.read_bytes()) + for p in destination.rglob("*.py") + } + changed = sorted(p for p, h in hashes.items() if h != old_hashes.get(p)) + manifest: SourceManifest = { + "base": BASE, + "trace_integrity": INTEGRITY, + "torchscript": TORCHSCRIPT, + "state_fix_hunks_from": STATE, + "changed_python_files": changed, + "dynamic_typename_fix": TYPENAME, + "source_sha256": hashes, + } + (destination / "source-manifest.json").write_text( + json.dumps(manifest, indent=2) + "\n" + ) + return manifest + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + print(json.dumps(build(args.repo.resolve(), args.output.resolve()), indent=2)) diff --git a/tools/fresh_validation/run_phase.py b/tools/fresh_validation/run_phase.py new file mode 100644 index 00000000..9e52a598 --- /dev/null +++ b/tools/fresh_validation/run_phase.py @@ -0,0 +1,159 @@ +"""Run one bounded correctness phase in a fresh OS process (Linux only).""" + +import argparse +import hashlib +import json +import os +import signal +import subprocess +import time +from pathlib import Path +from typing import Literal, TypedDict + + +class PhaseOptions(argparse.Namespace): + source: Path + output: Path + timeout: float + memory_gib: float + trace_mib: float + command: list[str] + + +StopReason = Literal["process_group_rss_budget", "timeout", "trace_size_budget"] + + +class PhaseResult(TypedDict): + exit_code: int + stop_reason: StopReason | None + source_unchanged: bool + pid: int + elapsed_seconds_operational_only: float + peak_process_group_rss_bytes_operational_only: int + + +def group_memory(group: int) -> tuple[int, list[int]]: + total = 0 + processes: list[int] = [] + for entry in Path("/proc").iterdir(): + if not entry.name.isdigit(): + continue + try: + stat = (entry / "stat").read_text().rsplit(")", 1)[1].split() + if int(stat[2]) != group: + continue + rss = next( + int(line.split()[1]) * 1024 + for line in (entry / "status").read_text().splitlines() + if line.startswith("VmRSS:") + ) + total += rss + processes.append(int(entry.name)) + except (FileNotFoundError, ProcessLookupError, StopIteration): + continue + return total, processes + + +def run(args: PhaseOptions) -> int: + args.output.mkdir(parents=True, exist_ok=False) + source_hashes = { + str(p.relative_to(args.source)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in args.source.rglob("*.py") + } + env = os.environ.copy() + env.update( + PYTHONPATH=str(args.source.resolve()), + PYTHONHASHSEED="0", + PYTHONUNBUFFERED="1", + PYTHONDONTWRITEBYTECODE="1", + CUDA_VISIBLE_DEVICES="", + OMP_NUM_THREADS="1", + MKL_NUM_THREADS="1", + OPENBLAS_NUM_THREADS="1", + NUMEXPR_NUM_THREADS="1", + VECLIB_MAXIMUM_THREADS="1", + ) + command = args.command[1:] if args.command[0] == "--" else args.command + started, peak = time.monotonic(), 0 + reason: StopReason | None = None + with (args.output / "stdout.log").open("w") as log: + proc = subprocess.Popen( + ["nice", "-n", "10", *command], + cwd=args.output, + env=env, + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + invocation = { + "command": command, + "pid": proc.pid, + "native_threads": 1, + "timeout_seconds": args.timeout, + "memory_limit_gib": args.memory_gib, + "trace_limit_mib": args.trace_mib, + "source_sha256": source_hashes, + "harness_sha256": { + p.name: hashlib.sha256(p.read_bytes()).hexdigest() + for p in Path(__file__).resolve().parent.glob("*.py") + }, + } + (args.output / "invocation.json").write_text( + json.dumps(invocation, indent=2) + "\n" + ) + try: + while proc.poll() is None: + memory, _ = group_memory(proc.pid) + peak = max(peak, memory) + if memory > args.memory_gib * 1024**3: + reason = "process_group_rss_budget" + if time.monotonic() - started > args.timeout: + reason = "timeout" + if ( + sum(p.stat().st_size for p in args.output.rglob("trace_API*.log")) + > args.trace_mib * 1024**2 + ): + reason = "trace_size_budget" + if reason: + os.killpg(proc.pid, signal.SIGTERM) + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + break + time.sleep(0.25) + except BaseException: + os.killpg(proc.pid, signal.SIGTERM) + proc.wait(timeout=10) + raise + unchanged = all( + hashlib.sha256((args.source / p).read_bytes()).hexdigest() == h + for p, h in source_hashes.items() + ) + assert proc.returncode is not None, "Supervisor must reap its child" + result: PhaseResult = { + "exit_code": proc.returncode, + "stop_reason": reason, + "source_unchanged": unchanged, + "pid": proc.pid, + "elapsed_seconds_operational_only": time.monotonic() - started, + "peak_process_group_rss_bytes_operational_only": peak, + } + (args.output / "phase-result.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result), flush=True) + return int(proc.returncode != 0 or reason is not None or not unchanged) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--timeout", type=float, default=600) + parser.add_argument("--memory-gib", type=float, default=8) + parser.add_argument("--trace-mib", type=float, default=256) + parser.add_argument("command", nargs=argparse.REMAINDER) + options = parser.parse_args(namespace=PhaseOptions()) + if not options.command: + parser.error("A command is required") + raise SystemExit(run(options)) diff --git a/tools/fresh_validation/run_study.py b/tools/fresh_validation/run_study.py new file mode 100644 index 00000000..0cbd0f48 --- /dev/null +++ b/tools/fresh_validation/run_study.py @@ -0,0 +1,197 @@ +"""Collect reference/control/holdout traces and check frozen same-workload rules. + +Every training, inference, and checking phase starts a separate bounded process. +Only the reference is used to infer rules. This is a correctness/coverage study; +the supervisor's time and memory observations are operational limits only. +""" + +import argparse +import hashlib +import json +import subprocess +from collections.abc import Sequence +from pathlib import Path +from typing import Literal + + +class StudyOptions(argparse.Namespace): + source: Path + python: Path + benchmark_root: Path + data_root: Path | None + output: Path + workload: Literal["mnist", "transformer"] + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--python", type=Path, required=True) + parser.add_argument("--benchmark-root", type=Path, required=True) + parser.add_argument("--data-root", type=Path) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--workload", choices=("mnist", "transformer"), required=True) + args = parser.parse_args(namespace=StudyOptions()) + if args.workload == "mnist" and args.data_root is None: + parser.error("MNIST requires an existing real IDX data cache") + args.output = args.output.resolve() + args.output.mkdir(parents=True, exist_ok=False) + harness = Path(__file__).resolve().parent + hashes = { + p.name: hashlib.sha256(p.read_bytes()).hexdigest() for p in harness.glob("*.py") + } + (args.output / "study-plan.json").write_text( + json.dumps( + { + "workload": args.workload, + "reference_steps": 10, + "holdout_steps": 20, + "reference_seed": 42, + "holdout_seed": 43, + "data_steps": 20, + "reference_inputs_overlap_first_half_of_holdout": True, + "disabled_relations": ["FunctionLeadRelation", "FunctionCoverRelation"], + "harness_sha256": hashes, + }, + indent=2, + ) + + "\n" + ) + + def phase(name: str, command: Sequence[str | Path | int]) -> None: + subprocess.run( + [ + str(args.python), + str(harness / "run_phase.py"), + "--source", + str(args.source.resolve()), + "--output", + str(args.output / name), + "--", + str(args.python), + *map(str, command), + ], + check=True, + ) + + for role, steps, seed, mode in ( + ("reference", 10, 42, "instrumented"), + ("control", 10, 42, "instrumented"), + ("holdout", 20, 43, "instrumented"), + ("baseline-holdout", 20, 43, "baseline"), + ): + command: list[str | Path | int] = [ + harness / "benchmark_workload.py", + "--benchmark-root", + args.benchmark_root.resolve(), + "--workload", + args.workload, + "--steps", + steps, + "--data-steps", + 20, + "--seed", + seed, + "--batch-size", + 2, + "--device", + "cpu", + "--mode", + mode, + "--output", + args.output / role, + ] + if args.data_root is not None: + command.extend(["--data-root", args.data_root.resolve()]) + phase(role, command) + invariants = args.output / "inference/invariants.jsonl" + traces = sorted((args.output / "reference/traces").glob("trace_API*.log")) + if not traces: + raise RuntimeError("No reference API traces") + phase( + "inference", + [ + "-m", + "traincheck.infer_engine", + "-t", + *traces, + "-o", + invariants, + "--backend", + "pandas", + "-j", + 1, + "--disable-relation", + "FunctionLeadRelation", + "FunctionCoverRelation", + ], + ) + for role in ("reference", "control", "holdout"): + traces = sorted((args.output / role / "traces").glob("trace_API*.log")) + phase( + "check-" + role, + [ + harness / "check_coverage.py", + "--traces", + *traces, + "--invariants", + invariants, + "--output", + args.output / ("check-" + role) / "coverage.json", + ], + ) + traces = sorted((args.output / "holdout/traces").glob("trace_API*.log")) + phase( + "check-holdout-direct", + [ + harness / "check_coverage.py", + "--traces", + *traces, + "--invariants", + invariants, + "--observer-off", + "--output", + args.output / "check-holdout-direct/coverage.json", + ], + ) + + phase( + "prepare-fault", + [ + harness / "fault_control.py", + "--traces", + *traces, + "--invariants", + invariants, + "--healthy-coverage", + args.output / "check-holdout/coverage.json", + "--output", + args.output / "fault", + ], + ) + faulty_traces = sorted((args.output / "fault/traces").glob("trace_API*.log")) + phase( + "check-fault", + [ + harness / "check_coverage.py", + "--traces", + *faulty_traces, + "--invariants", + invariants, + "--output", + args.output / "check-fault/coverage.json", + ], + ) + subprocess.run( + [ + str(args.python), + str(harness / "compare_study.py"), + "--root", + str(args.output), + ], + check=True, + ) + + +if __name__ == "__main__": + main() diff --git a/traincheck/infer_engine.py b/traincheck/infer_engine.py index d98182a5..4e48a4c3 100644 --- a/traincheck/infer_engine.py +++ b/traincheck/infer_engine.py @@ -5,6 +5,7 @@ import os import time from concurrent.futures import ProcessPoolExecutor, as_completed +from random import Random from tqdm import tqdm @@ -132,14 +133,19 @@ def generate_hypothesis(self) -> dict[Hypothesis, list[int]]: active_relations = [ r for r in relation_pool if r not in self.disabled_relations ] + # One stream for the fixed parent merge order, independent of reservoir + # construction and stage merging. Never reset it per hypothesis. + merge_rng = Random( + f"traincheck-example-merge-v1:cross-trace:{config.EXAMPLE_SAMPLING_SEED!r}" + ) if self.num_workers <= 1: self._generate_hypothesis_serial( - active_relations, hypotheses_and_trace_idxs, hypo_lookup + active_relations, hypotheses_and_trace_idxs, hypo_lookup, merge_rng ) else: self._generate_hypothesis_parallel( - active_relations, hypotheses_and_trace_idxs, hypo_lookup + active_relations, hypotheses_and_trace_idxs, hypo_lookup, merge_rng ) total = len(hypotheses_and_trace_idxs) @@ -153,6 +159,7 @@ def _merge_hypotheses( trace_idx: int, hypotheses_and_trace_idxs: dict[Hypothesis, list[int]], hypo_lookup: dict[Hypothesis, Hypothesis], + merge_rng: Random, ) -> None: """Merge one trace's inferred hypotheses into the running collection. @@ -166,31 +173,19 @@ def _merge_hypotheses( else: hypotheses_and_trace_idxs[hypo].append(trace_idx) original_hypo = hypo_lookup[hypo] - orig_num_pos_exps = len(original_hypo.positive_examples) - orig_num_neg_exps = len(original_hypo.negative_examples) - original_hypo.positive_examples.examples.extend( - hypo.positive_examples.examples + original_hypo.positive_examples.merge( + hypo.positive_examples, rng=merge_rng ) - original_hypo.negative_examples.examples.extend( - hypo.negative_examples.examples + original_hypo.negative_examples.merge( + hypo.negative_examples, rng=merge_rng ) - assert len( - hypo_lookup[hypo].positive_examples - ) == orig_num_pos_exps + len( - hypo.positive_examples - ), f"Expected {orig_num_pos_exps} + {len(hypo.positive_examples)} positive examples, got {len(hypo_lookup[hypo].positive_examples)}" - assert len( - hypo_lookup[hypo].negative_examples - ) == orig_num_neg_exps + len( - hypo.negative_examples - ), f"Expected {orig_num_neg_exps} + {len(hypo.negative_examples)} negative examples, got {len(hypo_lookup[hypo].negative_examples)}" - def _generate_hypothesis_serial( self, active_relations: list[type[Relation]], hypotheses_and_trace_idxs: dict[Hypothesis, list[int]], hypo_lookup: dict[Hypothesis, Hypothesis], + merge_rng: Random, ) -> None: """Serial generation with the same per-task sampling as the worker path.""" n_traces = len(self.traces) @@ -225,6 +220,7 @@ def _generate_hypothesis_serial( trace_idx, hypotheses_and_trace_idxs, hypo_lookup, + merge_rng, ) def _generate_hypothesis_parallel( @@ -232,6 +228,7 @@ def _generate_hypothesis_parallel( active_relations: list[type[Relation]], hypotheses_and_trace_idxs: dict[Hypothesis, list[int]], hypo_lookup: dict[Hypothesis, Hypothesis], + merge_rng: Random, ) -> None: """Parallel hypothesis generation. @@ -284,6 +281,7 @@ def _generate_hypothesis_parallel( trace_idx, hypotheses_and_trace_idxs, hypo_lookup, + merge_rng, ) def collect_examples(self, hypotheses: dict[Hypothesis, list[int]]): diff --git a/traincheck/instrumentor/dumper.py b/traincheck/instrumentor/dumper.py index b808e986..5a852d33 100644 --- a/traincheck/instrumentor/dumper.py +++ b/traincheck/instrumentor/dumper.py @@ -4,7 +4,7 @@ import threading import time from queue import Empty, Queue -from typing import Hashable +from typing import Callable, Hashable import orjson import torch @@ -181,43 +181,53 @@ def get_trace_VAR_dumper_queue(): class TraceBuffer: """Buffer traces and flush them to the trace log file periodically or when the buffer is full. - Caveat: **This class must be instantiated PER THREAD*, as the buffer is not thread-safe. + Caveat: **This class must be instantiated PER THREAD*, to preserve per-thread trace routing. Also, the queue must be created at the same time as the buffer (as did in the __init__ method). Previously, the queue was dynamic getted from the queue_getter method but it caused a mysterious bug, where a trace file contains traces of different thread ids and process ids. """ def __init__( - self, queue_getter, buffer_size=BUFFER_SIZE, flush_interval=FLUSH_INTERVAL - ): + self, + queue_getter: Callable[[], Queue[str]], + buffer_size: int = BUFFER_SIZE, + flush_interval: float = FLUSH_INTERVAL, + ) -> None: self.queue = queue_getter() self.buffer_size = buffer_size self.flush_interval = flush_interval - self.buffer = [] + self.buffer: list[str] = [] + self._lock = threading.Lock() self.last_flush_time = time.time() self.flush_thread = threading.Thread(target=self._flush_periodically) self.flush_thread.daemon = True self.flush_thread.start() - def add_trace(self, trace): - self.buffer.append( - serialize(trace) - ) # TODO: serialization step cannot be buffered rn as trace dicts might get modified - if ( - len(self.buffer) >= self.buffer_size - or (time.time() - self.last_flush_time) >= self.flush_interval - ): - self._flush() + def add_trace(self, trace: dict[str, object]) -> None: + # Snapshot mutable values in the producer, before returning to the caller. + serialized = serialize(trace) + with self._lock: + self.buffer.append(serialized) + if ( + len(self.buffer) >= self.buffer_size + or (time.time() - self.last_flush_time) >= self.flush_interval + ): + self._flush_locked() + + def _flush(self) -> None: + with self._lock: + self._flush_locked() - def _flush(self): + def _flush_locked(self) -> None: if not self.buffer: return - # serialize all traces in the buffer + # Keep enqueue and clear atomic with respect to the producer and timer. + # Enqueue under the lock also preserves batch ordering. self.queue.put("\n".join(self.buffer)) self.buffer.clear() self.last_flush_time = time.time() - def _flush_periodically(self): + def _flush_periodically(self) -> None: while not stop_event.is_set(): time.sleep(self.flush_interval) self._flush() diff --git a/traincheck/instrumentor/torchscript_compat.py b/traincheck/instrumentor/torchscript_compat.py new file mode 100644 index 00000000..4f35f8c8 --- /dev/null +++ b/traincheck/instrumentor/torchscript_compat.py @@ -0,0 +1,87 @@ +"""Preserve TorchScript operator identity for instrumented builtin functions.""" + +import threading +import warnings +from typing import Callable, Literal, TypedDict + + +class PreservedBuiltin(TypedDict): + status: Literal["preserved"] + operator: str + + +class UnregisteredBuiltin(TypedDict): + status: Literal["not-registered"] + operator: None + + +class UnsupportedBuiltinAPI(TypedDict): + status: Literal["unsupported-private-api"] + operator: None + + +BuiltinPreservation = PreservedBuiltin | UnregisteredBuiltin | UnsupportedBuiltinAPI + +# TorchScript indexes by id only. Keep registered wrappers alive, like PyTorch's +# own _builtin_ops list, so a later unrelated object cannot reuse a stale ID. +# These callables are retained/passed through, never invoked by this module. +_REGISTERED_WRAPPERS: dict[Callable[..., object], str] = {} +_MISSING = object() +_UNSUPPORTED_WARNING_LOCK = threading.Lock() +_UNSUPPORTED_WARNING_EMITTED = False + + +def warn_unsupported_torchscript() -> None: + """Report missing private support once per process, including always filters.""" + global _UNSUPPORTED_WARNING_EMITTED + with _UNSUPPORTED_WARNING_LOCK: + if not _UNSUPPORTED_WARNING_EMITTED: + warnings.warn( + "TrainCheck cannot preserve TorchScript builtin identity: this " + "PyTorch version lacks the private builtin-registration API. " + "Eager instrumentation remains available.", + RuntimeWarning, + stacklevel=2, + ) + _UNSUPPORTED_WARNING_EMITTED = True + + +def preserve_torchscript_builtin( + original: Callable[..., object], + wrapped: Callable[..., object], + torch_module: object, +) -> BuiltinPreservation: + """Preserve a builtin mapping before publishing wrapped into its module. + + TorchScript compiles the existing operator directly and does not run Python + recorder hooks inside its graph. Eager calls still execute the wrapper. + Missing private APIs are unsupported; present malformed APIs or metadata + are errors. Exceptions from framework callbacks propagate unchanged. + """ + # Missing capabilities are expected across PyTorch versions. A sentinel + # distinguishes absence from present-but-invalid None callback attributes. + jit: object = getattr(torch_module, "jit", _MISSING) + builtins: object = getattr(jit, "_builtins", _MISSING) + find_builtin: object = getattr(builtins, "_find_builtin", _MISSING) + register_builtin: object = getattr(builtins, "_register_builtin", _MISSING) + for name, callback in ( + ("_find_builtin", find_builtin), + ("_register_builtin", register_builtin), + ): + if callback is not _MISSING and not callable(callback): + raise TypeError(f"torch.jit._builtins.{name} must be callable") + if find_builtin is _MISSING or register_builtin is _MISSING: + return {"status": "unsupported-private-api", "operator": None} + # These checks narrow dynamic framework attributes after the missing case. + assert callable(find_builtin) and callable(register_builtin) + # _find_builtin also initializes the lazy table before the first replacement. + operator: object = find_builtin(original) + if operator is None: + return {"status": "not-registered", "operator": None} + if not isinstance(operator, str) or not operator: + raise TypeError( + "TorchScript builtin operator must be a nonempty string or None" + ) + register_builtin(wrapped, operator) + _REGISTERED_WRAPPERS[wrapped] = operator + return {"status": "preserved", "operator": operator} diff --git a/traincheck/instrumentor/tracer.py b/traincheck/instrumentor/tracer.py index 2808ba30..10bb81ab 100644 --- a/traincheck/instrumentor/tracer.py +++ b/traincheck/instrumentor/tracer.py @@ -8,8 +8,9 @@ import time import traceback import types +from contextlib import ContextDecorator from importlib.util import find_spec -from typing import Callable, Optional +from typing import Callable, Literal, Optional import torch @@ -38,6 +39,10 @@ funcs_to_be_replaced, is_funcs_to_be_unproxied, ) +from traincheck.instrumentor.torchscript_compat import ( + preserve_torchscript_builtin, + warn_unsupported_torchscript, +) from traincheck.utils import get_timestamp_ns, get_unique_id, typename _instancemethod_t = type(torch._C._distributed_c10d.ProcessGroup.broadcast) @@ -46,8 +51,6 @@ IS_INSTRUMENTING = False -DISABLE_WRAPPER = False - # for prompt generation tasks using the transformers library (see traincheck/developer/instr_stage_annotation.py:annotate_answer_start_token_ids) GENERATE_START_TOKEN_ID: None | int = None GENERATE_START_TOKEN_ID_INCLUDE_START_TOKEN = False @@ -55,7 +58,58 @@ COLLECT_OVERHEAD_METRICS = os.environ.get("COLLECT_OVERHEAD_METRICS", "0") == "1" -THREAD_DATA = threading.local() +class _TracingThreadState(threading.local): + # IDs remain lazy, matching get_process_thread_id's existing behavior. + thread_id: int + process_id: int + + def __init__(self) -> None: + self.suppression_depth: int = 0 + + +THREAD_DATA = _TracingThreadState() + + +class suppress_tracing(ContextDecorator): + """Suppress recorder-owned calls independently per thread, exception-safely. + + Use a class rather than a generator context manager: Python 3.11+ contextlib + assigns exception.__traceback__, which breaks frozen framework exceptions. + Never hold this scope over the original application call. + """ + + depth: int + + def _recreate_cm(self) -> "suppress_tracing": + # Decorated conversions may nest or run concurrently on different threads. + return type(self)() + + def __enter__(self) -> "suppress_tracing": + self.depth = THREAD_DATA.suppression_depth + THREAD_DATA.suppression_depth = self.depth + 1 + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: types.TracebackType | None, + ) -> Literal[False]: + THREAD_DATA.suppression_depth = self.depth + return False + + +class application_tracing(suppress_tracing): + """Restore application tracing while advancing a user-supplied generator.""" + + def __enter__(self) -> "application_tracing": + super().__enter__() + THREAD_DATA.suppression_depth = 0 + return self + + +def tracing_suppressed() -> bool: + return THREAD_DATA.suppression_depth > 0 class TraceLineType: @@ -103,9 +157,8 @@ def get_owner_class(func): return owner +@suppress_tracing() def to_dict_args_kwargs(args, kwargs, dump_args_config=None) -> dict: - global DISABLE_WRAPPER - DISABLE_WRAPPER = True if dump_args_config is None: result = { "args": {i: var_to_serializable(arg) for i, arg in enumerate(args)}, @@ -125,20 +178,17 @@ def to_dict_args_kwargs(args, kwargs, dump_args_config=None) -> dict: result["args"] = args_dicts result["kwargs"] = kwargs_dicts - DISABLE_WRAPPER = False return result +@suppress_tracing() def to_dict_return_value(result) -> dict | list[dict]: - global DISABLE_WRAPPER - DISABLE_WRAPPER = True result_dict: dict | list[dict] if isinstance(result, tuple): result_dict = [var_to_serializable(r) for r in result] else: result_dict = var_to_serializable(result) - DISABLE_WRAPPER = False return result_dict @@ -153,7 +203,7 @@ def function_wrapper( dump_ret: bool, dump_ret_config, trigger_var_dump: bool, - var_dump_config: dict, + var_dump_config: dict[str, object] | None, need_unproxy_args_kwargs: bool, *args, **kwargs, @@ -167,7 +217,7 @@ def function_wrapper( global PROCESS_ID - if config.DISABLE_WRAPPER: + if config.DISABLE_WRAPPER or tracing_suppressed(): # TODO: all meta vars update should be done outside the function_wrapper (e.g. step increment) by applying a separate wrapper return original_function(*args, **kwargs) @@ -179,79 +229,81 @@ def function_wrapper( # tracing here and just delegate to the original function. return original_function(*args, **kwargs) - if COLLECT_OVERHEAD_METRICS: - ENTER_PERF_TIME = time.perf_counter() - - func_call_id = get_unique_id() - process_id, thread_id = get_process_thread_id() - - pre_meta_vars = get_meta_vars() - if IS_INSTRUMENTING: # during instrumentation, skip the dumping to avoid infinite recursion # and interference with the import system return original_function(*args, **kwargs) - pre_record = { - "func_call_id": func_call_id, - "thread_id": thread_id, - "process_id": process_id, - "meta_vars": pre_meta_vars, - "type": TraceLineType.FUNC_CALL_PRE, - "function": original_function_name, - "is_bound_method": is_bound_method, - "obj_id": None if not is_bound_method else id(args[0]), - } - - if dump_stack_trace: - pre_record["stack_trace"] = traceback.format_stack() - - if scan_proxy_in_args: - proxy_in_args = [] - - def find_proxy_in_args(args): - for i, arg in enumerate(args): - if is_proxied(arg) or is_proxyparameter(arg): - proxy_in_args.append(arg) - elif type(arg) in [list, tuple]: - find_proxy_in_args(arg) - elif isinstance(arg, types.GeneratorType) and not isinstance( - arg, tuple - ): - arg_list = list(arg) - args[i] = iter(arg_list) - find_proxy_in_args(arg_list) - - args = list(args) # type: ignore[assignment] - find_proxy_in_args(args) - args = tuple(args) - - if proxy_in_args: - if "proxy_obj_names" not in pre_record: - pre_record["proxy_obj_names"] = [] - for proxy in proxy_in_args: - if is_proxyparameter(proxy): - pre_record["proxy_obj_names"].append( - [proxy.__dict__["var_name"], "Parameter"] - ) - else: - pre_record["proxy_obj_names"].append( - [proxy.__dict__["var_name"], type(proxy._obj).__name__] - ) - if dump_args: - dict_args_kwargs = to_dict_args_kwargs(args, kwargs, dump_args_config) - pre_record["args"] = dict_args_kwargs["args"] - pre_record["kwargs"] = dict_args_kwargs["kwargs"] - dump_trace_API(pre_record) - - if trigger_var_dump: - """Mimicking the behavior the observer wrapper: pre-observe""" - get_global_registry().dump_modified( - dump_loc=original_function_name, dump_config=var_dump_config - ) + with suppress_tracing(): + if COLLECT_OVERHEAD_METRICS: + ENTER_PERF_TIME = time.perf_counter() + + func_call_id = get_unique_id() + process_id, thread_id = get_process_thread_id() + + pre_meta_vars = get_meta_vars() + + pre_record = { + "func_call_id": func_call_id, + "thread_id": thread_id, + "process_id": process_id, + "meta_vars": pre_meta_vars, + "type": TraceLineType.FUNC_CALL_PRE, + "function": original_function_name, + "is_bound_method": is_bound_method, + "obj_id": None if not is_bound_method else id(args[0]), + } - if need_unproxy_args_kwargs: - args, kwargs = unproxy_args_kwargs(args, kwargs) + if dump_stack_trace: + pre_record["stack_trace"] = traceback.format_stack() + + if scan_proxy_in_args: + proxy_in_args = [] + + def find_proxy_in_args(args): + for i, arg in enumerate(args): + if is_proxied(arg) or is_proxyparameter(arg): + proxy_in_args.append(arg) + elif type(arg) in [list, tuple]: + find_proxy_in_args(arg) + elif isinstance(arg, types.GeneratorType) and not isinstance( + arg, tuple + ): + with application_tracing(): + arg_list = list(arg) + args[i] = iter(arg_list) + find_proxy_in_args(arg_list) + + args = list(args) # type: ignore[assignment] + find_proxy_in_args(args) + args = tuple(args) + + if proxy_in_args: + if "proxy_obj_names" not in pre_record: + pre_record["proxy_obj_names"] = [] + for proxy in proxy_in_args: + if is_proxyparameter(proxy): + pre_record["proxy_obj_names"].append( + [proxy.__dict__["var_name"], "Parameter"] + ) + else: + pre_record["proxy_obj_names"].append( + [proxy.__dict__["var_name"], type(proxy._obj).__name__] + ) + if dump_args: + dict_args_kwargs = to_dict_args_kwargs(args, kwargs, dump_args_config) + pre_record["args"] = dict_args_kwargs["args"] + pre_record["kwargs"] = dict_args_kwargs["kwargs"] + dump_trace_API(pre_record) + + if trigger_var_dump: + """Mimicking the behavior the observer wrapper: pre-observe""" + get_global_registry().dump_modified( + dump_loc=original_function_name, dump_config=var_dump_config + ) + + if need_unproxy_args_kwargs: + args, kwargs = unproxy_args_kwargs(args, kwargs) try: if COLLECT_OVERHEAD_METRICS: @@ -260,151 +312,155 @@ def find_proxy_in_args(args): if COLLECT_OVERHEAD_METRICS: ORIG_EXIT_PERF_TIME = time.perf_counter() except Exception as e: - if COLLECT_OVERHEAD_METRICS: - ORIG_EXIT_PERF_TIME = time.perf_counter() + with suppress_tracing(): + if COLLECT_OVERHEAD_METRICS: + ORIG_EXIT_PERF_TIME = time.perf_counter() + + if trigger_var_dump: + """Mimicking the behavior the observer wrapper: post-observe""" + get_global_registry().dump_modified( + dump_loc=original_function_name, dump_config=var_dump_config + ) + dump_trace_API( + { + "func_call_id": func_call_id, + "thread_id": thread_id, + "process_id": process_id, + "meta_vars": pre_meta_vars, + "type": TraceLineType.FUNC_CALL_POST_EXCEPTION, + "function": original_function_name, + "exception": typename(e, is_runtime=True), + "exception_msg": str(e), + "is_bound_method": is_bound_method, + "obj_id": None if not is_bound_method else id(args[0]), + }, + ) + + if COLLECT_OVERHEAD_METRICS: + EXIT_PERF_TIME = time.perf_counter() + print( + f"WRAPPER TIME: {original_function_name},{ORIG_EXIT_PERF_TIME - ORIG_ENTER_PERF_TIME},{EXIT_PERF_TIME - ENTER_PERF_TIME}" + ) + raise e + + with suppress_tracing(): if trigger_var_dump: - """Mimicking the behavior the observer wrapper: post-observe""" get_global_registry().dump_modified( dump_loc=original_function_name, dump_config=var_dump_config ) - dump_trace_API( - { - "func_call_id": func_call_id, - "thread_id": thread_id, - "process_id": process_id, - "meta_vars": pre_meta_vars, - "type": TraceLineType.FUNC_CALL_POST_EXCEPTION, - "function": original_function_name, - "exception": typename(e, is_runtime=True), - "exception_msg": str(e), - "is_bound_method": is_bound_method, - "obj_id": None if not is_bound_method else id(args[0]), - }, - ) + post_record = { + "func_call_id": func_call_id, + "thread_id": thread_id, + "process_id": process_id, + "meta_vars": pre_meta_vars, + "type": TraceLineType.FUNC_CALL_POST, + "function": original_function_name, + "is_bound_method": is_bound_method, + "obj_id": None if not is_bound_method else id(args[0]), + } + + result_to_dump = result + + # if the current function name is transformers.generate, then we will dump the response tokens only, let's see. + # a concrete name: "transformers.models.whisper.modeling_whisper.WhisperForConditionalGeneration.generate" + # we want a pattern that abstracts the specific model name + pattern = "transformers.models.*.generate" + # find matches in the pattern + import re + + if ( + GENERATE_START_TOKEN_ID is not None + and re.match(pattern, original_function_name) + and isinstance(result, torch.Tensor) + ): + print(f"Found match for {original_function_name}") + # the first dimension is the batch size, and each corresponds to a separate response, let's try to match the batch size with the start token ids first + response_starting_indices = [] + for i in range(result.size(0)): + # try to find the match of the start token ids in the response + response = result[i] + # Find all indices where the start_token_id matches + matches = (response == GENERATE_START_TOKEN_ID).nonzero(as_tuple=True)[ + 0 + ] + indexes = matches.tolist() + if len(indexes) == 0: + # No occurrences found + print( + f"start_token_id ({GENERATE_START_TOKEN_ID}) not found in response {i}" + ) + start_index = -1 # Handle case where token is not found + elif len(indexes) > 1: + # Multiple occurrences found, raise an error + raise ValueError( + f"Multiple occurrences of start_token_id ({GENERATE_START_TOKEN_ID}) found in response {i}: {matches.tolist()}" + ) + else: + # Single occurrence found, get the index + start_index = indexes[0] + if not GENERATE_START_TOKEN_ID_INCLUDE_START_TOKEN: + start_index += 1 + + response_starting_indices.append(start_index) + + # compute the length of each response + response_lengths = [] + for i in range(result.size(0)): + response = result[i] + start_index = response_starting_indices[i] + if start_index == -1: + response_lengths.append(0) + else: + response_lengths.append(response.size(0) - start_index) + + result_to_dump = result.detach() + setattr( + result_to_dump, + "_TRAINCHECK_RESPONSE_STARTING_INDICES", + response_starting_indices, + ) + setattr(result_to_dump, "_TRAINCHECK_RESPONSE_LENGTHS", response_lengths) + + print(response_starting_indices) + print(response_lengths) + if dump_ret: + post_record["return_values"] = to_dict_return_value(result_to_dump) + dump_trace_API(post_record) if COLLECT_OVERHEAD_METRICS: EXIT_PERF_TIME = time.perf_counter() print( f"WRAPPER TIME: {original_function_name},{ORIG_EXIT_PERF_TIME - ORIG_ENTER_PERF_TIME},{EXIT_PERF_TIME - ENTER_PERF_TIME}" ) - raise e - - if trigger_var_dump: - get_global_registry().dump_modified( - dump_loc=original_function_name, dump_config=var_dump_config - ) - - post_record = { - "func_call_id": func_call_id, - "thread_id": thread_id, - "process_id": process_id, - "meta_vars": pre_meta_vars, - "type": TraceLineType.FUNC_CALL_POST, - "function": original_function_name, - "is_bound_method": is_bound_method, - "obj_id": None if not is_bound_method else id(args[0]), - } - - result_to_dump = result - - # if the current function name is transformers.generate, then we will dump the response tokens only, let's see. - # a concrete name: "transformers.models.whisper.modeling_whisper.WhisperForConditionalGeneration.generate" - # we want a pattern that abstracts the specific model name - pattern = "transformers.models.*.generate" - # find matches in the pattern - import re - - if ( - GENERATE_START_TOKEN_ID is not None - and re.match(pattern, original_function_name) - and isinstance(result, torch.Tensor) - ): - print(f"Found match for {original_function_name}") - # the first dimension is the batch size, and each corresponds to a separate response, let's try to match the batch size with the start token ids first - response_starting_indices = [] - for i in range(result.size(0)): - # try to find the match of the start token ids in the response - response = result[i] - # Find all indices where the start_token_id matches - matches = (response == GENERATE_START_TOKEN_ID).nonzero(as_tuple=True)[0] - indexes = matches.tolist() - if len(indexes) == 0: - # No occurrences found - print( - f"start_token_id ({GENERATE_START_TOKEN_ID}) not found in response {i}" - ) - start_index = -1 # Handle case where token is not found - elif len(indexes) > 1: - # Multiple occurrences found, raise an error - raise ValueError( - f"Multiple occurrences of start_token_id ({GENERATE_START_TOKEN_ID}) found in response {i}: {matches.tolist()}" - ) - else: - # Single occurrence found, get the index - start_index = indexes[0] - if not GENERATE_START_TOKEN_ID_INCLUDE_START_TOKEN: - start_index += 1 - - response_starting_indices.append(start_index) - - # compute the length of each response - response_lengths = [] - for i in range(result.size(0)): - response = result[i] - start_index = response_starting_indices[i] - if start_index == -1: - response_lengths.append(0) - else: - response_lengths.append(response.size(0) - start_index) - - result_to_dump = result.detach() - setattr( - result_to_dump, - "_TRAINCHECK_RESPONSE_STARTING_INDICES", - response_starting_indices, - ) - setattr(result_to_dump, "_TRAINCHECK_RESPONSE_LENGTHS", response_lengths) - - print(response_starting_indices) - print(response_lengths) - if dump_ret: - post_record["return_values"] = to_dict_return_value(result_to_dump) - dump_trace_API(post_record) - - if COLLECT_OVERHEAD_METRICS: - EXIT_PERF_TIME = time.perf_counter() - print( - f"WRAPPER TIME: {original_function_name},{ORIG_EXIT_PERF_TIME - ORIG_ENTER_PERF_TIME},{EXIT_PERF_TIME - ENTER_PERF_TIME}" - ) - return result + return result def core_wrapper_proxy(original_function, *args, **kwargs): """Core wrapper that only handles unproxying for built-in functions.""" - global DISABLE_WRAPPER - if DISABLE_WRAPPER: + if tracing_suppressed(): return original_function(*args, **kwargs) - args, kwargs = unproxy_args_kwargs(args, kwargs) + with suppress_tracing(): + args, kwargs = unproxy_args_kwargs(args, kwargs) return original_function(*args, **kwargs) def wrapper( - original_function, - is_bound_method, - scan_proxy_in_args, - dump_stack_trace, - disable_dump=False, - dump_args=True, - dump_args_config=None, - dump_ret=True, - dump_ret_config=None, - handle_proxy=True, - trigger_var_dump=False, - var_dump_config=None, -): + original_function: Callable[..., object], + is_bound_method: bool, + scan_proxy_in_args: bool, + dump_stack_trace: bool, + disable_dump: bool = False, + dump_args: bool = True, + dump_args_config: dict[str, object] | None = None, + dump_ret: bool = True, + dump_ret_config: dict[str, object] | None = None, + handle_proxy: bool = True, + trigger_var_dump: bool = False, + var_dump_config: dict[str, object] | None = None, +) -> Callable[..., object]: is_builtin = is_c_level_function(original_function) need_unproxy_args_kwargs = handle_proxy and ( is_builtin or is_funcs_to_be_unproxied(original_function) @@ -415,7 +471,7 @@ def wrapper( METRIC_INSTRUMENTED_FUNC_LIST["dump"].append(original_function_name) @functools.wraps(original_function) - def wrapped(*args, **kwargs): + def wrapped(*args: object, **kwargs: object) -> object: return function_wrapper( original_function, original_function_name, @@ -438,18 +494,26 @@ def wrapped(*args, **kwargs): if need_unproxy_args_kwargs: @functools.wraps(original_function) - def wrapped(*args, **kwargs): + def wrapped(*args: object, **kwargs: object) -> object: return core_wrapper_proxy(original_function, *args, **kwargs) else: @functools.wraps(original_function) - def wrapped(*args, **kwargs): + def wrapped(*args: object, **kwargs: object) -> object: return original_function(*args, **kwargs) - wrapped._traincheck_original_function = original_function - wrapped._traincheck_instrumented = True - wrapped._traincheck_dump_disabled = disable_dump + # Function attributes are runtime instrumentation metadata, attached after + # functools.wraps copies the original function's attributes. + setattr(wrapped, "_traincheck_original_function", original_function) + setattr(wrapped, "_traincheck_instrumented", True) + setattr(wrapped, "_traincheck_dump_disabled", disable_dump) + builtin_preservation = preserve_torchscript_builtin( + original_function, wrapped, torch + ) + setattr(wrapped, "_traincheck_torchscript_builtin", builtin_preservation) + if builtin_preservation["status"] == "unsupported-private-api": + warn_unsupported_torchscript() return wrapped @@ -901,9 +965,9 @@ def get_wrapped_function(self, func_obj: Callable) -> Callable: if func_name not in self.instr_opts.funcs_instr_opts: return wrapper( func_obj, - is_bound_method=None, - scan_proxy_in_args=None, - dump_stack_trace=None, + is_bound_method=False, + scan_proxy_in_args=False, + dump_stack_trace=False, disable_dump=True, handle_proxy=used_proxy, ) diff --git a/traincheck/invariant/base_cls.py b/traincheck/invariant/base_cls.py index 588e7260..d2f340d2 100644 --- a/traincheck/invariant/base_cls.py +++ b/traincheck/invariant/base_cls.py @@ -182,7 +182,7 @@ def __init__( and param.default != inspect.Parameter.empty ): self.arguments[param_name] = var_to_serializable(param.default) - + def __getstate__(self) -> dict: # ``signature`` is a cached ``inspect.Signature`` derived from # ``func_name``. It is not picklable -- its ``_parameters`` is a @@ -1677,20 +1677,49 @@ def __eq__(self, value: object) -> bool: class ExampleList: - def __init__(self, group_names: set[str]): + """A bounded sample of example occurrences, including duplicate values. + + ``num_seen`` counts the underlying stream, while ``len(self)`` counts the + retained examples. Different lists retain the historical equal-seed RNG + behavior, so their samples can be correlated; merging does not promise a + uniform joint sample across lists. + """ + + def __init__(self, group_names: set[str]) -> None: self.group_names = group_names self.examples: list[Example] = [] - # Bound memory: some relations produce an unbounded number of examples - # (see config.MAX_EXAMPLE_LIST_SIZE). Once at capacity we keep a uniform - # random subset via reservoir sampling; num_seen tracks the true number - # of examples added so the retained subset stays uniform. The RNG is - # seeded so results are deterministic and identical across the serial - # and parallel code paths. - self.max_size = config.MAX_EXAMPLE_LIST_SIZE - self.num_seen = 0 - self._rng = random.Random(config.EXAMPLE_SAMPLING_SEED) - - def add_example(self, example: Example): + self.max_size: int | None = config.MAX_EXAMPLE_LIST_SIZE + self._validate_cap(self.max_size) + self.num_seen = 0 + self._rng = random.Random(config.EXAMPLE_SAMPLING_SEED) + + @staticmethod + def _validate_cap(cap: int | None) -> None: + # Reject bool and int subclasses rather than accepting isinstance(int). + if cap is not None and (type(cap) is not int or cap < 0): # noqa: E721 + raise ValueError( + "Example reservoir capacity must be a nonnegative integer or None" + ) + + def _validate_state(self) -> None: + self._validate_cap(self.max_size) + # A population count must be a builtin integer. + if type(self.num_seen) is not int or self.num_seen < 0: # noqa: E721 + raise ValueError( + "Example reservoir population must be a nonnegative integer" + ) + expected = ( + self.num_seen + if self.max_size is None + else min(self.max_size, self.num_seen) + ) + if len(self.examples) != expected: + raise ValueError( + "Example reservoir is underfilled or has inconsistent population accounting: " + f"retained {len(self.examples)}, expected {expected} for {self.num_seen} seen" + ) + + def add_example(self, example: Example) -> None: if len(self.group_names) == 0: assert len(self.examples) == 0 self.group_names = example.get_group_names() @@ -1698,17 +1727,80 @@ def add_example(self, example: Example): assert ( example.get_group_names() == self.group_names ), f"Example groups do not match the expected group names, expected: {self.group_names}, got: {set(example.trace_groups.keys())}" - #self.examples.append(example) - self.num_seen += 1 - if self.max_size is None or len(self.examples) < self.max_size: - self.examples.append(example) - else: - # reservoir sampling: replace an existing example with probability - # max_size / num_seen, keeping self.examples a uniform random sample - j = self._rng.randrange(self.num_seen) - if j < self.max_size: + self.num_seen += 1 + if self.max_size is None or len(self.examples) < self.max_size: + self.examples.append(example) + else: + # Algorithm R retains each occurrence with probability cap / num_seen. + j = self._rng.randrange(self.num_seen) + if j < self.max_size: self.examples[j] = example + def merge(self, other: ExampleList, *, rng: random.Random) -> None: + """Append another stream's reservoir without exceeding this list's cap. + + The caller's RNG must be independent of both input selections. Share one + RNG sequentially within a merge task; do not replay its state in nested + merge operations. This preserves each occurrence's marginal inclusion + probability, even when the input reservoirs are correlated. A uniform + joint sample additionally requires independent input reservoirs. + + A sampled source cannot be expanded beyond its retained capacity or + merged into an uncapped destination: discarded examples cannot be + recovered. Invalid inputs are rejected before changing either list or + consuming the supplied RNG. Complete streams preserve concatenation + order whenever they fit in the destination. + """ + if self is other: + raise ValueError("Cannot merge an example reservoir into itself") + self._validate_state() + other._validate_state() + adopt_groups = not self.group_names and self.num_seen == 0 + source_uninitialized = not other.group_names and other.num_seen == 0 + if ( + not adopt_groups + and not source_uninitialized + and self.group_names != other.group_names + ): + raise ValueError( + "Cannot merge example reservoirs with different group names" + ) + + total = self.num_seen + other.num_seen + target = total if self.max_size is None else min(self.max_size, total) + for source in (self, other): + if len(source.examples) < min(target, source.num_seen): + raise ValueError( + "Source reservoir retains too few examples for the destination capacity" + ) + if other.num_seen == 0: + if adopt_groups: + self.group_names = other.group_names.copy() + return + + if target == total: + # Both inputs are complete: no sampling or RNG consumption is needed. + examples = self.examples + other.examples + continuation_rng = self._rng + else: + # Sampling population positions gives an exact hypergeometric split + # in O(cap) space, without expanding either represented population. + take_self = sum( + index < self.num_seen for index in rng.sample(range(total), target) + ) + examples = rng.sample(self.examples, take_self) + rng.sample( + other.examples, target - take_self + ) + # Further additions must not replay RNG draws used by a longer source + # with the same initial reservoir seed. + continuation_rng = random.Random(rng.getrandbits(256)) + + self.examples = examples + self.num_seen = total + self._rng = continuation_rng + if adopt_groups: + self.group_names = other.group_names.copy() + def get_group_from_examples(self, group_name: str) -> list[list[dict]]: return [example.get_group(group_name) for example in self.examples] @@ -1725,14 +1817,10 @@ def __len__(self): def from_iterable_of_examples( input: Iterable[Example], preset_group_names: set[str] ) -> ExampleList: - examples = [] + example_list = ExampleList(preset_group_names) for exp in input: assert preset_group_names == exp.get_group_names() - examples.append(exp) - - example_list = ExampleList(preset_group_names) - example_list.examples = examples - example_list.num_seen = len(examples) + example_list.add_example(exp) return example_list diff --git a/traincheck/invariant/contain_relation.py b/traincheck/invariant/contain_relation.py index cdc82829..b68231fc 100644 --- a/traincheck/invariant/contain_relation.py +++ b/traincheck/invariant/contain_relation.py @@ -1,10 +1,12 @@ import copy import logging import time +from random import Random from typing import Type import numpy as np +import traincheck.config.config as config from traincheck.config.config import ANALYSIS_SKIP_FUNC_NAMES from traincheck.inference_sampling import sampling_rng from traincheck.instrumentor.tracer import TraceLineType @@ -209,6 +211,15 @@ def _merge_hypotheses(hypotheses: list[Hypothesis]) -> list[Hypothesis]: assert len(hypotheses) > 0, "Expected at least one hypotheses to be merged" logger = logging.getLogger(__name__) + if any( + examples.num_seen > len(examples) + for hypothesis in hypotheses + for examples in (hypothesis.positive_examples, hypothesis.negative_examples) + ): + # This optional generalization uses set unions and differences. Reservoirs + # cannot recover the discarded values or their overlap, so keep the + # original hypotheses instead of treating retained samples as full sets. + return hypotheses pos_exp_group_names = hypotheses[0].positive_examples.get_group_names() neg_exp_group_names = hypotheses[0].negative_examples.get_group_names() logger.debug( @@ -221,9 +232,10 @@ def _merge_hypotheses(hypotheses: list[Hypothesis]) -> list[Hypothesis]: all_likelihoods = [hypo.calc_likelihood() for hypo in hypotheses] # calculate the number of positive and negative examples after merge - all_positive_examples = set() + # Insertion-ordered sets keep constructor sampling independent of hash order. + all_positive_examples: dict[Example, None] = {} all_positive_parent_examples = set() - all_negative_examples = set() + all_negative_examples: dict[Example, None] = {} for idx in range(len(hypotheses)): @@ -234,16 +246,22 @@ def _merge_hypotheses(hypotheses: list[Hypothesis]) -> list[Hypothesis]: Example({PARENT_GROUP_NAME: pre_record}) for pre_record in pre_records ] all_positive_parent_examples.update(exps_with_only_parent) - all_positive_examples.update(hypotheses[idx].positive_examples.examples) - all_negative_examples.update(hypotheses[idx].negative_examples.examples) + all_positive_examples.update( + (example, None) for example in hypotheses[idx].positive_examples.examples + ) + all_negative_examples.update( + (example, None) for example in hypotheses[idx].negative_examples.examples + ) # let's remove those negative examples that are present in the positive examples of other hypotheses - if used_dynamic_analysis: - all_negative_examples = all_negative_examples.difference(all_positive_examples) - else: - all_negative_examples = all_negative_examples.difference( - all_positive_parent_examples - ) + positive_comparison = ( + all_positive_examples if used_dynamic_analysis else all_positive_parent_examples + ) + all_negative_examples = { + example: None + for example in all_negative_examples + if example not in positive_comparison + } # calculate the likelihood of the merged hypotheses now merged_likelihood = calc_likelihood( @@ -575,15 +593,20 @@ def infer( merged_invariants = [] merged_successful_hypotheses: list[Hypothesis] = [] all_stages = trace.get_all_stages() + # Reuse one independent stream for this stage-merge task. Cross-trace + # merging uses a different domain, so nested merges do not replay draws. + merge_rng = Random( + f"traincheck-example-merge-v1:stage:{config.EXAMPLE_SAMPLING_SEED!r}" + ) for invariant in invariants_by_stage: supported_stages = set(invariants_by_stage[invariant]) hypothesis = invariants_to_hypos[invariant][0] for other_hypothesis in invariants_to_hypos[invariant][1:]: - hypothesis.positive_examples.examples.extend( - other_hypothesis.positive_examples.examples + hypothesis.positive_examples.merge( + other_hypothesis.positive_examples, rng=merge_rng ) - hypothesis.negative_examples.examples.extend( - other_hypothesis.negative_examples.examples + hypothesis.negative_examples.merge( + other_hypothesis.negative_examples, rng=merge_rng ) hypothesis.add_stage_info(supported_stages, all_stages) merged_invariants.append(hypothesis.get_invariant(all_stages)) diff --git a/traincheck/invariant/precondition.py b/traincheck/invariant/precondition.py index 45c17f5e..c6dd10d7 100644 --- a/traincheck/invariant/precondition.py +++ b/traincheck/invariant/precondition.py @@ -425,7 +425,7 @@ def find_precondition_from_single_group( negative_examples: list[list[dict]], traces: list[Trace], keys_to_skip: list[str] = [], - _pruned_clauses: set[PreconditionClause] = set(), + _pruned_clauses: set[PreconditionClause] | None = None, _skip_pruning: bool = False, _current_depth: int = 0, ) -> list[Precondition]: @@ -445,6 +445,10 @@ def find_precondition_from_single_group( To implement the invariant split OP. We need to determine how this verification / pruning process should be done, because now all the `Precondition` objects have to be violated in the negative examples. """ + # Recursive children share pruning state; independent solves start fresh. + if _pruned_clauses is None: + _pruned_clauses = set() + logger.debug( f"Calling precondition inference with \n# positive examples: {len(positive_examples)}, \n# negative examples: {len(negative_examples)}, at depth {_current_depth}" ) diff --git a/traincheck/parallel.py b/traincheck/parallel.py index 4d8bcb45..218d40d9 100644 --- a/traincheck/parallel.py +++ b/traincheck/parallel.py @@ -13,42 +13,100 @@ access the traces through ``get_worker_traces()``. """ +from collections.abc import Mapping +from typing import TypeAlias, TypedDict + import traincheck.config.config as config import traincheck.utils as _tc_utils +from traincheck.trace.trace import Trace + +RandomSeed: TypeAlias = int | float | str | bytes | bytearray | None + + +class InferenceConfigSnapshot(TypedDict): + """Required runtime settings transferred between our parent and workers.""" + + ENABLE_PRECOND_SAMPLING: bool + PRECOND_SAMPLING_THRESHOLD: int + MAX_EXAMPLE_LIST_SIZE: int | None + EXAMPLE_SAMPLING_SEED: RandomSeed + -_WORKER_TRACES: list | None = None +_WORKER_TRACES: list[Trace] | None = None -def config_snapshot() -> dict: - """CLI-overridable config values that must be propagated to workers. +def config_snapshot() -> InferenceConfigSnapshot: + """Runtime inference config values that must be propagated to workers. - These are set in the entry points' main() at runtime; workers started with - the 'spawn' method begin from a fresh interpreter and would otherwise see - only the module defaults (harmless under 'fork'). Add any future - CLI-overridable config here. + These include CLI sampling options and programmatic reservoir settings. + Workers started with 'spawn' begin from a fresh interpreter and would + otherwise see only the module defaults. Add future runtime-overridable + inference configuration here. """ return { "ENABLE_PRECOND_SAMPLING": config.ENABLE_PRECOND_SAMPLING, "PRECOND_SAMPLING_THRESHOLD": config.PRECOND_SAMPLING_THRESHOLD, + "MAX_EXAMPLE_LIST_SIZE": config.MAX_EXAMPLE_LIST_SIZE, + "EXAMPLE_SAMPLING_SEED": config.EXAMPLE_SAMPLING_SEED, + } + + +def _validate_snapshot(snapshot: Mapping[str, object]) -> InferenceConfigSnapshot: + """Reject incomplete/corrupt worker settings before changing process state.""" + required = { + "ENABLE_PRECOND_SAMPLING", + "PRECOND_SAMPLING_THRESHOLD", + "MAX_EXAMPLE_LIST_SIZE", + "EXAMPLE_SAMPLING_SEED", + } + unexpected = set(snapshot) - required + if unexpected: + raise ValueError(f"Unexpected inference settings: {sorted(unexpected)}") + # Direct access makes every field required; missing fields retain KeyError. + enabled = snapshot["ENABLE_PRECOND_SAMPLING"] + threshold = snapshot["PRECOND_SAMPLING_THRESHOLD"] + capacity = snapshot["MAX_EXAMPLE_LIST_SIZE"] + seed = snapshot["EXAMPLE_SAMPLING_SEED"] + if type(enabled) is not bool: # noqa: E721 - booleans are not integer settings + raise ValueError("ENABLE_PRECOND_SAMPLING must be a bool") + if type(threshold) is not int: # noqa: E721 - match the integer CLI contract + raise ValueError("PRECOND_SAMPLING_THRESHOLD must be an integer") + if capacity is not None and ( + type(capacity) is not int or capacity < 0 # noqa: E721 - reservoir contract + ): + raise ValueError("MAX_EXAMPLE_LIST_SIZE must be a nonnegative integer or None") + # Preserve the documented random.Random seed types, including None and bytes. + if seed is not None and not isinstance(seed, (int, float, str, bytes, bytearray)): + raise ValueError("EXAMPLE_SAMPLING_SEED must be a supported random.Random seed") + return { + "ENABLE_PRECOND_SAMPLING": enabled, + "PRECOND_SAMPLING_THRESHOLD": threshold, + "MAX_EXAMPLE_LIST_SIZE": capacity, + "EXAMPLE_SAMPLING_SEED": seed, } -def worker_init(traces: list, snapshot: dict) -> None: - """Initializer run once per worker process. +def worker_init(traces: list[Trace], snapshot: Mapping[str, object]) -> None: + """Validate required settings, then initialize this worker atomically. - Stores the traces in a module global (one copy per worker, not per task) and - restores CLI-overridable config values (see config_snapshot). Also suppresses - the per-invariant inner progress bars, otherwise every worker would fight - over the terminal. + A malformed snapshot must not install some settings or replace worker traces. + The parent supplies config_snapshot(); accepting a Mapping also lets this + process boundary validate deserialized or otherwise untrusted payloads. """ + settings = _validate_snapshot(snapshot) global _WORKER_TRACES + config.ENABLE_PRECOND_SAMPLING = settings["ENABLE_PRECOND_SAMPLING"] + config.PRECOND_SAMPLING_THRESHOLD = settings["PRECOND_SAMPLING_THRESHOLD"] + # The legacy config module infers int from defaults; programmatic reservoir + # options also support None and the other stdlib seed types. Fixed attribute + # names avoid pretending arbitrary keys are supported configuration fields. + setattr(config, "MAX_EXAMPLE_LIST_SIZE", settings["MAX_EXAMPLE_LIST_SIZE"]) + setattr(config, "EXAMPLE_SAMPLING_SEED", settings["EXAMPLE_SAMPLING_SEED"]) _WORKER_TRACES = traces - for name, value in snapshot.items(): - setattr(config, name, value) _tc_utils._suppress_inner_progress = True -def get_worker_traces() -> list: +def get_worker_traces() -> list[Trace]: """Return the traces stored by worker_init; only valid inside a worker.""" assert _WORKER_TRACES is not None, "Worker traces were not initialized" return _WORKER_TRACES diff --git a/traincheck/trace/trace.py b/traincheck/trace/trace.py index c04ebeda..d5636d29 100644 --- a/traincheck/trace/trace.py +++ b/traincheck/trace/trace.py @@ -1,4 +1,5 @@ import logging +from collections.abc import Sequence import polars as pl @@ -183,14 +184,14 @@ def get_end_time(self, process_id=None, thread_id=None) -> float: "This function should be implemented in the child class." ) - def get_process_ids(self) -> list[int]: - """Find all process ids from the trace.""" + def get_process_ids(self) -> Sequence[object]: + """Find process identities, preserving the backend scalar types.""" raise NotImplementedError( "This function should be implemented in the child class." ) - def get_thread_ids(self) -> list[int]: - """Find all thread ids from the trace.""" + def get_thread_ids(self) -> Sequence[object]: + """Find thread identities, preserving the backend scalar types.""" raise NotImplementedError( "This function should be implemented in the child class." ) diff --git a/traincheck/trace/trace_pandas.py b/traincheck/trace/trace_pandas.py index 8bb89ca7..b8d35805 100644 --- a/traincheck/trace/trace_pandas.py +++ b/traincheck/trace/trace_pandas.py @@ -1,7 +1,11 @@ import logging +import math import re +from bisect import bisect_left, bisect_right +from numbers import Real from typing import Any +import numpy as np import pandas as pd from traincheck.config import config @@ -15,6 +19,7 @@ ContextManagerState, FuncCallEvent, FuncCallExceptionEvent, + IncompleteFuncCallEvent, Liveness, VarChangeEvent, VarInstId, @@ -65,11 +70,11 @@ def __init__(self, events, truncate_incomplete_func_calls=True): self.function_pool = None self.cache_id_2_func_call_event: dict[ - str, FuncCallEvent | FuncCallExceptionEvent + str, FuncCallEvent | FuncCallExceptionEvent | IncompleteFuncCallEvent ] = {} self.all_func_call_ids: dict[str, list[str]] = {} - self.context_manager_states: dict[PTID, list[ContextManagerState]] + self.context_manager_states: dict[PTID, list[ContextManagerState]] | None self.cache_high_level_events_within_id: dict[ str, list[FuncCallEvent | FuncCallExceptionEvent | VarChangeEvent] ] = {} @@ -101,6 +106,29 @@ def __init__(self, events, truncate_incomplete_func_calls=True): "Done truncating incomplete trailing function calls from the trace." ) + # Build call-record indexes lazily, after incomplete-call truncation. + # Like the other trace caches, these assume stable event values after + # initialization. Replacing the frame or changing its length rebuilds + # this index; same-shape in-place cell edits require a new TracePandas. + self._func_call_positions_source: pd.DataFrame | None = None + self._func_call_positions_length = -1 + self._func_call_positions: dict[str, tuple[list[int], list[int]]] | None = None + + # Inference treats event identities as immutable after initialization. + # Keep the frame alive so object identity cannot be reused across frames. + self._identity_values_source: pd.DataFrame | None = None + self._identity_values_length = -1 + self._identity_values: dict[str, tuple[object, ...]] = {} + + # As with the other caches, time/type/PID/TID values, dtypes, and row + # order must remain stable. Same-size in-place edits require a new trace. + # Frame replacement or length changes invalidate this positional index. + self._func_time_source: pd.DataFrame | None = None + self._func_time_length = -1 + self._func_time_positions: ( + dict[tuple[int, int], tuple[list[int | float], list[int]]] | None + ) = None + self.column_dtypes_cached = {} self.stage_traces: dict[str, "TracePandas"] = {} @@ -283,12 +311,14 @@ def _rm_incomplete_trailing_func_calls(self): # test_dump(self.events) - def _index_context_manager_meta_vars(self): + def _index_context_manager_meta_vars( + self, + ) -> dict[PTID, list[ContextManagerState]] | None: """Identify context manager entry and exit events, and add them to the meta_vars.""" # 1. Find all trace records that are related to __enter__ and __exit__ functions if "function" not in self.events.columns: self.context_manager_states = None - return + return None if ( hasattr(self, "context_manager_states") @@ -312,15 +342,103 @@ def _index_context_manager_meta_vars(self): ) ]["function"].unique() - context_manager_init_pre_records = [] - for context_manager_name in context_manager_names: - context_manager_name = context_manager_name.removesuffix(".__enter__") - context_manager_init_pre_records.extend( - self.events[ - (self.events["function"] == f"{context_manager_name}.__init__") - & (self.events["type"] == TraceLineType.FUNC_CALL_PRE) - ].to_dict(orient="records") + # Group relevant events once, retaining their original row order. Context + # names and init records keep the old class-first encounter order. + init_functions = [ + f"{name.removesuffix('.__enter__')}.__init__" + for name in context_manager_names + ] + if not init_functions: + self.context_manager_states = all_context_managers + return None + + init_records = self.events[ + self.events["function"].isin(init_functions) + & (self.events["type"] == TraceLineType.FUNC_CALL_PRE) + ] + init_by_function = { + name: records.to_dict(orient="records") + for name, records in init_records.groupby("function", sort=False) + } + context_manager_init_pre_records = [ + record + for name in init_functions + for record in init_by_function.get(name, []) + ] + if not context_manager_init_pre_records: + self.context_manager_states = all_context_managers + return None + + context_names = [ + name.removesuffix(".__enter__") for name in context_manager_names + ] + group_columns = ["function", "type", "process_id", "thread_id", "obj_id"] + matching_events = self.events.loc[ + ( + self.events["function"].isin( + [f"{name}.__enter__" for name in context_names] + ) + & (self.events["type"] == TraceLineType.FUNC_CALL_POST) + ) + | ( + self.events["function"].isin( + [f"{name}.__exit__" for name in context_names] + ) + & (self.events["type"] == TraceLineType.FUNC_CALL_PRE) + ), + group_columns + ["time"], + ] + event_times: dict[ + tuple[object, object, object, object, object], tuple[pd.Series, bool] + ] = {} + for key, records in matching_events.groupby( + group_columns, sort=False, dropna=True, observed=True + ): + times = records["time"] + # Binary search is equivalent to the original first matching row + # only when this group's numeric times are sorted and non-null. + # Nullable extension arrays may promote UInt64 search values to + # floats and lose precision above 2**53, so keep their comparison. + ordered = ( + not isinstance(times.dtype, pd.api.extensions.ExtensionDtype) + and times.dtype.kind in "biuf" + and not times.isna().any() + and times.is_monotonic_increasing ) + event_times[key] = (times, ordered) + + # pandas object/extension columns may return arbitrary comparable scalar + # types. Keep Any only for this legacy comparison boundary: narrowing to + # float/int (or coercing) changes nullable, UInt64, and object semantics. + def first_time_after( + function: str, + event_type: str, + process_id: object, + thread_id: object, + obj_id: object, + time: Any, + ) -> Any: + if not isinstance(time, Real): + # Keep pandas validation for nonnumeric/null scalar thresholds, + # including malformed nullable timestamps, even with no group. + self.events["time"] > time + if any(pd.isna(key) for key in (process_id, thread_id, obj_id)): + # pandas equality never matches missing identity values, even + # when a custom non-null key compares equal to None. + raise IndexError("No matching context event") + group = event_times.get( + (function, event_type, process_id, thread_id, obj_id) + ) + if group is None: + raise IndexError("No matching context event") + times, ordered = group + if ordered and isinstance(time, Real) and not pd.isna(time): + position = times.searchsorted(time, side="right") + return times.iloc[position : position + 1].tolist()[0] + # Unsorted times must retain row-order selection, not nearest time. + # This also preserves pandas comparison behavior for nullable and + # nonnumeric timestamp dtypes. + return times[times > time].iloc[:1].tolist()[0] for context_manager_init_pre_record in context_manager_init_pre_records: init_pre_record = context_manager_init_pre_record @@ -329,55 +447,35 @@ def _index_context_manager_meta_vars(self): context_manager_name = init_pre_record["function"].removesuffix(".__init__") - # find nearest enter and exit events with the same obj_id + # Select the first later event in row order with the same identity. obj_id = init_pre_record["obj_id"] try: - enter_post_record = ( - self.events[ - (self.events["type"] == TraceLineType.FUNC_CALL_POST) - & ( - self.events["function"] - == f"{context_manager_name}.__enter__" - ) - & (self.events["obj_id"] == obj_id) - & (self.events["time"] > init_pre_record["time"]) - & (self.events["process_id"] == process_id) - & (self.events["thread_id"] == thread_id) - ] - .iloc[0] - .to_dict() + start_time = first_time_after( + f"{context_manager_name}.__enter__", + TraceLineType.FUNC_CALL_POST, + process_id, + thread_id, + obj_id, + init_pre_record["time"], ) - - exit_pre_record = ( - self.events[ - (self.events["type"] == TraceLineType.FUNC_CALL_PRE) - & ( - self.events["function"] - == f"{context_manager_name}.__exit__" - ) - & (self.events["obj_id"] == obj_id) - & (self.events["time"] > enter_post_record["time"]) - & (self.events["process_id"] == process_id) - & (self.events["thread_id"] == thread_id) - ] - .iloc[0] - .to_dict() + end_time = first_time_after( + f"{context_manager_name}.__exit__", + TraceLineType.FUNC_CALL_PRE, + process_id, + thread_id, + obj_id, + start_time, ) except IndexError: - # sometimes this happens. an enter might not have a corresponding exit, vice versa. e.g. torch.serialization._opener.__enter__ and torch.serialization._open_zipfile_writer_file.__exit__ + # Some traces have an unmatched entry or exit event. logger.warning( f"Context manager {context_manager_name} not used properly. Skipping." ) continue assert ( - enter_post_record["time"] < exit_pre_record["time"] - ), f"enter not before exit: {enter_post_record.to_dict()} {exit_pre_record.to_dict()}" - - start_time, end_time = ( - enter_post_record["time"], - exit_pre_record["time"], - ) + start_time < end_time + ), f"enter not before exit: {start_time} {end_time}" args = init_pre_record["args"] kwargs = init_pre_record["kwargs"] @@ -411,6 +509,7 @@ def _index_context_manager_meta_vars(self): logger.info(f"Found {len(all_context_managers)} context managers.") self.context_manager_states = all_context_managers + return None def is_stage_annotated(self): # ideally we want to have a static manifest for the trace produced by the instrumentor according to the args @@ -507,23 +606,46 @@ def get_end_time(self, process_id=None, thread_id=None) -> float: return self.events["time"].max() - def get_process_ids(self) -> list[int]: + def _get_identity_values(self, column: str) -> list[object]: + """Cache stable identity columns; return a fresh list for every caller. + + As with call-record indexes, frame replacement or length changes rebuild + the cache. Same-size in-place identity edits, row reordering, or dtype + changes require a new TracePandas. + Identity columns are schemaless: their scalars are not necessarily ints. + Keep pandas' dropna/unique conversion to preserve dtype and encounter + order instead of replacing it with Python set semantics. + """ + if ( + self._identity_values_source is not self.events + or self._identity_values_length != len(self.events) + ): + self._identity_values = {} + self._identity_values_source = self.events + self._identity_values_length = len(self.events) + if column not in self._identity_values: + self._identity_values[column] = tuple( + self.events[column].dropna().unique().tolist() + ) + return list(self._identity_values[column]) + + def get_process_ids(self) -> list[object]: """Find all process ids from the trace.""" if "process_id" not in self.events.columns: logger.warning( "process_id column not found in the events, no process related invariants will be extracted." ) return [] - return self.events["process_id"].dropna().unique().tolist() + return self._get_identity_values("process_id") - def get_thread_ids(self) -> list[int]: + def get_thread_ids(self) -> list[object]: """Find all thread ids from the trace.""" if "thread_id" not in self.events.columns: logger.warning( "thread_id column not found in the events, no thread related invariants will be extracted." ) return [] - return self.events["thread_id"].dropna().unique().tolist() + return self._get_identity_values("thread_id") def get_func_call_ids(self, func_name: str = "") -> list[str]: """Find all function call ids from the trace.""" @@ -558,7 +680,7 @@ def get_column_dtype(self, column_name: str) -> type: return self.column_dtypes_cached[column_name] filtered_values = self.events[column_name].dropna() - #filtered_values = filtered_values[filtered_values != MD_NONE()] + # filtered_values = filtered_values[filtered_values != MD_NONE()] if filtered_values.dtype == object: filtered_values = filtered_values[filtered_values != MD_NONE()] @@ -1007,13 +1129,83 @@ def query_var_changes_within_func_call( (start_time, end_time), process_id=pre_record["process_id"] ) + def _get_func_call_record_positions( + self, func_call_id: str, *, post: bool + ) -> list[int] | None: + """Return indexed positions, or None to use the original pandas filter. + + Index only ordinary strings/nulls: pandas equality for arbitrary objects + and extension dtypes can differ from Python dictionary equality. + """ + if type(func_call_id) is not str: # noqa: E721 - exclude custom equality + return None + + if ( + self._func_call_positions_source is not self.events + or self._func_call_positions_length != len(self.events) + ): + ids = self.events["func_call_id"] + event_types = self.events["type"] + positions: dict[str, tuple[list[int], list[int]]] | None = None + if all( + isinstance(column, pd.Series) + and ( + pd.api.types.is_object_dtype(column.dtype) + or isinstance(column.dtype, pd.StringDtype) + ) + for column in (ids, event_types) + ): + positions = {} + for position, (call_id, event_type) in enumerate(zip(ids, event_types)): + # None, NaN, and pd.NA cannot equal an ordinary string in + # the original pandas filter. All other nonstrings use it. + if any( + type(value) is not str # noqa: E721 - exclude custom equality + and value is not None + and value is not pd.NA + and not (isinstance(value, float) and pd.isna(value)) + for value in (call_id, event_type) + ): + positions = None + break + if ( + type(call_id) is not str # noqa: E721 - exact scalar types + or type(event_type) is not str # noqa: E721 + ): + continue + if event_type == TraceLineType.FUNC_CALL_PRE: + side = 0 + elif event_type in ( + TraceLineType.FUNC_CALL_POST, + TraceLineType.FUNC_CALL_POST_EXCEPTION, + ): + side = 1 + else: + continue + if call_id not in positions: + positions[call_id] = ([], []) + positions[call_id][side].append(position) + + self._func_call_positions = positions + self._func_call_positions_source = self.events + self._func_call_positions_length = len(self.events) + + if self._func_call_positions is None: + return None + records = self._func_call_positions.get(func_call_id) + return records[int(post)] if records is not None else [] + def get_pre_func_call_record(self, func_call_id: str) -> dict: """Get the pre call record of a function given its func_call_id.""" # TODO: check if this is correct - pre_records = self.events[ - (self.events["func_call_id"] == func_call_id) - & (self.events["type"] == TraceLineType.FUNC_CALL_PRE) - ] + positions = self._get_func_call_record_positions(func_call_id, post=False) + if positions is None: + pre_records = self.events[ + (self.events["func_call_id"] == func_call_id) + & (self.events["type"] == TraceLineType.FUNC_CALL_PRE) + ] + else: + pre_records = self.events.iloc[positions] assert ( len(pre_records) == 1 @@ -1028,17 +1220,21 @@ def get_post_func_call_record(self, func_call_id: str) -> dict | None: Returns None if the post call event is not found and the pre-call event is the outermost function call. (see the doc of `_rm_incomplete_trailing_func_calls`) """ # TODO: check if this is correct - post_records = self.events[ - (self.events["func_call_id"] == func_call_id) - & ( - self.events["type"].isin( - [ - TraceLineType.FUNC_CALL_POST, - TraceLineType.FUNC_CALL_POST_EXCEPTION, - ] + positions = self._get_func_call_record_positions(func_call_id, post=True) + if positions is None: + post_records = self.events[ + (self.events["func_call_id"] == func_call_id) + & ( + self.events["type"].isin( + [ + TraceLineType.FUNC_CALL_POST, + TraceLineType.FUNC_CALL_POST_EXCEPTION, + ] + ) ) - ) - ] + ] + else: + post_records = self.events.iloc[positions] assert ( len(post_records) <= 1 @@ -1072,6 +1268,119 @@ def get_post_func_call_record(self, func_call_id: str) -> dict | None: return post_record + def _get_func_call_time_positions( + self, time_range: object, process_id: object, thread_id: object + ) -> list[int] | None: + """Return exact positional matches, or None to retain pandas semantics. + + The fast path deliberately excludes extension/object numeric columns and + mixed scalar comparisons: pandas may coerce those differently from Python + ordering (notably integer/float comparisons above 2**53). + """ + columns = ("type", "time", "process_id", "thread_id") + if any( + name not in self.events or not isinstance(self.events[name], pd.Series) + for name in columns + ): + return None + if ( + type(time_range) is not tuple # noqa: E721 - exclude custom indexing + or len(time_range) != 2 + or type(process_id) is not int # noqa: E721 - exact numeric comparison + or type(thread_id) is not int # noqa: E721 - exact numeric comparison + or any(not -(2**63) <= value < 2**63 for value in (process_id, thread_id)) + or any( + not isinstance(self.events[name].dtype, np.dtype) + or self.events[name].dtype != "int64" + for name in columns[2:] + ) + ): + return None + start_time: object = time_range[0] + end_time: object = time_range[1] + time_dtype = self.events["time"].dtype + if not isinstance(time_dtype, np.dtype): + return None + if time_dtype == "int64": + if ( + type(start_time) is not int # noqa: E721 - exclude numeric subclasses + or type(end_time) is not int # noqa: E721 + or not -(2**63) <= start_time < 2**63 + or not -(2**63) <= end_time < 2**63 + ): + return None + elif time_dtype == "float64": + if ( + type(start_time) is not float # noqa: E721 - preserve pandas coercion + or type(end_time) is not float # noqa: E721 + or math.isnan(start_time) + or math.isnan(end_time) + ): + return None + else: + return None + + if self._func_time_source is not self.events or self._func_time_length != len( + self.events + ): + event_types = self.events["type"] + positions: ( + dict[tuple[int, int], tuple[list[int | float], list[int]]] | None + ) = None + # A custom __eq__ or extension dtype must still execute in pandas. + if ( + isinstance(event_types.dtype, np.dtype) + and pd.api.types.is_object_dtype(event_types.dtype) + and all( + type(value) is str # noqa: E721 - exclude custom equality + or value is None + or value is pd.NA + or (type(value) is float and math.isnan(value)) # noqa: E721 + for value in event_types + ) + ): + rows: dict[tuple[int, int], list[tuple[int | float, int]]] = {} + types: list[object] = event_types.tolist() + # The exact NumPy dtypes validated above make Series.tolist() + # return builtin numeric scalars; extension/object data falls back. + times: list[int | float] = self.events["time"].tolist() + processes: list[int] = self.events["process_id"].tolist() + threads: list[int] = self.events["thread_id"].tolist() + for position, (kind, time, process, thread) in enumerate( + zip(types, times, processes, threads) + ): + if type(kind) is not str or kind not in ( # noqa: E721 + TraceLineType.FUNC_CALL_PRE, + TraceLineType.FUNC_CALL_POST, + TraceLineType.FUNC_CALL_POST_EXCEPTION, + ): + continue + if time_dtype == "float64" and math.isnan(time): + continue + rows.setdefault((process, thread), []).append((time, position)) + positions = {} + for identity, records in rows.items(): + records.sort() + positions[identity] = ( + [time for time, _ in records], + [position for _, position in records], + ) + self._func_time_positions = positions + self._func_time_source = self.events + self._func_time_length = len(self.events) + + if self._func_time_positions is None: + return None + indexed = self._func_time_positions.get((process_id, thread_id)) + if indexed is None: + return [] + times, row_positions = indexed + left = bisect_right(times, start_time) + right = bisect_left(times, end_time) + # pandas groupby preserves input row order within each call. Time order + # cannot replace that order, even when traces arrive out of order. + return sorted(row_positions[left:right]) + def query_func_call_events_within_time( self, time_range: tuple[int | float, int | float], @@ -1082,25 +1391,30 @@ def query_func_call_events_within_time( # TODO: check if this is correct func_call_events: list[FuncCallEvent | FuncCallExceptionEvent] = [] - func_call_records = self.events[ - ( - self.events["type"].isin( - [ - TraceLineType.FUNC_CALL_PRE, - TraceLineType.FUNC_CALL_POST, - TraceLineType.FUNC_CALL_POST_EXCEPTION, - ] + positions = self._get_func_call_time_positions( + time_range, process_id, thread_id + ) + if positions is not None: + func_call_records = self.events.iloc[positions] + else: + func_call_records = self.events[ + ( + self.events["type"].isin( + [ + TraceLineType.FUNC_CALL_PRE, + TraceLineType.FUNC_CALL_POST, + TraceLineType.FUNC_CALL_POST_EXCEPTION, + ] + ) ) - ) - & ( - self.events["time"].between( - time_range[0], time_range[1], inclusive="neither" + & ( + self.events["time"].between( + time_range[0], time_range[1], inclusive="neither" + ) ) - ) - & (self.events["process_id"] == process_id) - & (self.events["thread_id"] == thread_id) - ] - + & (self.events["process_id"] == process_id) + & (self.events["thread_id"] == thread_id) + ] # group function calls by func_call_id func_call_groups = func_call_records.groupby("func_call_id") diff --git a/traincheck/utils.py b/traincheck/utils.py index 332cfe04..30de83bc 100644 --- a/traincheck/utils.py +++ b/traincheck/utils.py @@ -47,7 +47,7 @@ def typename_compile(o): return "compile_stage" -def typename(o, is_runtime=False): +def typename(o: object, is_runtime: bool = False) -> str: if isinstance(o, torch.nn.Parameter): return "torch.nn.Parameter" if isinstance(o, torch.Tensor): @@ -68,7 +68,7 @@ def typename(o, is_runtime=False): if not last_name: last_name = safe_getattr(o, "__name__", "") is_class_name_qualname = False - if not last_name: + if not isinstance(last_name, str) or not last_name: last_name = safe_getattr(o, "__class__", type(o)).__name__ is_class_name_qualname = False # Handle the typename logic for method descriptors. The `is_class_name_qualname` flag is used as a heuristic to determine if `o` is likely a method descriptor.